block SemrushBot which is spamming our logs

4 years ago · dafb337ed1
parent 9fbd60bc1e
commit dafb337ed1
1 changed files with 3 additions and 0 deletions
--- a/robots.txt
+++ b/robots.txt
@ -1,2 +1,5 @@
 User-agent: *
 Disallow: /source/
+
+User-agent: SemrushBot
+Disallow: /