block SemrushBot which is spamming our logs

2019-03-06 13:46:52 +01:00
parent 9fbd60bc1e
commit dafb337ed1
1 changed files with 3 additions and 0 deletions
--- a/robots.txt
+++ b/robots.txt
@@ -1,2 +1,5 @@
 User-agent: *
 Disallow: /source/
+
+User-agent: SemrushBot
+Disallow: /