fix: update robots.txt (#1049)

block bad bots using up bandwidth for no reason
styfle · Jun 10, 2024 · 0b55683 · 0b55683
1 parent d47ae4b
commit 0b55683
Showing 1 changed file with 22 additions and 1 deletion.
diff --git a/public/robots.txt b/public/robots.txt
@@ -1 +1,22 @@
-User-agent: *
+User-agent: AhrefsBot
+Disallow: /
+User-agent: Barkrowler
+Disallow: /
+User-agent: Bytespider
+Disallow: /
+User-agent: DotBot
+Disallow: /
+User-agent: GPTBot
+Disallow: /
+User-agent: marketgoo
+Disallow: /
+User-agent: MJ12bot
+Disallow: /
+User-agent: Moz dotbot
+Disallow: /
+User-agent: PetalBot
+Disallow: /
+User-agent: prerender
+Disallow: /
+User-agent: SEMrush
+Disallow: /