Update robots...

skyeto · Dec 21, 2024 · 4cdb1a2 · 4cdb1a2
1 parent a886dd0
commit 4cdb1a2
Showing 1 changed file with 45 additions and 0 deletions.
diff --git a/public/robots.txt b/public/robots.txt
@@ -1,3 +1,42 @@
+User-agent: Applebot
+Allow: /
+
+User-agent: baiduspider
+Allow: /
+
+User-agent: Bingbot
+Allow: /
+
+User-agent: Facebot
+Allow: /
+
+User-agent: Googlebot
+Allow: /
+
+User-agent: msnbot
+Allow: /
+
+User-agent: Naverbot
+Allow: /
+
+User-agent: seznambot
+Allow: /
+
+User-agent: Slurp
+Allow: /
+
+User-agent: teoma
+Allow: /
+
+User-agent: Twitterbot
+Allow: /
+
+User-agent: Yandex
+Allow: /
+
+User-agent: Yeti
+Allow: /
+
 User-agent: ChatGPT-User
 Disallow: /
 
@@ -6,3 +45,9 @@ Disallow: /
 
 User-agent: OpenAI
 Disallow: /
+
+User-agent: ClaudeBot
+Disallow: /
+
+User-agent: *
+Disallow: /