robots: Exclusion IA bots

2025-11-05 10:17:13 +01:00
parent 11dfbc7dfc
commit a0a2c0061b
1 changed files with 10 additions and 2 deletions
--- a/robots.txt
+++ b/robots.txt
@@ -1,5 +1,13 @@
 # https://www.robotstxt.org/

-# Deny crawling of all content
-User-agent: *
+User-Agent: GPTBot
+User-Agent: ClaudeBot
+User-agent: Claude-Web
+User-Agent: PerplexityBot
+User-Agent: CCBot
+User-Agent: Google-Extended
+Disallow: /
+
+User-agent: Yandex
+User-agent: Baiduspider
 Disallow: /