diff --git a/robots.txt b/robots.txt index 482af66..65cd83e 100644 --- a/robots.txt +++ b/robots.txt @@ -1,5 +1,13 @@ # https://www.robotstxt.org/ -# Deny crawling of all content -User-agent: * +User-Agent: GPTBot +User-Agent: ClaudeBot +User-agent: Claude-Web +User-Agent: PerplexityBot +User-Agent: CCBot +User-Agent: Google-Extended +Disallow: / + +User-agent: Yandex +User-agent: Baiduspider Disallow: /