{"total":13,"bots":[{"name":"GPTBot","class":"training","vendor":"OpenAI","purpose":"Collects public web content for OpenAI model training.","control_token_only":false,"user_agent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)","docs_url":"https://platform.openai.com/docs/bots"},{"name":"ClaudeBot","class":"training","vendor":"Anthropic","purpose":"Collects public web content for Anthropic model training.","control_token_only":false,"user_agent":"Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)","docs_url":"https://support.anthropic.com/en/articles/8896518"},{"name":"CCBot","class":"training","vendor":"Common Crawl","purpose":"Open crawling dataset used by many model builders.","control_token_only":false,"user_agent":"Mozilla/5.0 (compatible; CCBot/2.0; +https://commoncrawl.org/faq/)","docs_url":"https://commoncrawl.org/faq/"},{"name":"Meta-ExternalAgent","class":"training","vendor":"Meta","purpose":"Collects public web content for Meta AI models.","control_token_only":false,"user_agent":"Mozilla/5.0 (compatible; meta-externalagent/1.1; +https://developers.facebook.com/docs/sharing/webmasters/crawler)","docs_url":"https://developers.facebook.com/docs/sharing/webmasters/crawler"},{"name":"Bytespider","class":"training","vendor":"ByteDance","purpose":"Collects public web content for ByteDance models.","control_token_only":false,"user_agent":"Mozilla/5.0 (compatible; Bytespider; spider-feedback@bytedance.com)","docs_url":"https://www.bytedance.com/en/robots"},{"name":"Google-Extended","class":"training","vendor":"Google","purpose":"Robots.txt control token that opts a site out of Gemini training.","control_token_only":true,"user_agent":null,"docs_url":"https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers"},{"name":"Applebot-Extended","class":"training","vendor":"Apple","purpose":"Robots.txt control token that opts a site out of Apple Intelligence training.","control_token_only":true,"user_agent":null,"docs_url":"https://support.apple.com/en-us/119829"},{"name":"OAI-SearchBot","class":"search","vendor":"OpenAI","purpose":"Indexes pages for ChatGPT search results.","control_token_only":false,"user_agent":"Mozilla/5.0 (compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot)","docs_url":"https://platform.openai.com/docs/bots"},{"name":"Claude-SearchBot","class":"search","vendor":"Anthropic","purpose":"Indexes pages so Claude can cite them.","control_token_only":false,"user_agent":"Mozilla/5.0 (compatible; Claude-SearchBot/1.0; +claudesearchbot@anthropic.com)","docs_url":"https://support.anthropic.com/en/articles/8896518"},{"name":"PerplexityBot","class":"search","vendor":"Perplexity","purpose":"Indexes pages for Perplexity search results.","control_token_only":false,"user_agent":"Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)","docs_url":"https://docs.perplexity.ai/guides/bots"},{"name":"ChatGPT-User","class":"user","vendor":"OpenAI","purpose":"Fetches a page when a person asks ChatGPT about it.","control_token_only":false,"user_agent":"Mozilla/5.0 (compatible; ChatGPT-User/1.0; +https://openai.com/bot)","docs_url":"https://platform.openai.com/docs/bots"},{"name":"Claude-User","class":"user","vendor":"Anthropic","purpose":"Fetches a page when a person asks Claude about it.","control_token_only":false,"user_agent":"Mozilla/5.0 (compatible; Claude-User/1.0; +Anthropic-AI-User)","docs_url":"https://support.anthropic.com/en/articles/8896518"},{"name":"Perplexity-User","class":"user","vendor":"Perplexity","purpose":"Fetches a page when a person asks Perplexity about it.","control_token_only":false,"user_agent":"Mozilla/5.0 (compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)","docs_url":"https://docs.perplexity.ai/guides/bots"}]}