# Chrome Goldmine - https://chromegoldmine.com
# Robots.txt — Updated 2026-07-20
# LLM-friendly: training, search, and crawling are all allowed.
# Content Signals: https://contentsignals.org/

# OpenAI
User-agent: GPTBot
Allow: /
Content-Signal: ai-train=yes, search=yes, ai-input=yes

User-agent: OAI-SearchBot
Allow: /
Content-Signal: ai-train=yes, search=yes, ai-input=yes

User-agent: ChatGPT-User
Allow: /
Content-Signal: ai-train=yes, search=yes, ai-input=yes

# Anthropic
User-agent: ClaudeBot
Allow: /
Content-Signal: ai-train=yes, search=yes, ai-input=yes

# Perplexity
User-agent: PerplexityBot
Allow: /
Content-Signal: ai-train=yes, search=yes, ai-input=yes

# Google AI
User-agent: Google-Extended
Allow: /
Content-Signal: ai-train=yes, search=yes, ai-input=yes

# Common Crawl (used by many AI projects)
User-agent: CCBot
Allow: /
Content-Signal: ai-train=yes, search=yes, ai-input=yes

# All other bots (including search engines and AI crawlers)
User-agent: *
Allow: /
Disallow: /admin/
Content-Signal: ai-train=yes, search=yes, ai-input=yes

# Additional AI / LLM crawlers — explicitly allowed
User-agent: Applebot
Allow: /
Content-Signal: ai-train=yes, search=yes, ai-input=yes

User-agent: Meta-ExternalAgent
Allow: /
Content-Signal: ai-train=yes, search=yes, ai-input=yes

User-agent: Amazonbot
Allow: /
Content-Signal: ai-train=yes, search=yes, ai-input=yes

User-agent: cohere-ai
Allow: /
Content-Signal: ai-train=yes, search=yes, ai-input=yes

# Sitemaps
Sitemap: https://chromegoldmine.com/sitemap.xml

# LLM / AI context files
# Navigational summary: https://chromegoldmine.com/llms.txt          (mirror: /.well-known/llms.txt)
# Full-content dump:    https://chromegoldmine.com/llms-full.txt     (mirror: /.well-known/llms-full.txt)
# AI usage policy:      https://chromegoldmine.com/ai.txt            (mirror: /.well-known/ai.txt)