# careranks.com robots.txt
#
# Policy (2026-08-02, see Drafts/SEO_ASSESSMENT_2026-08-02.md):
#   - Search engines: full crawl access except internal search + previews.
#   - AI retrieval / answer engines: welcome (the site ships llms.txt,
#     markdown twins, and JSON data endpoints for exactly these consumers).
#   - AI training-corpus crawlers: blocked.
# Cloudflare's managed "block AI bots" robots.txt injection must stay OFF —
# this file is the authority. If a managed block reappears in the live
# response, check the Cloudflare dashboard toggle.

User-agent: *
Disallow: /search
Disallow: /components-preview
Allow: /

# --- AI retrieval / answer engines: explicitly allowed ---
# Each group re-states BOTH wildcard disallows: a matching specific group
# replaces the * group entirely (2026-08-02 Codex review finding 10).
User-agent: ClaudeBot
Disallow: /search
Disallow: /components-preview
Allow: /

User-agent: OAI-SearchBot
Disallow: /search
Disallow: /components-preview
Allow: /

User-agent: PerplexityBot
Disallow: /search
Disallow: /components-preview
Allow: /

User-agent: Applebot
Disallow: /search
Disallow: /components-preview
Allow: /

# --- AI training-corpus crawlers: blocked ---
User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: CCBot
Disallow: /

# Curated (Google-facing) index. The full index, sitemap-index-nongoogle.xml,
# is submitted directly in Bing Webmaster Tools and pushed via IndexNow —
# deliberately NOT advertised here.
Sitemap: https://careranks.com/sitemap-index.xml