# actuary.info robots.txt
# Strategy: Maximum AI visibility — allow all crawlers
# Updated: 2026-03-26

# ——— TRADITIONAL SEARCH ENGINES ———
User-agent: Googlebot
Allow: /
Disallow: /go/

User-agent: Bingbot
Allow: /
Disallow: /go/

# ——— OPENAI (ChatGPT) ———
# OAI-SearchBot: powers ChatGPT search results (not used for training)
User-agent: OAI-SearchBot
Allow: /

# ChatGPT-User: user-initiated browsing from ChatGPT conversations
User-agent: ChatGPT-User
Allow: /

# GPTBot: training data collection for future models
User-agent: GPTBot
Allow: /

# ——— ANTHROPIC (Claude) ———
# Claude-SearchBot: powers Claude search citations
User-agent: Claude-SearchBot
Allow: /

# Claude-User: user-initiated fetches from Claude conversations
User-agent: Claude-User
Allow: /

# ClaudeBot: training data collection
User-agent: ClaudeBot
Allow: /

# ——— PERPLEXITY ———
# PerplexityBot: search index builder
User-agent: PerplexityBot
Allow: /

# Perplexity-User: real-time retrieval for user queries
User-agent: Perplexity-User
Allow: /

# ——— GOOGLE AI / GEMINI ———
# Google-Extended: Gemini training (separate from Googlebot search index)
User-agent: Google-Extended
Allow: /

# ——— APPLE INTELLIGENCE ———
User-agent: Applebot-Extended
Allow: /

# ——— META AI ———
User-agent: FacebookBot
Allow: /

User-agent: meta-externalagent
Allow: /

# ——— OTHER AI CRAWLERS ———
User-agent: Amazonbot
Allow: /

User-agent: CCBot
Allow: /

User-agent: cohere-ai
Allow: /

# ——— DEFAULT ———
User-agent: *
Allow: /

# Internal paths — not useful for search or AI
Disallow: /data/
Disallow: /scraper/
Disallow: /archive/

# Outbound affiliate/app redirects (302 to the App Store) — no index value,
# and they show up in GSC as "Page with redirect". Keep crawlers off them.
Disallow: /go/

# Sitemap (trailing slash required for Netlify + GSC compatibility)
Sitemap: https://actuary.info/sitemap.xml/