User-agent: ia_archiver
Disallow: /

User-agent: *
Disallow: /wip/

# ---------------------------------------
# Verifiable Robots.txt
# Purpose: Allow AI crawlers (OpenAI, Perplexity, Claude, Gemini)
# and all major search engines to access public content
# ---------------------------------------
# Allow OpenAI's GPTBot (used for ChatGPT)
User-agent: GPTBot
Allow: /
# Allow OpenAI's ChatGPT-User crawler
User-agent: ChatGPT-User
Allow: /
# Allow Anthropic's ClaudeBot
User-agent: ClaudeBot
Allow: /
# Allow Google's AI extension (Gemini/Bard data gathering)
User-agent: Google-Extended
Allow: /
# Allow Perplexity.ai's crawler
User-agent: PerplexityBot
Allow: /
# Allow all other crawlers by default
User-agent: *
Allow: /