# ===========================================
# Dave Farley's Weblog — robots.txt
# ===========================================

# Standard search engines
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# -------------------------------------------
# AI Training Crawlers
# -------------------------------------------
# OpenAI training crawler
User-agent: GPTBot
Allow: /

# Anthropic training crawler
User-agent: ClaudeBot
Allow: /

# Google AI training (Gemini)
User-agent: Google-Extended
Allow: /

# Apple AI training (Apple Intelligence)
User-agent: Applebot-Extended
Allow: /

# Common Crawl (used by many AI projects)
User-agent: CCBot
Allow: /

# Cohere AI training
User-agent: cohere-ai
Allow: /

# Meta AI training
User-agent: meta-externalagent
Allow: /

# ByteDance/TikTok AI
User-agent: Bytespider
Allow: /

# -------------------------------------------
# AI Search & Citation Crawlers
# -------------------------------------------
# OpenAI search (powers ChatGPT search)
User-agent: OAI-SearchBot
Allow: /

# Perplexity search
User-agent: PerplexityBot
Allow: /

# DuckDuckGo AI answers
User-agent: DuckAssistBot
Allow: /

# You.com AI search
User-agent: YouBot
Allow: /

# Mistral AI
User-agent: MistralAI-User
Allow: /

# -------------------------------------------
# AI User-Triggered Fetchers
# -------------------------------------------
# ChatGPT fetching pages for a user
User-agent: ChatGPT-User
Allow: /

# Perplexity fetching pages for a user
User-agent: Perplexity-User
Allow: /

# -------------------------------------------
# Social & Preview Crawlers
# -------------------------------------------
User-agent: LinkedInBot
Allow: /

User-agent: FacebookBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Amazonbot
Allow: /

# -------------------------------------------
# Default: Allow all other crawlers
# -------------------------------------------
User-agent: *
Allow: /

# -------------------------------------------
# Sitemaps
# -------------------------------------------