# Reroute robots policy.
#
# Every AI crawler is allowed, deliberately and by name. Being quoted by an
# answer engine is free distribution for a pre-launch product, and everything
# here is public transit information we don't paywall. See docs/seo/README.md.
#
# The named blocks are NOT redundant with the wildcard block. A matching
# `User-agent` block fully REPLACES the wildcard block for that agent, so each
# one has to repeat the /admin and /dev disallows. Deleting them to "simplify"
# would expose the admin routes to every crawler listed below.
# apps/web/app/robots.test.ts guards this.

User-agent: *
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /dev
Disallow: /dev/

# OpenAI: GPTBot trains, OAI-SearchBot powers ChatGPT search, ChatGPT-User
# fetches a page live when a user asks about it.
User-agent: GPTBot
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /dev
Disallow: /dev/

User-agent: OAI-SearchBot
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /dev
Disallow: /dev/

User-agent: ChatGPT-User
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /dev
Disallow: /dev/

# Anthropic.
User-agent: ClaudeBot
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /dev
Disallow: /dev/

User-agent: Claude-User
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /dev
Disallow: /dev/

# Claude-SearchBot indexes pages for Claude's search results; ClaudeBot
# (training) and Claude-User (live fetch) are separate agents.
User-agent: Claude-SearchBot
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /dev
Disallow: /dev/

User-agent: PerplexityBot
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /dev
Disallow: /dev/

# Perplexity-User fetches a page live when a Perplexity user asks about it.
User-agent: Perplexity-User
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /dev
Disallow: /dev/

# Google-Extended controls training and grounding in some other Google AI
# systems. Googlebot controls Search, including AI Overviews and AI Mode.
User-agent: Google-Extended
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /dev
Disallow: /dev/

User-agent: Applebot-Extended
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /dev
Disallow: /dev/

User-agent: meta-externalagent
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /dev
Disallow: /dev/

# Common Crawl. Feeds a large share of every open training corpus, and is also
# where several answer engines source historical page text.
User-agent: CCBot
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /dev
Disallow: /dev/

# Bing indexes for Bing AND backs ChatGPT's web results, so this line is GEO
# work, not only Bing traffic.
User-agent: Bingbot
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /dev
Disallow: /dev/

Sitemap: https://rerouteapp.ca/sitemap.xml