# Crawlers are managed per purpose, not with one switch. Search, live user
# requests and model training are three different jobs done by three different
# sets of bots, and they get three different answers.
#
# A named User-agent group replaces the `*` group for that bot rather than
# adding to it, so every group below repeats the endpoint exclusions.

# Default.
User-agent: *
Disallow: /api/
Disallow: /admin/

# Search. These bots make a page eligible to be cited in ChatGPT Search, Claude
# and Perplexity. Blocking them opts us out of AI search entirely.
User-agent: OAI-SearchBot
Disallow: /api/
Disallow: /admin/

User-agent: Claude-SearchBot
Disallow: /api/
Disallow: /admin/

User-agent: PerplexityBot
Disallow: /api/
Disallow: /admin/

# Live user requests: someone asks an assistant about one of our pages and it
# fetches that page there and then. Not search, not training.
User-agent: ChatGPT-User
Disallow: /api/
Disallow: /admin/

User-agent: Claude-User
Disallow: /api/
Disallow: /admin/

# Model training. Allowed deliberately — we are happy for our public writing to
# be trained on. Change these to `Disallow: /` to opt out. Doing so does not
# affect whether we get cited in AI search, which the search bots above govern.
User-agent: GPTBot
Disallow: /api/
Disallow: /admin/

User-agent: ClaudeBot
Disallow: /api/
Disallow: /admin/

User-agent: CCBot
Disallow: /api/
Disallow: /admin/

# Google-Extended controls whether Google may train on or ground its AI in our
# content. It does not affect indexing or AI Overviews eligibility, which normal
# Googlebot indexing governs.
User-agent: Google-Extended
Disallow: /api/
Disallow: /admin/

Sitemap: https://crabnebula.dev/sitemap.xml