# The Caddie Book: robots.txt
#
# Public policy: https://thecaddiebook.com/terms
# We welcome known search and citation crawlers on public pages. We do not
# authorize model-training crawlers, general-purpose AI collection, or
# user-directed AI fetch agents. Private and token-bearing routes are excluded
# from all crawlers that are otherwise allowed.

# Traditional search and AI search/citation.
User-agent: Googlebot
User-agent: Bingbot
User-agent: DuckDuckBot
User-agent: Applebot
User-agent: OAI-SearchBot
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: Amzn-SearchBot
Allow: /
Disallow: /account
Disallow: /admin/
Disallow: /internal/
Disallow: /join/
Disallow: /list/
Disallow: /lists
Disallow: /saved
Disallow: /savedhomepage$
Disallow: /submit/

# Training and general-purpose AI collection.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: CCBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: Amazonbot
User-agent: Bytespider
User-agent: Diffbot
User-agent: Omgilibot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: PanguBot
User-agent: Timpibot
Disallow: /

# User-directed AI fetchers.
User-agent: ChatGPT-User
User-agent: Claude-User
User-agent: Perplexity-User
User-agent: Amzn-User
User-agent: Google-CloudVertexBot
User-agent: Google-GeminiNotebook
Disallow: /

# Unknown automation is blocked by default. This does not affect human browsers.
User-agent: *
Disallow: /

Sitemap: https://thecaddiebook.com/sitemap.xml