# Commercial Lending X — https://www.commerciallendingx.com
# AI assistants: /llms.txt holds a structured summary of this site.
#
# Note: robots.txt groups do NOT inherit. A crawler that matches a named
# User-agent below ignores the "*" group entirely, so every group repeats
# the same Disallow list. Keep them in sync when editing.
#
# Also note: src/lib/sitemap/get-static-pages.ts reads every Disallow line
# in this file without regard to which User-agent it sits under, and drops
# those paths from sitemap.xml. Adding a new prefix to any group below
# removes it from the sitemap for all crawlers.

User-agent: *
Allow: /
Disallow: /api/
Disallow: /_private/
Disallow: /unsubscribe
Disallow: /test
Disallow: /test2
Disallow: /resources/

# OpenAI — training, ChatGPT search, user-initiated fetches, ad review.
# https://developers.openai.com/api/docs/bots
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: OAI-AdsBot
Allow: /
Disallow: /api/
Disallow: /_private/
Disallow: /unsubscribe
Disallow: /test
Disallow: /test2
Disallow: /resources/

# Anthropic — training, Claude search, user-initiated fetches.
# https://support.claude.com/en/articles/8896518
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
Allow: /
Disallow: /api/
Disallow: /_private/
Disallow: /unsubscribe
Disallow: /test
Disallow: /test2
Disallow: /resources/

# Perplexity — search index and user-initiated fetches.
# https://docs.perplexity.ai/guides/bots
User-agent: PerplexityBot
User-agent: Perplexity-User
Allow: /
Disallow: /api/
Disallow: /_private/
Disallow: /unsubscribe
Disallow: /test
Disallow: /test2
Disallow: /resources/

# Other AI assistants and answer engines.
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: Amazonbot
User-agent: DuckAssistBot
User-agent: MistralAI-User
User-agent: cohere-ai
User-agent: YouBot
Allow: /
Disallow: /api/
Disallow: /_private/
Disallow: /unsubscribe
Disallow: /test
Disallow: /test2
Disallow: /resources/

Sitemap: https://www.commerciallendingx.com/sitemap.xml