# LLMs: https://marcopontili.com/llms.txt

User-agent: *
# Content Signals (IETF draft): search and AI answers are welcome, model training is not.
Content-Signal: ai-train=no, search=yes, ai-input=yes
Crawl-delay: 2
Disallow: /404
Disallow: /404.php
Disallow: /500
Disallow: /500.php
Disallow: /thank-you
Disallow: /thank-you.php
Disallow: /php-includes/
# /imprint, /privacy-policy and /cookie-policy are deliberately crawlable here:
# they send `noindex, follow`, and a crawler that may not fetch a URL never reads
# its noindex. Disallowing them would leave the URLs index-eligible from the
# footer links instead of dropping them.

# AI crawlers - allowed (max citation visibility).
# Every AI group repeats the generic legal/utility disallows and adds /imprint
# and /cookie-policy, which carry the postal address and the tax/VAT numbers.
# /privacy-policy stays fetchable as a trust anchor (/privacy redirects there).
# Content-Signal is repeated per group on purpose: RFC 9309 has a crawler obey
# only its most specific matching group, so a signal left in `*` alone would
# never reach any of the agents named below.

# OpenAI: GPTBot (training), OAI-SearchBot (ChatGPT Search), ChatGPT-User (user-triggered)
User-agent: GPTBot
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /
Disallow: /imprint
Disallow: /cookie-policy
Disallow: /thank-you
Disallow: /php-includes/

User-agent: OAI-SearchBot
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /
Disallow: /imprint
Disallow: /cookie-policy
Disallow: /thank-you
Disallow: /php-includes/

User-agent: ChatGPT-User
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /
Disallow: /imprint
Disallow: /cookie-policy
Disallow: /thank-you
Disallow: /php-includes/

# Anthropic: ClaudeBot (training), Claude-SearchBot (Claude search), Claude-User (user-triggered)
User-agent: ClaudeBot
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /
Disallow: /imprint
Disallow: /cookie-policy
Disallow: /thank-you
Disallow: /php-includes/

User-agent: Claude-SearchBot
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /
Disallow: /imprint
Disallow: /cookie-policy
Disallow: /thank-you
Disallow: /php-includes/

User-agent: Claude-User
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /
Disallow: /imprint
Disallow: /cookie-policy
Disallow: /thank-you
Disallow: /php-includes/

# Google (Gemini/Vertex training opt-in token)
User-agent: Google-Extended
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /
Disallow: /imprint
Disallow: /cookie-policy
Disallow: /thank-you
Disallow: /php-includes/

User-agent: Google-CloudVertexBot
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /
Disallow: /imprint
Disallow: /cookie-policy
Disallow: /thank-you
Disallow: /php-includes/

# Perplexity: PerplexityBot (index), Perplexity-User (user-triggered)
User-agent: PerplexityBot
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /
Disallow: /imprint
Disallow: /cookie-policy
Disallow: /thank-you
Disallow: /php-includes/

User-agent: Perplexity-User
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /
Disallow: /imprint
Disallow: /cookie-policy
Disallow: /thank-you
Disallow: /php-includes/

# Apple Intelligence training opt-in token
User-agent: Applebot-Extended
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /
Disallow: /imprint
Disallow: /cookie-policy
Disallow: /thank-you
Disallow: /php-includes/

# Other answer/assistant crawlers
User-agent: Amazonbot
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /
Disallow: /imprint
Disallow: /cookie-policy
Disallow: /thank-you
Disallow: /php-includes/

User-agent: Meta-ExternalAgent
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /
Disallow: /imprint
Disallow: /cookie-policy
Disallow: /thank-you
Disallow: /php-includes/

User-agent: MistralAI-User
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /
Disallow: /imprint
Disallow: /cookie-policy
Disallow: /thank-you
Disallow: /php-includes/

User-agent: DuckAssistBot
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /
Disallow: /imprint
Disallow: /cookie-policy
Disallow: /thank-you
Disallow: /php-includes/

User-agent: DeepSeekBot
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /
Disallow: /imprint
Disallow: /cookie-policy
Disallow: /thank-you
Disallow: /php-includes/

User-agent: ora-agent
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /
Disallow: /imprint
Disallow: /cookie-policy
Disallow: /thank-you
Disallow: /php-includes/

# Common Crawl (feeds many downstream models) - allowed for reach
User-agent: CCBot
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /
Disallow: /imprint
Disallow: /cookie-policy
Disallow: /thank-you
Disallow: /php-includes/

Sitemap: https://marcopontili.com/sitemap.xml