# ============================================================
# SITEMAP
# The Sitemap: line is injected automatically by
# eliashaeussler/typo3-sitemap-robots, but ONLY when the site's config.yaml
# sets `sitemap_robots_inject: all` (or `default`). EXT:seo provides the
# sitemap file; enabling it alone is NOT enough to add the line here.
# Uncomment and edit the line below only to set a sitemap URL manually.
# ============================================================
# Sitemap: https://www.example.com/sitemap.xml

# ============================================================
# DEFAULT BLOCK (all unknown bots are blocked)
# ============================================================
User-agent: *
Disallow: /

# ============================================================
# SOCIAL MEDIA
# No path restrictions: this agent fetches a URL it was handed and
# renders it to build the share preview, so it needs the images -
# including the derivatives under /fileadmin/_processed_/.
# ============================================================
User-agent: facebookexternalhit
Allow: /

# ============================================================
# TAG MANAGEMENT / ANALYTICS
# Stape = allowed - server-side tag management. Its scanner checks
# that the site owner's own tags and pixels fire correctly, and
# renders the page (CSS, images, video) to do so. Same shape of job
# as facebookexternalhit above, so same unrestricted access: it
# fetches URLs it was handed rather than crawling the site.
# ============================================================
User-agent: Stape
Allow: /

# ============================================================
# CLASSIC SEARCH ENGINES
# ============================================================
User-agent: Googlebot
Allow: /
Disallow: /typo3/
Disallow: /typo3conf/
Disallow: /typo3temp/
Disallow: /vendor/
Disallow: /fileadmin/_processed_/
Disallow: /*tx_solr
Disallow: /*?q=


User-agent: bingbot
Allow: /
Disallow: /typo3/
Disallow: /typo3conf/
Disallow: /typo3temp/
Disallow: /vendor/
Disallow: /fileadmin/_processed_/
Disallow: /*tx_solr
Disallow: /*?q=

# ============================================================
# ANTHROPIC (Claude)
# ClaudeBot = BLOCKED - trains AI models on your content
# Claude-SearchBot = allowed - indexes for Claude's search results
# Claude-User = allowed - fetches pages on user requests
# ============================================================
User-agent: ClaudeBot
Disallow: /

User-agent: Claude-SearchBot
Allow: /
Disallow: /typo3/
Disallow: /typo3conf/
Disallow: /typo3temp/
Disallow: /vendor/
Disallow: /fileadmin/_processed_/
Disallow: /*tx_solr
Disallow: /*?q=
Crawl-delay: 2

User-agent: Claude-User
Allow: /
Disallow: /typo3/
Disallow: /typo3conf/
Disallow: /typo3temp/
Disallow: /vendor/
Disallow: /fileadmin/_processed_/
Disallow: /*tx_solr
Disallow: /*?q=

# ============================================================
# OPENAI (ChatGPT)
# GPTBot = BLOCKED - trains AI models on your content
# OAI-SearchBot = allowed - indexes for ChatGPT search results
# ChatGPT-User = allowed - fetches pages on user requests
# ============================================================
User-agent: GPTBot
Disallow: /

User-agent: OAI-SearchBot
Allow: /
Disallow: /typo3/
Disallow: /typo3conf/
Disallow: /typo3temp/
Disallow: /vendor/
Disallow: /fileadmin/_processed_/
Disallow: /*tx_solr
Disallow: /*?q=
Crawl-delay: 2

User-agent: ChatGPT-User
Allow: /
Disallow: /typo3/
Disallow: /typo3conf/
Disallow: /typo3temp/
Disallow: /vendor/
Disallow: /fileadmin/_processed_/
Disallow: /*tx_solr
Disallow: /*?q=

# ============================================================
# GOOGLE (Gemini / AI Overviews)
# Google-Extended = BLOCKED - used for Google's AI training
# ============================================================
User-agent: Google-Extended
Disallow: /

# ============================================================
# PERPLEXITY AI
# PerplexityBot = allowed - indexes for Perplexity's search results
# ============================================================
User-agent: PerplexityBot
Allow: /
Disallow: /typo3/
Disallow: /typo3conf/
Disallow: /typo3temp/
Disallow: /vendor/
Disallow: /fileadmin/_processed_/
Disallow: /*tx_solr
Disallow: /*?q=
Crawl-delay: 2