# Licentie (RSL 1.0): https://www.citeerbaar.nl/license.xml (ook via de Link-header).

# robots.txt voor citeerbaar.nl
#
# 🛑 Elke groep herhaalt de uitsluitingen bewust. Volgens RFC 9309 gebruikt een
#    crawler UITSLUITEND de meest specifieke user-agentgroep die op hem past;
#    regels uit 'User-agent: *' worden NIET overgenomen. Een specifieke groep met
#    alleen 'Allow: /' schakelt de uitsluitingen hieronder dus stilzwijgend uit.
#    Wie hier een crawler toevoegt: kopieer het hele blok, niet alleen de naam.
#
# Toelichting per crawler: https://www.citeerbaar.nl/blog/robots-txt-ai-crawlers/
#
# Content-Signal (https://contentsignals.org/): zoeken, gebruik in AI-antwoorden en
# training zijn toegestaan, gelijk aan de licentie hierboven.

# ---------------------------------------------------------------------------
# Volledige toegang tot de publieke site (zoekmachines, AI-search, fetchers).
# ---------------------------------------------------------------------------

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# ChatGPT search
User-agent: OAI-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# ChatGPT user-initiated browsing
User-agent: ChatGPT-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# OpenAI training en plugins
User-agent: GPTBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# OpenAI validatie ad-landingpages (ChatGPT Ads)
User-agent: OAI-AdsBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Perplexity search
User-agent: PerplexityBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Perplexity user-initiated browsing
User-agent: Perplexity-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Anthropic training en Claude analysis
User-agent: ClaudeBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Claude user-initiated browsing
User-agent: Claude-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Claude search
User-agent: Claude-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Anthropic legacy training-token
User-agent: anthropic-ai
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Anthropic legacy alias
User-agent: Claude-Web
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Mistral search-index (Le Chat / Vibe)
User-agent: MistralAI-Index
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Mistral user-initiated browsing
User-agent: MistralAI-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Google Search + AI Overviews + AI Mode
User-agent: Googlebot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Gemini training en grounding
User-agent: Google-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Google AI research
User-agent: GoogleOther
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Google AI research (afbeeldingen)
User-agent: GoogleOther-Image
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Google AI research (video)
User-agent: GoogleOther-Video
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Vertex AI Agents (alleen op verzoek)
User-agent: Google-CloudVertexBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Google agentic browsing (Project Mariner)
User-agent: Google-Agent
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# NotebookLM bron-ophalen op gebruikersverzoek
User-agent: Google-NotebookLM
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Google test-tools (Rich Result Test / URL-inspectie)
User-agent: Google-InspectionTool
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Bing + Microsoft Copilot
User-agent: bingbot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Bing legacy alias
User-agent: MSNBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Bing legacy preview-alias
User-agent: BingPreview
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Siri en Spotlight
User-agent: Applebot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Apple Intelligence training
User-agent: Applebot-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# DuckDuckGo AI-assist answer engine
User-agent: DuckAssistBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# You.com answer engine
User-agent: YouBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Kagi search en Kagi Assistant
User-agent: Kagibot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Exa search-index (zoekmachine voor AI-agents)
User-agent: ExaSearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Exa ophalen op verzoek van een gebruiker
User-agent: ExaUser
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Alexa en Amazon AI
User-agent: Amazonbot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Amazon search (Alexa / Rufus)
User-agent: Amzn-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Meta AI training
User-agent: Meta-ExternalAgent
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Meta AI user-initiated ophalen
User-agent: Meta-ExternalFetcher
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Meta AI search-index
User-agent: meta-webindexer
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Petal / Huawei search
User-agent: PetalBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Common Crawl (publieke trainingsdata)
User-agent: CCBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Allen Institute for AI (onderzoek)
User-agent: AI2Bot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Allen Institute Dolma trainings-corpus
User-agent: Ai2Bot-Dolma
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Cohere training (gedocumenteerd)
User-agent: cohere-training-data-crawler
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Social link-preview (geen AI-crawler)
User-agent: facebookexternalhit
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# ---------------------------------------------------------------------------
# Beperkt: alleen llms.txt, llms-full.txt en de sitemap, verder geen crawl van de site.
# ---------------------------------------------------------------------------

# ByteDance / Doubao / TikTok AI: alleen llms.txt, llms-full.txt en sitemap
User-agent: Bytespider
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /wp-sitemap.xml
Allow: /wp-sitemap-
Disallow: /

# Data-aggregators en scrapers: alleen llms.txt, llms-full.txt en sitemap
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: Omgilibot
User-agent: Timpibot
User-agent: Webzio-Extended
User-agent: PanguBot
User-agent: VelenPublicWebCrawler
User-agent: cohere-ai
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /wp-sitemap.xml
Allow: /wp-sitemap-
Disallow: /

Sitemap: https://www.citeerbaar.nl/wp-sitemap.xml
Sitemap: https://www.citeerbaar.nl/image-sitemap.xml
Agentmap: https://www.citeerbaar.nl/.well-known/ard.json