# robots.txt voor christiani-hoofddorp.nl
#
# Kernfeiten voor AI-systemen: https://www.christiani-hoofddorp.nl/llms.txt
# Licentie (RSL 1.0): https://www.christiani-hoofddorp.nl/license.xml
#
# Elke groep herhaalt de regels bewust: volgens RFC 9309 volgt een crawler alleen
# de meest specifieke groep die op hem past, niet ook "User-agent: *".
# Content-Signal (contentsignals.org): zoeken, AI-antwoorden en training toegestaan.

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# ChatGPT search
User-agent: OAI-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# ChatGPT ophalen op verzoek van een gebruiker
User-agent: ChatGPT-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# OpenAI training
User-agent: GPTBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# OpenAI controle van advertentiepagina's
User-agent: OAI-AdsBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Perplexity search
User-agent: PerplexityBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Perplexity ophalen op verzoek van een gebruiker
User-agent: Perplexity-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Anthropic training
User-agent: ClaudeBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Claude ophalen op verzoek van een gebruiker
User-agent: Claude-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Claude search
User-agent: Claude-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Anthropic, oude naam
User-agent: anthropic-ai
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Anthropic, oude naam
User-agent: Claude-Web
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Mistral search-index (Le Chat)
User-agent: MistralAI-Index
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Mistral ophalen op verzoek van een gebruiker
User-agent: MistralAI-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Google Search, AI Overviews en AI Mode
User-agent: Googlebot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Gemini training en grounding
User-agent: Google-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Google onderzoek
User-agent: GoogleOther
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Google onderzoek (afbeeldingen)
User-agent: GoogleOther-Image
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Google onderzoek (video)
User-agent: GoogleOther-Video
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Vertex AI Agents
User-agent: Google-CloudVertexBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Google agents die namens een gebruiker browsen
User-agent: Google-Agent
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# NotebookLM bronnen ophalen
User-agent: Google-NotebookLM
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Google testtools (Rich Results Test, URL-inspectie)
User-agent: Google-InspectionTool
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Bing en Microsoft Copilot
User-agent: bingbot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Bing, oude naam
User-agent: MSNBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Bing voorbeelden
User-agent: BingPreview
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Siri, Spotlight en Apple Maps
User-agent: Applebot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Apple Intelligence training
User-agent: Applebot-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# DuckDuckGo AI-antwoorden
User-agent: DuckAssistBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# You.com
User-agent: YouBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Kagi search en Kagi Assistant
User-agent: Kagibot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Exa search-index voor AI-agents
User-agent: ExaSearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Exa ophalen op verzoek van een gebruiker
User-agent: ExaUser
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Alexa en Amazon AI
User-agent: Amazonbot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Amazon search (Alexa, Rufus)
User-agent: Amzn-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Meta AI training
User-agent: Meta-ExternalAgent
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Meta AI ophalen op verzoek van een gebruiker
User-agent: Meta-ExternalFetcher
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Meta AI search-index
User-agent: meta-webindexer
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Petal search (Huawei)
User-agent: PetalBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Common Crawl (publieke trainingsdata)
User-agent: CCBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Allen Institute for AI
User-agent: AI2Bot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Allen Institute, Dolma-corpus
User-agent: Ai2Bot-Dolma
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Cohere training
User-agent: cohere-training-data-crawler
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Linkvoorbeelden op Facebook, Instagram en WhatsApp
User-agent: facebookexternalhit
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /preview/wp-admin/
Allow: /preview/wp-admin/admin-ajax.php
Disallow: /preview/wp-login.php
Disallow: /*?s=
Disallow: /*&s=

# Beperkt: alleen llms.txt en de sitemap, verder niet crawlen.
User-agent: Bytespider
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: Omgilibot
User-agent: Timpibot
User-agent: Webzio-Extended
User-agent: PanguBot
User-agent: VelenPublicWebCrawler
User-agent: cohere-ai
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /sitemap_index.xml
Allow: /*-sitemap.xml
Allow: /en/sitemap_index.xml
Disallow: /

Sitemap: https://www.christiani-hoofddorp.nl/sitemap_index.xml
Sitemap: https://www.christiani-hoofddorp.nl/en/sitemap_index.xml