# robots.txt for gawel.ch
# All content is public legal and professional information: full access for
# search engines and for the AI agents that answer questions from it.

# As a condition of accessing this website, you agree to abide by the following
# content signals:
#
# (a) If a content-signal = yes, you may collect content for the corresponding
#     use.
# (b) If a content-signal = no, you may not collect content for the
#     corresponding use.
# (c) If the website operator does not include a content signal for a
#     corresponding use, the website operator neither grants nor restricts
#     permission via content signal with respect to the corresponding use.
#
# The content signals and their meanings are:
#
# search: building a search index and providing search results (e.g., returning
#     hyperlinks and short excerpts from your website's contents). Search does
#     not include providing AI-generated search summaries.
# ai-input: inputting content into one or more AI models (e.g., retrieval
#     augmented generation, grounding, or other real-time taking of content for
#     generative AI search answers).
# ai-train: training or fine-tuning AI models.
#
# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF
# RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT
# AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET.

# Default rules for all crawlers
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /cdn-cgi/

# Named agents are listed so that none can be caught by a future default change.
# Each group repeats the content signals and Disallow: /cdn-cgi/ because per the
# robots.txt spec named-agent groups do not inherit from User-agent: *

# OpenAI: training, ChatGPT search, and user-initiated fetches
User-agent: GPTBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /cdn-cgi/

User-agent: OAI-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /cdn-cgi/

User-agent: ChatGPT-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /cdn-cgi/

# Google AI (Gemini)
User-agent: Google-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /cdn-cgi/

# Common Crawl (used by many AI models)
User-agent: CCBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /cdn-cgi/

# Anthropic (Claude): training, search, and user-initiated fetches
User-agent: anthropic-ai
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /cdn-cgi/

User-agent: ClaudeBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /cdn-cgi/

User-agent: Claude-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /cdn-cgi/

User-agent: Claude-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /cdn-cgi/

User-agent: Claude-Web
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /cdn-cgi/

# Perplexity: search index and user-initiated fetches
User-agent: PerplexityBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /cdn-cgi/

User-agent: Perplexity-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /cdn-cgi/

# DuckDuckGo AI-assisted answers
User-agent: DuckAssistBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /cdn-cgi/

# Mistral AI: user-initiated fetches
User-agent: MistralAI-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /cdn-cgi/

# You.com AI
User-agent: YouBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /cdn-cgi/

# Meta AI: training, indexing, and user-initiated fetches
User-agent: FacebookBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /cdn-cgi/

User-agent: meta-externalagent
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /cdn-cgi/

User-agent: meta-externalfetcher
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /cdn-cgi/

# Apple: Siri and Spotlight search, and Apple Intelligence training
User-agent: Applebot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /cdn-cgi/

User-agent: Applebot-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /cdn-cgi/

# Amazon (Alexa and Amazon AI)
User-agent: Amazonbot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /cdn-cgi/

# ByteDance/TikTok
User-agent: Bytespider
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /cdn-cgi/

# Cohere AI
User-agent: cohere-ai
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /cdn-cgi/

# AI2 (Allen Institute)
User-agent: ai2bot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /cdn-cgi/

# A Markdown index of this site for AI agents (the practice, every service,
# topic and article with its canonical URL): https://gawel.ch/llms.txt

# Sitemap location
Sitemap: https://gawel.ch/sitemap.xml