# robots.txt for IAutos (web-v3.5)
# Served by Next.js as a static file, takes priority over any CDN default.

# Allow all crawlers
User-agent: *
# AI usage preferences, per https://contentsignals.org/ (draft-romm-aipref-contentsignals):
# indexing and real-time citation (grounding) allowed, mass training refused
# (product decision of 05-08-2026).
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

# Block the BFF proxy, auth funnels, and private surfaces. `/personal/`
# replaces the legacy `/account/` prefix: this app's private surface lives at
# /personal/account, not at the apps/web path. `/messages/` is parked
# (KIL-231 canceled 23-08-2026) and stays disallowed so a leftover URL
# cannot index. `/checkout/`,
# `/dashboard/`, `/dev-tools/` and `/impersonate` follow the same rule — the
# disallow is in place before the route, so a tunnel can never index by
# omission (parity with apps/web, KIL-176). No `/pro/` rule on purpose: the
# pro CRM lives on pro.iautos.fr (apps/pro), and this app's only /pro path is
# /pro/api, a public SEO page.
# `/histovec/new` follows the same rule (KIL-170): the landing at /histovec
# is served and indexable, its assisted generation tunnel (plate/carte-grise
# scan) is not built yet, so the disallow is inert until it lands.
Disallow: /api/
Disallow: /personal/
Disallow: /dashboard/
Disallow: /dev-tools/
Disallow: /connexion
Disallow: /inscription
Disallow: /mot-de-passe-oublie
Disallow: /reset-password
Disallow: /oauth/
Disallow: /impersonate
Disallow: /verify-email
Disallow: /premiere-connexion
Disallow: /histovec/new
Disallow: /je-cherche/new
Disallow: /vendeurs/onboarding
Disallow: /messages/
Disallow: /ia-chat
Disallow: /ia-chat/
Disallow: /checkout/

# Block the faceted-search crawl traps (unbounded query-string filter
# combinations). SEO-indexable filtered pages use clean path-based URLs, not
# query strings, so this does not affect indexing. /annonces-auto? remains
# even after the intentions cluster shipped: faceted permutations of that
# hub must not be crawled.
Disallow: /annonces?
Disallow: /annonces-auto?

# ai-train=no (05-08-2026): search engines and on-demand citation agents (GEO)
# stay allowed, mass-training harvesters are blocked, consistent with the
# Cloudflare "AI Crawl Control" WAF rule on the iautos.fr zone.

# --- On-demand citation agents (GEO): allowed ---

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /
Crawl-delay: 2

User-agent: Perplexity-User
Allow: /

User-agent: meta-externalfetcher
Allow: /

User-agent: Applebot
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: cohere-ai
Allow: /
Crawl-delay: 5

User-agent: FacebookBot
Allow: /

User-agent: Mistral-AI
Allow: /
Crawl-delay: 5

# --- Mass-training harvesters: blocked (ai-train=no), consistent with the WAF ---

User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: AI2Bot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: img2dataset
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: PanguBot
Disallow: /

User-agent: cohere-training-data-crawler
Disallow: /

User-agent: FriendlyCrawler
Disallow: /

# Google-Extended and Applebot-Extended: robots.txt is their only effective
# opt-out channel (not representable in the WAF, no distinct HTTP User-Agent).
User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

# Rate-limit SEO crawlers
User-agent: SemrushBot
Crawl-delay: 10

User-agent: AhrefsBot
Crawl-delay: 10

# LLM context files
# https://llmstxt.org/
# llms.txt: https://iautos.fr/llms.txt
# llms-full.txt: https://iautos.fr/llms-full.txt

# Sitemap index (lists every paginated sitemap)
Sitemap: https://iautos.fr/sitemap.xml