# ---- AI SEARCH / CITATION crawlers: ALLOWED ----
# These are the bots that read the site to CITE it in AI answers
# (ChatGPT search, Perplexity, Google Gemini/AI Overviews, Claude).
# Allowing them is required for our GEO goal — they put us in front of
# buyers. They are distinct from the "training" bots blocked further
# down. (Explicit Allow groups so the intent is unambiguous even if a
# CDN concatenates its own managed block.)
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

# Claude's search index crawler (distinct from ClaudeBot) and OpenAI's
# agentic browser (Operator). Both fetch on a user's behalf to answer or
# act on a shopping question — exactly the traffic we want.
User-agent: Claude-SearchBot
Allow: /

User-agent: OAI-Operator
Allow: /

# ---- AI TRAINING / scraper crawlers: BLOCKED ----
# We don't contribute our catalogue to model *training* datasets. These
# are training/scraping bots, NOT search — blocking them does not affect
# AI-search citation above. Loosen individually if you change that policy.
User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: Claude-Web
Disallow: /

# ---- Regular search crawlers ----
User-agent: *
Allow: /

# Facet cleanup, stage 1 (September 2026): keep shop queries crawlable on
# /pood/, /veikals/ and /kauppa/, including their slashless redirects.
# Active filter/sort pages emit noindex,follow; Google must fetch that tag
# before any later crawl restriction. Legacy filter_*, query_type_* and
# per_page URLs must also stay crawlable so their cleanup redirects work.
# Pure ?page=N URLs have distinct products and canonicals: do not block.
# Reassess facet-only crawl restrictions after GSC confirms deindexing.

# Block internal pages
Disallow: /api/

# Cart / checkout / order tracking — per locale (et, lv, fi).
# /kassa is shared by ET and FI; LV uses /kase.
Disallow: /ostukorv
Disallow: /grozs
Disallow: /ostoskori
Disallow: /kassa
Disallow: /kase
Disallow: /jalgi-tellimust
Disallow: /izsekot-pasutijumu
Disallow: /seuraa-tilausta

# Internal search results (thin, parameter-driven) — per locale
Disallow: /otsing
Disallow: /meklet
Disallow: /haku

# Sitemaps
Sitemap: https://kohvrimaailm.ee/sitemap.xml
Sitemap: https://koferuveikals.lv/sitemap.xml

# AI agents: MCP server https://kohvrimaailm.ee/mcp (EE), https://koferuveikals.lv/mcp (LV)
# LLM index https://kohvrimaailm.ee/llms.txt (EE), https://koferuveikals.lv/llms.txt (LV)
# matkamaailma.fi (FI) not built yet — re-add its sitemap when the domain goes live.