# Skygem Concierge — robots.txt
# Updated 2026-05-16 via /SEO baseline audit.
#
# Crawlability:
#   - Public marketing + journal: index everything by default.
#   - Affiliate redirects (/go/) are noindex, but we also disallow them at the
#     robots layer so polite crawlers skip the link entirely.
#
# Last verified: 2026-05-16

Sitemap: https://diamonds.skygem.tech/sitemap-index.xml

# ---------------------------------------------------------------------------
# Default policy — all general-purpose crawlers (Googlebot, Bingbot fallback,
# etc.) can read everything except affiliate redirects and Supabase function
# endpoints.
# ---------------------------------------------------------------------------
User-agent: *
Disallow: /go/
Disallow: /api/

# ---------------------------------------------------------------------------
# AI / Answer-Engine bot allowlist (explicit).
# Permitted to crawl and use content for retrieval-augmented answers.
# Block list at the bottom rejects two known abusive crawlers.
# ---------------------------------------------------------------------------

# OpenAI / ChatGPT
User-agent: GPTBot
Allow: /
Disallow: /go/

User-agent: ChatGPT-User
Allow: /
Disallow: /go/

User-agent: OAI-SearchBot
Allow: /
Disallow: /go/

# Anthropic / Claude
User-agent: ClaudeBot
Allow: /
Disallow: /go/

User-agent: anthropic-ai
Allow: /
Disallow: /go/

User-agent: Claude-Web
Allow: /
Disallow: /go/

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /go/

User-agent: Perplexity-User
Allow: /
Disallow: /go/

# Google (Gemini / AI Overviews)
User-agent: Google-Extended
Allow: /
Disallow: /go/

User-agent: GoogleOther
Allow: /
Disallow: /go/

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /
Disallow: /go/

# Bing / Copilot
User-agent: Bingbot
Allow: /
Disallow: /go/

# Common Crawl
User-agent: CCBot
Allow: /
Disallow: /go/

# DuckDuckGo
User-agent: DuckAssistBot
Allow: /
Disallow: /go/

# Diffbot
User-agent: Diffbot
Allow: /
Disallow: /go/

# You.com
User-agent: YouBot
Allow: /
Disallow: /go/

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /go/

User-agent: Meta-ExternalFetcher
Allow: /
Disallow: /go/

User-agent: FacebookBot
Allow: /
Disallow: /go/

# Mistral
User-agent: mistralai-User
Allow: /
Disallow: /go/

# Amazon / Alexa
User-agent: Amazonbot
Allow: /
Disallow: /go/

# Cohere
User-agent: cohere-ai
Allow: /
Disallow: /go/

# ---------------------------------------------------------------------------
# Block: known abusive crawlers.
# Bytespider (TikTok/ByteDance) and PetalBot (Huawei/Petal) ignore robots
# directives intermittently and have no useful answer-engine surface.
# ---------------------------------------------------------------------------
User-agent: Bytespider
Disallow: /

User-agent: PetalBot
Disallow: /