# robots.txt for Skene App
# https://skene.ai/robots.txt

# ── Default policy ──
User-agent: *
Allow: /
Disallow: /_next/static/
Disallow: /workspace/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /auth/
Disallow: /invite/
Disallow: /debug-workspace/
Disallow: /maintenance/

# ── AI crawlers (explicit allow with same private-route guards) ──
# OpenAI
User-agent: GPTBot
Allow: /
Disallow: /workspace/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /auth/
Disallow: /invite/
Disallow: /debug-workspace/
Disallow: /maintenance/

User-agent: OAI-SearchBot
Allow: /
Disallow: /workspace/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /auth/
Disallow: /invite/
Disallow: /debug-workspace/
Disallow: /maintenance/

User-agent: ChatGPT-User
Allow: /
Disallow: /workspace/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /auth/
Disallow: /invite/
Disallow: /debug-workspace/
Disallow: /maintenance/

# Anthropic
User-agent: ClaudeBot
Allow: /
Disallow: /workspace/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /auth/
Disallow: /invite/
Disallow: /debug-workspace/
Disallow: /maintenance/

User-agent: Claude-Web
Allow: /
Disallow: /workspace/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /auth/
Disallow: /invite/
Disallow: /debug-workspace/
Disallow: /maintenance/

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /workspace/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /auth/
Disallow: /invite/
Disallow: /debug-workspace/
Disallow: /maintenance/

# Google (generative / extended training)
User-agent: Google-Extended
Allow: /
Disallow: /workspace/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /auth/
Disallow: /invite/
Disallow: /debug-workspace/
Disallow: /maintenance/

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /
Disallow: /workspace/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /auth/
Disallow: /invite/
Disallow: /debug-workspace/
Disallow: /maintenance/

# Bing / Microsoft Copilot
User-agent: Bingbot
Allow: /
Disallow: /workspace/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /auth/
Disallow: /invite/
Disallow: /debug-workspace/
Disallow: /maintenance/

# Common Crawl (training data feed for many LLMs)
User-agent: CCBot
Allow: /
Disallow: /workspace/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /auth/
Disallow: /invite/
Disallow: /debug-workspace/
Disallow: /maintenance/

# Meta / Facebook
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /workspace/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /auth/
Disallow: /invite/
Disallow: /debug-workspace/
Disallow: /maintenance/

# Amazon
User-agent: Amazonbot
Allow: /
Disallow: /workspace/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /auth/
Disallow: /invite/
Disallow: /debug-workspace/
Disallow: /maintenance/

# ByteDance / TikTok
User-agent: Bytespider
Allow: /
Disallow: /workspace/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /auth/
Disallow: /invite/
Disallow: /debug-workspace/
Disallow: /maintenance/

# Brave Search (independent index; feeds Brave Leo AI)
User-agent: Brave-Web-Discovery
Allow: /
Disallow: /workspace/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /auth/
Disallow: /invite/
Disallow: /debug-workspace/
Disallow: /maintenance/

# Mojeek (independent UK index)
User-agent: MojeekBot
Allow: /
Disallow: /workspace/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /auth/
Disallow: /invite/
Disallow: /debug-workspace/
Disallow: /maintenance/

# ── LLM-friendly corpus (https://llms-txt.org) ──
# /llms.txt       — concise link index for LLMs
# /llms-full.txt  — full plain-text corpus for retrieval / grounding

# ── Sitemap ──
Sitemap: https://www.skene.ai/sitemap.xml