# https://www.robotstxt.org/robotstxt.html
# InclusiveSpeak robots.txt — SEO + AEO (Answer-Engine Optimization)

# ────────────────────────────────────────────────────────────────
# Default policy — applies to traditional search engine crawlers
# ────────────────────────────────────────────────────────────────
User-agent: *
Allow: /
Allow: /blog
Allow: /features
Allow: /pricing
Allow: /how-it-works
Allow: /about
Allow: /contact
Allow: /faq
Allow: /careers
Allow: /support
Allow: /privacy
Allow: /terms

# Authenticated / functional routes — do not waste crawl budget
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /admin
Disallow: /admin/
Disallow: /login
Disallow: /register
Disallow: /parent-dashboard
Disallow: /invited-parent-dashboard
Disallow: /accept-invitation
Disallow: /parent-invitation
Disallow: /payment-success
Disallow: /payment-cancel
Disallow: /payment-failure
Disallow: /qr-test
Disallow: /api/
Disallow: /webhook
Disallow: /uploads/

# Avoid duplicate content from internal search / utm querystrings
Disallow: /*?utm_*
Disallow: /*?ref=*
Disallow: /*&utm_*

# ────────────────────────────────────────────────────────────────
# Preview / social crawlers — explicitly allowed so unfurls work
# ────────────────────────────────────────────────────────────────
User-agent: Googlebot-Image
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: Discordbot
Allow: /

User-agent: TelegramBot
Allow: /

# ────────────────────────────────────────────────────────────────
# AI / answer-engine crawlers
# We *opt in* so InclusiveSpeak can be cited as a source in
# ChatGPT Search, Perplexity, Claude, Gemini, Copilot, etc.
# Disallow private/authenticated areas explicitly.
# ────────────────────────────────────────────────────────────────

# OpenAI — training crawler
User-agent: GPTBot
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /api/
Disallow: /uploads/
Disallow: /login
Disallow: /register
Disallow: /accept-invitation
Disallow: /parent-dashboard
Disallow: /invited-parent-dashboard
Disallow: /payment-*

# OpenAI — live ChatGPT browsing/search
User-agent: ChatGPT-User
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /api/

User-agent: OAI-SearchBot
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /api/

# Anthropic — Claude
User-agent: ClaudeBot
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /api/

User-agent: Claude-Web
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /api/

User-agent: anthropic-ai
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /api/

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /api/

User-agent: Perplexity-User
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /api/

# Google — Gemini / AI Overviews opt-in is separate from regular indexing
User-agent: Google-Extended
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /api/

User-agent: GoogleOther
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /api/

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /api/

# Microsoft Copilot / Bing
User-agent: bingbot
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /api/

# Common Crawl — feeds many open-source LLMs
User-agent: CCBot
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /api/

# Meta AI
User-agent: meta-externalagent
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /api/

User-agent: FacebookBot
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /api/

# Amazon — Alexa / Nova
User-agent: Amazonbot
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /api/

# Cohere
User-agent: cohere-ai
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /api/

# You.com
User-agent: YouBot
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /api/

# DiffBot / Mistral / Phind / Andi / Komo
User-agent: DiffBot
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /api/

User-agent: MistralAI-User
Allow: /
Disallow: /dashboard
Disallow: /admin

User-agent: PhindBot
Allow: /
Disallow: /dashboard
Disallow: /admin

User-agent: Andibot
Allow: /
Disallow: /dashboard
Disallow: /admin

User-agent: Komo
Allow: /
Disallow: /dashboard
Disallow: /admin

User-agent: Webzio-Extended
Allow: /
Disallow: /dashboard
Disallow: /admin

User-agent: Timpibot
Allow: /
Disallow: /dashboard
Disallow: /admin

# ────────────────────────────────────────────────────────────────
# Bytedance / TikTok — disallowed (frequently abusive, low referral value)
# ────────────────────────────────────────────────────────────────
User-agent: Bytespider
Disallow: /

User-agent: ImagesiftBot
Disallow: /

# ────────────────────────────────────────────────────────────────
# Known SEO-spam / scraper bots — blocked entirely
# ────────────────────────────────────────────────────────────────
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

# ────────────────────────────────────────────────────────────────
# Sitemaps & LLM resource files
# ────────────────────────────────────────────────────────────────
Sitemap: https://inclusivespeak.com/sitemap.xml
Host: https://inclusivespeak.com