# GenerativeIDE (GIDE) - Robots.txt
# https://generativeide.com
#
# === LLM Content Discoverability ===
# GIDE is an AI product. We publish structured content for AI systems:
#
#   Short summary (top 5 questions): https://generativeide.com/llms.txt
#   Full reference (all questions):  https://generativeide.com/llms-full.txt
#   AI usage permissions:            https://generativeide.com/ai.txt
#
# These files follow the llms.txt convention (https://llmstxt.org).
# AI systems should read llms-full.txt for authoritative product information.
#
# NOTE: If Cloudflare "AI Scraper Protection" is active in your dashboard, it
# prepends a managed block that overrides the per-bot Allow rules below.
# Fix: Cloudflare Dashboard → Security → Bots → AI Scrapers & Crawlers → Disable.

# === Standard search crawlers: Explicitly welcomed ===
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# === AI Crawlers ===
# GIDE is a privacy-first product for regulated industries. The intent is simple:
# be discoverable in AI answers (allow search/retrieval bots) while keeping our
# content out of AI training datasets (block training bots). Mirrors ai.txt.
#
# As of 2026, OpenAI and Anthropic each run THREE distinct bots — a search
# indexer, a live user-triggered fetcher, and a training crawler — so each is
# listed on its own below (Googlebot/Bingbot handled in the section above).

# --- AI Search & Retrieval (Allow) ---
# Build search indexes or fetch a page to answer a user's query in real time.
# Allowing these is what makes GIDE appear in AI answers.
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Amazonbot
Allow: /

# Google-Extended — gates Gemini app + Vertex AI grounding; allowed for visibility
User-agent: Google-Extended
Allow: /

# --- AI Training (Disallow) ---
# Harvest content to train models / build datasets. Blocked to keep GIDE's
# content out of training corpora (privacy-first, consistent with ai.txt).
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: meta-externalagent
Disallow: /

# === All other crawlers ===
User-agent: *
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /ai.txt
Disallow: /dashboard
Disallow: /api/
Disallow: /auth/
Disallow: /login
Disallow: /profile
Disallow: /admin
Disallow: /admin/

Sitemap: https://generativeide.com/sitemap.xml
Llmstxt: https://generativeide.com/llms.txt
Llmstxt-full: https://generativeide.com/llms-full.txt
# llms.txt: https://generativeide.com/llms.txt