Analysis Result

# robots.txt — AI-optimized (generated by GEO Checker on 2026-10-18)
# Source: https://gc.ranran.tw
#
# Strategy: Block training crawlers, allow search/retrieval
# and user-triggered fetchers for AI citation visibility.

# === General ===
User-agent: *
Allow: /

# === Training Crawlers (opt out of model training) ===
# Blocking these prevents your content from being used
# in AI model training, while keeping AI search visibility.

# OpenAI: Model training data
User-agent: GPTBot
Disallow: /

# Anthropic: Model training data
User-agent: ClaudeBot
Disallow: /

# Meta: Model training data
User-agent: Meta-ExternalAgent
Disallow: /

# Common Crawl: Open training corpus
User-agent: CCBot
Disallow: /

# Amazon: Alexa/Rufus training
User-agent: Amazonbot
Disallow: /

# Cohere: Model training data
User-agent: cohere-ai
Disallow: /

# ByteDance: Non-compliant — ignores robots.txt; block at CDN/WAF layer
User-agent: Bytespider
Disallow: /

# === Training Opt-Out Tokens ===
# These are NOT crawlers — they are robots.txt control tokens
# that opt out of AI training without affecting search visibility.

# Google: Opt-out token for Gemini/AI training (not a crawler)
User-agent: Google-Extended
Disallow: /

# Apple: Opt-out token for Apple Intelligence training (not a crawler)
User-agent: Applebot-Extended
Disallow: /

# === Search/Retrieval Crawlers (allow for AI answer visibility) ===
# Allowing these lets your content appear in AI-generated answers.

# OpenAI: ChatGPT Search indexing
User-agent: OAI-SearchBot
Allow: /

# Anthropic: Claude search indexing
User-agent: Claude-SearchBot
Allow: /

# Perplexity: AI search; documented non-compliant with robots.txt
User-agent: PerplexityBot
Allow: /

# Meta: Meta AI search indexing
User-agent: Meta-WebIndexer
Allow: /

# DuckDuckGo: AI assist answers
User-agent: DuckAssistBot
Allow: /

# You.com: You.com AI search
User-agent: YouBot
Allow: /

# Phind: Developer AI search
User-agent: PhindBot
Allow: /

# Mistral: Mistral search indexing
User-agent: MistralAI-Index
Allow: /

# Google: Vertex AI Agents site-owner-requested crawl
User-agent: Google-CloudVertexBot
Allow: /

# Microsoft: Bing / Copilot / AI Overviews retrieval
User-agent: Bingbot
Allow: /

# === User-Triggered Fetchers (allow for citation traffic) ===
# These fetch your pages when real users ask AI assistants questions.

# OpenAI: Fetches pages when users ask ChatGPT
User-agent: ChatGPT-User
Allow: /

# Anthropic: Fetches pages when users ask Claude
User-agent: Claude-User
Allow: /

# Perplexity: User-triggered; documented non-compliant
User-agent: Perplexity-User
Allow: /

# Mistral: Fetches pages when users ask Le Chat
User-agent: MistralAI-User
Allow: /

# Meta: Fetches pages when users ask Meta AI
User-agent: Meta-ExternalFetcher
Allow: /

# === Robots.txt Blind Spots (cannot be controlled here) ===
# Google-Agent (Google): Agentic fetcher — officially ignores robots.txt (Google policy)
# Gemini-Deep-Research (Google): Deep research fetcher — robots.txt compliance unclear
# xAI/Grok (xAI): No official UA; uses residential IP + spoofed browser UAs
# DeepSeek (DeepSeek): No official UA; crawls as regular browser traffic
# → Control these at CDN/WAF layer (e.g., Cloudflare, AWS WAF).

Sitemap: https://geo-teacher.com/sitemap.xml

# AI content index (low adoption; hygiene signal, not a ranking lever)
# https://geo-teacher.com/llms.txt

<script type="application/ld+json">
{
  "@type": "Article",
  "headline": "GEO Teacher",
  "description": "Enter a public URL to check page-level signals and receive practical suggestions for AI search readiness.",
  "@id": "https://geo-teacher.com/#article",
  "url": "https://geo-teacher.com/",
  "mainEntityOfPage": {
    "@id": "https://geo-teacher.com/"
  },
  "isPartOf": {
    "@id": "https://geo-teacher.com/#website"
  },
  "author": {
    "@type": "Person",
    "name": "[Your Name]"
  },
  "wordCount": 14,
  "@context": "https://schema.org"
}
</script>