Analysis Result

# robots.txt — AI-optimized (generated by GEO Checker on 2026-10-18)
# Source: https://gc.ranran.tw
#
# Strategy: Block training crawlers, allow search/retrieval
# and user-triggered fetchers for AI citation visibility.

# === General ===
User-agent: *
Allow: /

# === Training Crawlers (opt out of model training) ===
# Blocking these prevents your content from being used
# in AI model training, while keeping AI search visibility.

# OpenAI: Model training data
User-agent: GPTBot
Disallow: /

# Anthropic: Model training data
User-agent: ClaudeBot
Disallow: /

# Meta: Model training data
User-agent: Meta-ExternalAgent
Disallow: /

# Common Crawl: Open training corpus
User-agent: CCBot
Disallow: /

# Amazon: Alexa/Rufus training
User-agent: Amazonbot
Disallow: /

# Cohere: Model training data
User-agent: cohere-ai
Disallow: /

# ByteDance: Non-compliant — ignores robots.txt; block at CDN/WAF layer
User-agent: Bytespider
Disallow: /

# === Training Opt-Out Tokens ===
# These are NOT crawlers — they are robots.txt control tokens
# that opt out of AI training without affecting search visibility.

# Google: Opt-out token for Gemini/AI training (not a crawler)
User-agent: Google-Extended
Disallow: /

# Apple: Opt-out token for Apple Intelligence training (not a crawler)
User-agent: Applebot-Extended
Disallow: /

# === Search/Retrieval Crawlers (allow for AI answer visibility) ===
# Allowing these lets your content appear in AI-generated answers.

# OpenAI: ChatGPT Search indexing
User-agent: OAI-SearchBot
Allow: /

# Anthropic: Claude search indexing
User-agent: Claude-SearchBot
Allow: /

# Perplexity: AI search; documented non-compliant with robots.txt
User-agent: PerplexityBot
Allow: /

# Meta: Meta AI search indexing
User-agent: Meta-WebIndexer
Allow: /

# DuckDuckGo: AI assist answers
User-agent: DuckAssistBot
Allow: /

# You.com: You.com AI search
User-agent: YouBot
Allow: /

# Phind: Developer AI search
User-agent: PhindBot
Allow: /

# Mistral: Mistral search indexing
User-agent: MistralAI-Index
Allow: /

# Google: Vertex AI Agents site-owner-requested crawl
User-agent: Google-CloudVertexBot
Allow: /

# Microsoft: Bing / Copilot / AI Overviews retrieval
User-agent: Bingbot
Allow: /

# === User-Triggered Fetchers (allow for citation traffic) ===
# These fetch your pages when real users ask AI assistants questions.

# OpenAI: Fetches pages when users ask ChatGPT
User-agent: ChatGPT-User
Allow: /

# Anthropic: Fetches pages when users ask Claude
User-agent: Claude-User
Allow: /

# Perplexity: User-triggered; documented non-compliant
User-agent: Perplexity-User
Allow: /

# Mistral: Fetches pages when users ask Le Chat
User-agent: MistralAI-User
Allow: /

# Meta: Fetches pages when users ask Meta AI
User-agent: Meta-ExternalFetcher
Allow: /

# === Robots.txt Blind Spots (cannot be controlled here) ===
# Google-Agent (Google): Agentic fetcher — officially ignores robots.txt (Google policy)
# Gemini-Deep-Research (Google): Deep research fetcher — robots.txt compliance unclear
# xAI/Grok (xAI): No official UA; uses residential IP + spoofed browser UAs
# DeepSeek (DeepSeek): No official UA; crawls as regular browser traffic
# → Control these at CDN/WAF layer (e.g., Cloudflare, AWS WAF).

Sitemap: https://www.uniprobiotics.com.tw/sitemap.xml

# AI content index (low adoption; hygiene signal, not a ranking lever)
# https://www.uniprobiotics.com.tw/llms.txt

<script type="application/ld+json">
{
  "@context": "https://schema.org",
  "@graph": [
    {
      "@type": "Article",
      "headline": "統一益菌室",
      "description": "歡迎來到「統一益菌室」，我們提供專業益生菌相關產品，幫助您解決腸胃、過敏、體質等各種生活煩惱，快點來看看！",
      "@id": "https://www.uniprobiotics.com.tw/#article",
      "url": "https://www.uniprobiotics.com.tw/",
      "mainEntityOfPage": {
        "@id": "https://www.uniprobiotics.com.tw/"
      },
      "isPartOf": {
        "@id": "https://www.uniprobiotics.com.tw/#website"
      },
      "author": {
        "@type": "Person",
        "name": "[Your Name]"
      },
      "wordCount": 127
    },
    {
      "@type": "Organization",
      "name": "統一益菌室",
      "@id": "https://www.uniprobiotics.com.tw/#organization",
      "url": "https://www.uniprobiotics.com.tw/"
    }
  ]
}
</script>