# ==============================================================================
# Robots.txt for Web3 Knowledge Base (Optimized for SEO, GEO & AI Crawlers)
# https://githubskylh.github.io/web3-knowledge-base/
# ==============================================================================

User-agent: *
Allow: /
Disallow: /dist/
Disallow: /node_modules/

# -----------------------------------------------------------------------------
# Generative Engine Optimization (GEO) - Explicit Allowance for AI Crawlers
# -----------------------------------------------------------------------------
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: cohere-ai
Allow: /

# Sitemaps & LLMs Manifest
Sitemap: https://githubskylh.github.io/web3-knowledge-base/sitemap.xml
LLMs-Txt: https://githubskylh.github.io/web3-knowledge-base/llms.txt