# AiSearchEngine™ — crawl policy
# All console documents are publicly crawlable under /docs/
# The live Framework Codex (/framework) and Glial Core Engine (/glial)
# are rendered by JavaScript — all listed AI crawlers support JS rendering
# or will index the equivalent static /docs/ mirror pages.

User-agent: *
Allow: /
Allow: /docs/
Allow: /framework
Allow: /glial
Allow: /api/sitemap.xml
Disallow: /api/
Allow: /api/sitemap.xml

# Google — search + AI Overview / SGE
User-agent: Googlebot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Googlebot-Image
Allow: /

# OpenAI — ChatGPT Search, GPT plugins
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic — Claude.ai web search
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Microsoft — Bing, Copilot
User-agent: Bingbot
Allow: /

User-agent: msnbot
Allow: /

# Apple — Siri, Spotlight
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Cohere AI
User-agent: cohere-ai
Allow: /

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /

# Meta — Meta AI
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

# xAI — Grok
User-agent: Grok
Allow: /

# You.com
User-agent: YouBot
Allow: /

# Brave Search
User-agent: Bravesearch
Allow: /

# Baidu
User-agent: Baiduspider
Allow: /

# Common crawl (training data for many AI systems)
User-agent: CCBot
Allow: /

# Diffbot (AI data extraction)
User-agent: Diffbot
Allow: /

Sitemap: https://www.aisearchengine.com/api/sitemap.xml