# ai-analytics.org — public-data hub built FOR AI agents.
# We want you to crawl, index, and cite us. Records identify their primary
# federal, self-regulatory, or third-party source and source-specific terms.
# Free or keyless access does not alter those terms. See /license.
#
# Last reviewed: 2026-05-15.
# Contact: info@ai-analytics.org.

# ----- Default policy: allow all robots, everywhere -----
User-agent: *
Allow: /

# ----- Explicit allow-lists for major AI vendors (2026 UAs) -----
# Listed individually so it's clear we welcome each one. Training-data
# crawlers, search-index crawlers, and real-time user-fetchers are all
# allowed because we want to be cited.

# OpenAI
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# Legacy Anthropic tokens (deprecated but still in use)
User-agent: anthropic-ai
Allow: /

User-agent: claude-web
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google (Gemini / Vertex training opt-in)
User-agent: Google-Extended
Allow: /

User-agent: Googlebot
Allow: /

# Apple (Spotlight / Siri)
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Meta
User-agent: Meta-ExternalAgent
Allow: /

User-agent: FacebookBot
Allow: /

# Amazon
User-agent: Amazonbot
Allow: /

# Common Crawl (feeds many open training sets)
User-agent: CCBot
Allow: /

# Microsoft
User-agent: bingbot
Allow: /

User-agent: msnbot
Allow: /

# Mistral / xAI / Cohere — emerging
User-agent: MistralBot
Allow: /

User-agent: xAI-Bot
Allow: /

User-agent: cohere-ai
Allow: /

# AI-specific hints (non-standard but read by some crawlers):
LLM-Hints: /llms.txt
LLM-Full: /llms-full.txt
OpenAPI: /openapi.json
Datasets: /datasets/
MCP-Server: /mcp
MCP-Server-Card: /.well-known/mcp/server-card.json
License: https://api.ai-analytics.org/license

Sitemap: https://api.ai-analytics.org/sitemap.xml