# Robots.txt — Trusteed Discovery & Crawling Rules
# Updated: 2026-08-28

User-agent: *
Allow: /

# Restrict access to authenticated sections
Disallow: /en/login
Disallow: /es/login
Disallow: /dashboard/
Disallow: /admin/
Disallow: /api/auth/

# Allow discovery endpoints for LLMs and MCP clients
Allow: /llms.txt
Allow: /llms_large.txt
Allow: /mcp.json
Allow: /.well-known/
Allow: /api/v1/health

# AI crawlers — explicit allow for public/discovery paths, same Disallows as global
User-agent: GPTBot
Allow: /llms.txt
Allow: /llms_large.txt
Allow: /mcp.json
Allow: /.well-known/
Allow: /api/v1/
Allow: /en/
Allow: /es/
Disallow: /en/login
Disallow: /es/login
Disallow: /dashboard/
Disallow: /admin/
Disallow: /api/auth/

User-agent: OAI-SearchBot
Allow: /llms.txt
Allow: /llms_large.txt
Allow: /mcp.json
Allow: /.well-known/
Allow: /api/v1/
Allow: /en/
Allow: /es/
Disallow: /en/login
Disallow: /es/login
Disallow: /dashboard/
Disallow: /admin/
Disallow: /api/auth/

User-agent: ClaudeBot
Allow: /llms.txt
Allow: /llms_large.txt
Allow: /mcp.json
Allow: /.well-known/
Allow: /api/v1/
Allow: /en/
Allow: /es/
Disallow: /en/login
Disallow: /es/login
Disallow: /dashboard/
Disallow: /admin/
Disallow: /api/auth/

User-agent: anthropic-ai
Allow: /llms.txt
Allow: /llms_large.txt
Allow: /mcp.json
Allow: /.well-known/
Allow: /api/v1/
Allow: /en/
Allow: /es/
Disallow: /en/login
Disallow: /es/login
Disallow: /dashboard/
Disallow: /admin/
Disallow: /api/auth/

User-agent: PerplexityBot
Allow: /llms.txt
Allow: /llms_large.txt
Allow: /mcp.json
Allow: /.well-known/
Allow: /api/v1/
Allow: /en/
Allow: /es/
Disallow: /en/login
Disallow: /es/login
Disallow: /dashboard/
Disallow: /admin/
Disallow: /api/auth/

User-agent: ChatGPT-User
Allow: /llms.txt
Allow: /llms_large.txt
Allow: /mcp.json
Allow: /.well-known/
Allow: /api/v1/
Allow: /en/
Allow: /es/
Disallow: /en/login
Disallow: /es/login
Disallow: /dashboard/
Disallow: /admin/
Disallow: /api/auth/

User-agent: Claude-SearchBot
Allow: /llms.txt
Allow: /llms_large.txt
Allow: /mcp.json
Allow: /.well-known/
Allow: /api/v1/
Allow: /en/
Allow: /es/
Disallow: /en/login
Disallow: /es/login
Disallow: /dashboard/
Disallow: /admin/
Disallow: /api/auth/

User-agent: cohere-ai
Allow: /llms.txt
Allow: /llms_large.txt
Allow: /mcp.json
Allow: /.well-known/
Allow: /api/v1/
Allow: /en/
Allow: /es/
Disallow: /en/login
Disallow: /es/login
Disallow: /dashboard/
Disallow: /admin/
Disallow: /api/auth/

User-agent: Applebot
Allow: /llms.txt
Allow: /llms_large.txt
Allow: /mcp.json
Allow: /.well-known/
Allow: /api/v1/
Allow: /en/
Allow: /es/
Disallow: /en/login
Disallow: /es/login
Disallow: /dashboard/
Disallow: /admin/
Disallow: /api/auth/

# Rate limiting for aggressive crawlers
User-agent: AhrefsBot
Disallow: /
Crawl-delay: 10

User-agent: SemrushBot
Disallow: /
Crawl-delay: 10

# Sitemaps
Sitemap: https://trusteed.xyz/sitemap.xml

# Content Signals (draft-romm-aipref-contentsignals) — AI content preferences
Content-Signal: ai-train=no
Content-Signal: search=yes
Content-Signal: ai-input=yes
Content-Signal: license=proprietary