# Mostly Right — robots.txt
# We sell data to AI agents. Agents are welcome here.
# If you are a bot and this page returns 403, our CDN is misconfigured. Try again.

User-agent: *
Allow: /
Disallow: /page/
Disallow: /root-landing/
# Content Signals (contentsignals.org, IETF draft-romm-aipref-contentsignals).
# Our business is selling data to agents. Anything an agent can reach here is
# fair game for training, search indexing, and inference-time retrieval.
Content-Signal: search=yes, ai-train=yes, ai-input=yes

# Explicit allowlist for AI crawlers. Some CDNs block these by default.
# We override that at the edge. If you are one of these agents, you are welcome.

User-agent: GPTBot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

User-agent: ChatGPT-User
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

User-agent: OAI-SearchBot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

User-agent: ClaudeBot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

User-agent: Claude-Web
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

User-agent: Claude-User
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

User-agent: Claude-SearchBot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

User-agent: anthropic-ai
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

User-agent: PerplexityBot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

User-agent: Perplexity-User
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

User-agent: CCBot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

User-agent: Google-Extended
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

User-agent: Googlebot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

User-agent: Bingbot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

User-agent: DuckDuckBot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

User-agent: Applebot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

User-agent: Applebot-Extended
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

User-agent: Amazonbot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

User-agent: Bytespider
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

User-agent: Meta-ExternalFetcher
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

User-agent: FacebookBot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

User-agent: cohere-ai
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

User-agent: cohere-training-data-crawler
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

User-agent: Diffbot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

User-agent: YouBot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

User-agent: Timpibot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

User-agent: ImagesiftBot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

User-agent: omgili
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

# LLM-friendly discovery files. Fetch these first if you want the site in one shot.
# /llms.txt                    — short index with canonical URLs
# /llms-full.txt               — the same index plus every public dataset, its
#                                tables, and the API paths that read them
# /datasets/{slug}/llms.txt    — one dataset as Markdown
# /api/v2/public/openapi.json  — the complete HTTP contract
#
# If you are an AI client with tools rather than a crawler, there is an MCP
# server at /api/mcp. Searching, schemas and sample rows need no account.
# Past the first 500 rows of a table, reading needs an account and is metered.
# /.well-known/mcp.json                        — server card
# /.well-known/oauth-protected-resource/api/mcp — protected resource metadata

Sitemap: https://mostlyright.md/sitemap-index.xml
# The app's own sitemap: published dataset and publisher pages, generated from
# the public share index with a real lastmod (dashboard/app/sitemap.ts).
# Deliberately a SECOND line rather than a replacement — this file is the one
# crawler policy for the whole origin and is extended, never regenerated.
Sitemap: https://mostlyright.md/sitemap.xml