# Crawl us. Discovery is the point - but crawl the pages, not the paywall.
#
# Measured 2026-08-07 over 24h: Amazonbot spent 42% of its fetches on /v1/
# URLs that answer 402 and can never be indexed; Meta 34%, GoogleOther 34%.
# Googlebot figured this out on its own (1%). The disallow below points
# everyone else's budget at the HTML pages, which is where the content is.
# Everything under /v1/ is documented in the machine entry points below -
# an agent that wants the API does not need to crawl it blind.
# The exceptions are not API: /v1/answers/* and /v1/vat/determine serve
# HTML documentation pages and they are meant to be indexed. Longest
# match wins for Google and Bing, so these Allow rules beat Disallow.
# /provenance/{cc}/{series} dumps are source fingerprints for machines,
# not search results. Disallow them so crawl budget goes to /{cc}/{series}.
User-agent: *
Disallow: /v1/
Disallow: /provenance/
Allow: /v1/changes
Allow: /v1/calendar
Allow: /v1/business-days/
Allow: /v1/answers/
Allow: /v1/vat/determine

# Google and Bing get the same treatment as everyone else. They used to
# have a blanket Allow because both already self-manage their /v1/ budget
# sensibly - Googlebot spends about 1% there. Closed anyway: Search Console
# reports 130 pages 'Discovered - currently not indexed', which is Google
# declining to spend crawls, and every crawl of a 402 is one not spent on a
# page that can actually be indexed.
User-agent: Googlebot
Disallow: /v1/
Disallow: /provenance/
Allow: /v1/changes
Allow: /v1/calendar
Allow: /v1/business-days/
Allow: /v1/answers/
Allow: /v1/vat/determine

User-agent: Bingbot
Disallow: /v1/
Disallow: /provenance/
Allow: /v1/changes
Allow: /v1/calendar
Allow: /v1/business-days/
Allow: /v1/answers/
Allow: /v1/vat/determine

# Paid endpoints answer 402 (x402) to an unpaid request rather than 403 —
# nothing is hidden; the disallow is crawl-budget hygiene, not secrecy.
Sitemap: https://usaref.dev/sitemap.xml

# Machine-readable entry points:
#   https://usaref.dev/llms.txt                 endpoints, prices, example bodies
#   https://usaref.dev/catalog                  every country and series held
#   https://usaref.dev/openapi.json             OpenAPI description
#   https://usaref.dev/.well-known/x402.json    x402 resource manifest
#   https://usaref.dev/v1/changes               free change + freshness feed
#   https://usaref.dev/v1/calendar              free re-verification calendar (not a gazette schedule)
#   https://usaref.dev/v1/business-days/        free business-day tools: is-business-day, add, between

# Human-readable pages, one per country and per series:
#   https://usaref.dev/countries                every country we hold
#   https://usaref.dev/{country}/{series}       e.g. /za/vat — value, source, history