# Policy: crawl the live catalogue as hard and as fast as you like. Bandwidth is
# not a constraint here and wide ingestion is the point. One exception:
#   - Amazon is blocked outright
#

# Content Signals Policy (contentsignals.org): our preferences for how
# automated systems may use this content. All three are permitted - the
# catalogue is published CC0 and an MCP server is provided at /mcp for
# agents that would rather query it than crawl it.
#   search    = build a search index and link back
#   ai-input  = use as input to an AI answer (RAG, summarisation)
#   ai-train  = use to train or fine-tune a model

# Nothing is disallowed, and that is deliberate. The superseded renderer
# output used to be, and a Disallow was the wrong tool: every retired
# generation now 301s to the html6 homepage, but a Disallow stops a crawler
# fetching the URL at all, so it never sees the redirect and the entry stays
# in the index on the strength of inbound links. Google's coverage report on
# 2026-09-19 showed exactly that - four /python/html/ URLs "Indexed, though
# blocked by robots.txt", unremovable for as long as they were blocked.
# Letting crawlers in is what retires them.
#
# html6 is the only live tree. html, html1-html4 and html5 all redirect to
# it; crawl any of them and you will be sent there.
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
# An empty value is the standard spelling of "nothing is disallowed", and it
# keeps the group from being a User-agent line with no rule under it, which
# not every parser handles the same way.
Disallow:
Crawl-delay: 0

# Amazon go away
User-agent: Amazonbot
Disallow: /

User-agent: Amzn-SearchBot
Disallow: /

User-agent: Amzn-User
Disallow: /

# Sitemaps
# html6 only. html5 is still served and still indexed, but it was advertised
# twice here - /sitemap.xml and /python/sitemaps5/sitemapindex.xml.gz are two
# indexes over the same 1,279 chunks under /sitemaps5/ - and pointing crawlers
# at the superseded generation spends crawl budget that html6 needs. Both
# files remain in place so existing crawler state and direct requests still
# resolve; they are simply no longer advertised.
Sitemap: https://www.dataalchemy.com/python/sitemaps6/sitemapindex.xml.gz