# =============================================
# General search engine rules
# =============================================
# The SingleHTML build of a manual is no longer produced. It disappears
# from a manual with that manual's next render; until then the old file
# is still there, and it stays off limits: several megabytes per manual,
# up to ~33 MB, duplicating content that is already indexable page by
# page. Clients that need a whole manual should use the machine-readable
# artifacts described in https://docs.typo3.org/llms.txt (toc.json,
# objects.inv.json, objects.inv, _sources/), which are allowed below.
# A single page is best read as Markdown, at its own URL with ".md"
# instead of ".html"; nothing here disallows it.
User-agent: *
Disallow: /getthedocs/files/
Disallow: /services/
Disallow: /search/
Disallow: */draft/*
Disallow: */singlehtml/*
Allow: /*/toc.json
Allow: /*/Changelog-*.json
Allow: /*/objects.inv
Allow: /*/objects.inv.json
Allow: /*/_sources/

# =============================================
# AI and LLM-related crawlers
# Access limited to the machine-readable artifacts and the Markdown of
# a page
# =============================================

User-agent: Amazonbot
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: CCBot
User-agent: PerplexityBot
User-agent: AppleBot
User-agent: PhindBot
User-agent: YouBot

Disallow: /
Allow: /llms.txt
Allow: /permalink/

# Machine-readable artifacts. These are the preferred entry points:
# they are small, structured, cheaper for this host to serve than
# rendered HTML, and easier to parse correctly. They are allowed for
# every manual, every version and every language.
#   toc.json          the table of contents of a manual: its pages in
#                     reading order, nested, with their anchors. The
#                     cheapest way to learn what a manual covers -- a
#                     fraction of the inventory's size. Published for a
#                     manual once it has been rendered again, so not
#                     everywhere yet.
#   Changelog-*.json  the TYPO3 Core Changelog by major release: every
#                     entry with its permalink, title, kind, issue and
#                     tags. Core Changelog only.
#   objects.inv.json  inventory of a manual as JSON: every title,
#                     name and anchor. Use it to resolve a topic to an
#                     exact anchor, or to reach a named entity.
#                     Serve/request it gzipped.
#   objects.inv       the same inventory in Sphinx's native format
#                     (zlib-compressed; needs a Sphinx parser)
#   *.md              the Markdown of a page: the same content as the
#                     rendered page, without navigation or markup and
#                     at a fraction of its size. The format to read a
#                     page in. Published for a manual once it has been
#                     rendered again, so not everywhere yet.
#   _sources/         the reStructuredText source of each page.
#                     A cheaper supplement, not a replacement: includes
#                     and generated content (e.g. the ViewHelper
#                     reference) resolve only at build time, so the
#                     source can hold less than the rendered page.
#                     Superseded by the Markdown where that exists.
Allow: /*/toc.json
Allow: /*/Changelog-*.json
Allow: /*/objects.inv
Allow: /*/objects.inv.json
Allow: /*.md$
Allow: /*/_sources/