# robots.txt for turchi.pr.it — GENERATED. Do not edit this file.
# Source: src/lib/robots.ts, emitted by src/pages/robots.txt.ts.
#
# robots.txt controls CRAWLING; the `robots` meta tag controls INDEXING.
# A URL that is disallowed here is never fetched, so its `noindex` is never
# read and the URL can still be indexed from inbound links. Every page that
# must stay out of the index is therefore left crawlable below and carries
# `noindex` in its own HTML. Those pages are:
#   - the configurator tool, 23 URLs (e.g. /it/configuratore/avvia) — an
#     application, not a document: nothing on it answers a search
#   - the photovoltaic campaign landing page, 23 URLs (e.g.
#     /it/battipalo-fotovoltaico) — paid traffic only; it would compete with the
#     photovoltaic sector page organically
#   - the 404 documents, 24 URLs (/404 and
#     /<locale>/404) — a page that says a page does not exist must not be
#     the page a reader lands on from a search result
#
# Everything else is meant to be crawled AND indexed.

User-agent: *
Allow: /

# Internal component galleries. A production build does not emit them at
# all — src/pages/[locale]/dev/[slug].astro gates getStaticPaths on
# import.meta.env.DEV — so this is a second line of defence for a preview.
Disallow: /*/dev/

# JSON endpoints proxied by nginx (turchi-buf, turchi-core, swetsi). They
# return no HTML and are not pages; fetching them only spends rate limit.
Disallow: /api/

# AI retrieval and answer agents, named explicitly. The wildcard above
# already allows them, so these blocks change nothing today. They exist so
# the intent is legible: a CDN rule or a hosting default that blocks these
# agents is a common silent regression whose only symptom is disappearing
# from AI answers, with no ranking change to explain it. Two families, kept
# apart on purpose: search and citation agents, which is the traffic we
# want, and user-triggered fetchers, which retrieve a page a person asked
# about. Neither is a training crawler.

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Applebot
Allow: /

User-agent: Google-Extended
Allow: /

Sitemap: https://turchi.pr.it/sitemap-index.xml