# https://asha.news/robots.txt
#
# Asha News is evidence-first journalism meant to be read, cited and quoted —
# by people and by machines. AI crawlers and answer engines are welcome, and
# are named explicitly below rather than left to the wildcard, because several
# operators treat an unnamed group as ambiguous.

User-agent: *
Allow: /

# Private surfaces (/admin, /auth, /preferences, /subscription, /welcome) are
# NOT disallowed here on purpose. They are served with
# `X-Robots-Tag: noindex, nofollow` (server/lib/routeMeta.js), which applies to
# every agent regardless of which robots.txt group it follows. Disallowing them
# would stop crawlers fetching the header at all, and a disallowed URL can
# still be indexed URL-only — the weaker outcome.

Sitemap: https://asha.news/sitemap.xml

# --- Search engines ---------------------------------------------------------

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

# --- AI crawlers and answer engines -----------------------------------------
# Explicitly allowed. Keep in sync with server/lib/crawlerUserAgents.js —
# server/tests/robotsTxt.test.js fails if a name here goes missing.

User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: CCBot
Allow: /

# --- Agent discovery --------------------------------------------------------
# Machine-readable entry points. These are discovery targets, not index
# targets, so they are advertised here rather than in the XML sitemap.
#
# LLM prompt:        https://asha.news/llms.txt
# Full LLM context:  https://asha.news/llms-full.txt
# Agent descriptor:  https://asha.news/.well-known/agent
# Agent skills:      https://asha.news/.well-known/agent-skills/index.json
# OpenAPI contract:  https://asha.news/api/v1/openapi
# Editorial policy:  https://asha.news/.well-known/editorial-guidelines
# RSS:               https://asha.news/rss.xml