# www.mdlinx.com robots.txt (served from S3 through CloudFront; source of truth: docs/seo/www-robots.txt)
# Policy (MD-5292, 2026-09-09): allow search/answer and user-triggered AI agents, block training-only crawlers.
# Enforcement for bots that ignore robots.txt (e.g. Bytespider) is the HUMAN Security allowlist, not this file.

User-agent: *
Allow: /
Disallow: /qa-utils
Disallow: /auth-utils
Disallow: /api/

# AI search / answer engines and user-triggered fetchers: allowed (they send readers)
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Claude-SearchBot
User-agent: Claude-User
Allow: /
Disallow: /qa-utils
Disallow: /auth-utils
Disallow: /api/

# AI training crawlers: blocked (no attribution or referral back to MDLinx)
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: CCBot
User-agent: Google-Extended
User-agent: meta-externalagent
User-agent: Applebot-Extended
User-agent: Bytespider
User-agent: Amazonbot
Disallow: /

Sitemap: https://www.mdlinx.com/sitemap.xml