# robots.txt — textopian.com
# Updated: 2026-05-25 (explicit AI search crawler access + public edge smoke)
#
# Policy summary:
# - Public pages (authors, titles, genres, hub pages, blog, /read, /cite) are open to all crawlers.
# - /read/* pages serve a 2-verse free sample to unauthenticated callers. Full text requires sign-in.
#   The sample is identical for bots and anonymous humans (schema.org Flexible Sampling — not cloaking).
# - Private and session-scoped paths are blocked.
# - See /llms.txt for AI-specific guidance and canonical citation URLs.

########################
# Global defaults
########################
User-agent: *
Allow: /author/
Allow: /author
Allow: /genres/
Allow: /genres
Allow: /genre/
Allow: /read/
Allow: /clubs/
Allow: /clubs
Allow: /club/
Allow: /groups/
Allow: /groups
Allow: /group/
Allow: /cite/
Allow: /blog/
Allow: /blog
Allow: /docs/
Allow: /docs
Allow: /classic-books
Allow: /public-domain
Allow: /pricing
Allow: /patron
Allow: /teachers
Allow: /schools
Allow: /scholarship
Allow: /contact
Allow: /privacy-accessibility
Allow: /privacy
Allow: /terms
Allow: /cookies
Allow: /dmca
Allow: /gdpr
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /sitemap.xml
Allow: /sitemap-blog.xml
Allow: /sitemap-docs.xml
Allow: /blog/feed.xml

# Private, session-scoped, and operational areas
Disallow: /me/
Disallow: /account/
Disallow: /admin/
Disallow: /tenant/
Disallow: /document/
Disallow: /public/webviewer/
Disallow: /passage/
Disallow: /report/
Disallow: /scholarships/
Disallow: /search/
Disallow: /graphql
Disallow: /health/
Disallow: /dev/
Disallow: /s/
Disallow: /*?*

# Politeness (ignored by Google/Bing — honored by most others)
Crawl-delay: 5

Sitemap: https://textopian.com/sitemap.xml
Sitemap: https://textopian.com/sitemap-blog.xml
Sitemap: https://textopian.com/sitemap-docs.xml

########################
# Explicit AI/LLM crawlers
# Redundant with the global group, but makes intent crystal-clear for each bot.
########################

User-agent: GPTBot
Allow: /
Disallow: /me/
Disallow: /account/
Disallow: /admin/
Disallow: /tenant/
Disallow: /document/
Disallow: /public/webviewer/
Disallow: /passage/
Disallow: /report/
Disallow: /scholarships/
Disallow: /search/
Disallow: /graphql
Disallow: /health/
Disallow: /dev/
Disallow: /s/
Disallow: /*?*
Crawl-delay: 5

User-agent: OAI-SearchBot
Allow: /
Disallow: /me/
Disallow: /account/
Disallow: /admin/
Disallow: /tenant/
Disallow: /document/
Disallow: /public/webviewer/
Disallow: /passage/
Disallow: /report/
Disallow: /scholarships/
Disallow: /search/
Disallow: /graphql
Disallow: /health/
Disallow: /dev/
Disallow: /s/
Disallow: /*?*
Crawl-delay: 5

User-agent: ChatGPT-User
Allow: /
Disallow: /me/
Disallow: /account/
Disallow: /admin/
Disallow: /tenant/
Disallow: /document/
Disallow: /public/webviewer/
Disallow: /passage/
Disallow: /report/
Disallow: /scholarships/
Disallow: /search/
Disallow: /graphql
Disallow: /health/
Disallow: /dev/
Disallow: /s/
Disallow: /*?*
Crawl-delay: 5

User-agent: ClaudeBot
Allow: /
Disallow: /me/
Disallow: /account/
Disallow: /admin/
Disallow: /tenant/
Disallow: /document/
Disallow: /public/webviewer/
Disallow: /passage/
Disallow: /report/
Disallow: /scholarships/
Disallow: /search/
Disallow: /graphql
Disallow: /health/
Disallow: /dev/
Disallow: /s/
Disallow: /*?*
Crawl-delay: 5

User-agent: PerplexityBot
Allow: /
Disallow: /me/
Disallow: /account/
Disallow: /admin/
Disallow: /tenant/
Disallow: /document/
Disallow: /public/webviewer/
Disallow: /passage/
Disallow: /report/
Disallow: /scholarships/
Disallow: /search/
Disallow: /graphql
Disallow: /health/
Disallow: /dev/
Disallow: /s/
Disallow: /*?*
Crawl-delay: 5

User-agent: Applebot
Allow: /
Disallow: /me/
Disallow: /account/
Disallow: /admin/
Disallow: /tenant/
Disallow: /document/
Disallow: /public/webviewer/
Disallow: /passage/
Disallow: /report/
Disallow: /scholarships/
Disallow: /search/
Disallow: /graphql
Disallow: /health/
Disallow: /dev/
Disallow: /s/
Disallow: /*?*
Crawl-delay: 5

User-agent: Applebot-Extended
Allow: /
Disallow: /me/
Disallow: /account/
Disallow: /admin/
Disallow: /tenant/
Disallow: /document/
Disallow: /public/webviewer/
Disallow: /passage/
Disallow: /report/
Disallow: /scholarships/
Disallow: /search/
Disallow: /graphql
Disallow: /health/
Disallow: /dev/
Disallow: /s/
Disallow: /*?*
Crawl-delay: 5

User-agent: CCBot
Allow: /
Disallow: /me/
Disallow: /account/
Disallow: /admin/
Disallow: /tenant/
Disallow: /document/
Disallow: /public/webviewer/
Disallow: /passage/
Disallow: /report/
Disallow: /scholarships/
Disallow: /search/
Disallow: /graphql
Disallow: /health/
Disallow: /dev/
Disallow: /s/
Disallow: /*?*
Crawl-delay: 5

User-agent: Google-Extended
Allow: /
Disallow: /me/
Disallow: /account/
Disallow: /admin/
Disallow: /tenant/
Disallow: /document/
Disallow: /public/webviewer/
Disallow: /passage/
Disallow: /report/
Disallow: /scholarships/
Disallow: /search/
Disallow: /graphql
Disallow: /health/
Disallow: /dev/
Disallow: /s/
Disallow: /*?*

User-agent: meta-externalagent
Allow: /
Disallow: /me/
Disallow: /account/
Disallow: /admin/
Disallow: /tenant/
Disallow: /document/
Disallow: /public/webviewer/
Disallow: /passage/
Disallow: /report/
Disallow: /scholarships/
Disallow: /search/
Disallow: /graphql
Disallow: /health/
Disallow: /dev/
Disallow: /s/
Disallow: /*?*
Crawl-delay: 5

User-agent: Amazonbot
Allow: /
Disallow: /me/
Disallow: /account/
Disallow: /admin/
Disallow: /tenant/
Disallow: /document/
Disallow: /public/webviewer/
Disallow: /passage/
Disallow: /report/
Disallow: /scholarships/
Disallow: /search/
Disallow: /graphql
Disallow: /health/
Disallow: /dev/
Disallow: /s/
Disallow: /*?*
Crawl-delay: 5