# Great Library Project
# Free to read, quote and link. The translations, word glosses and notes are our work: cite the
# passage with a link. No bulk copying, republishing, or AI training without permission.
# Terms: https://greatlibraryproject.com/license.html

# Training-only crawlers: you may learn who we are (front door, about, shelves, llms.txt),
# but please stay out of the books themselves (readers and passage pages).
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: Bytespider
User-agent: meta-externalagent
User-agent: cohere-training-data-crawler
Allow: /$
Allow: /index.html
Allow: /about.html
Allow: /license.html
Allow: /help.html
Allow: /connect.html
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /collection/
Allow: /shelves/
Disallow: /

# Everyone else, including search engines and the AI search/answer bots that cite and link
# (Bingbot, Googlebot, OAI-SearchBot, ChatGPT-User, Claude-SearchBot, Claude-User, PerplexityBot).
User-agent: *
Allow: /

Sitemap: https://greatlibraryproject.com/sitemap.xml
Sitemap: https://greatlibraryproject.com/sitemap-passages.xml
Sitemap: https://greatlibraryproject.com/sitemap-explainers.xml
Sitemap: https://greatlibraryproject.com/sitemap-english.xml