User-agent: *
Disallow: /api/
Disallow: /client/token
Disallow: /payment/


# GPTBot is OpenAI web crawler
User-agent: GPTBot
Disallow: /

# Allows us to block Google's bot Bard
User-agent: Google-Extended
Disallow: /

# ChatGPT-User is OpenAI web crawler
User-agent: ChatGPT-User
Disallow: /

# Common Crawl bot
User-agent: CCBot
Disallow: /

# PiplBot is PiplBot's web crawler
User-agent: PiplBot
Disallow: /

# anthropic-ai is Anthropic's web crawler
User-agent: anthropic-ai
Disallow: /

# Claude-Web is Claude web crawler
User-agent: Claude-Web
Disallow: /

# TurnitinBot is Turnitin web crawler
User-agent: TurnitinBot
Disallow: /

# PetalBot is Petal web crawler
User-agent: PetalBot
Disallow: /

# MoodleBot is Moodle web crawler
User-agent: MoodleBot
Disallow: /

# magpie-crawler is Brandwatch.com web crawler
User-agent: magpie-crawler
Disallow: /

# ia_archiver is Wayback Machine web crawler
User-agent: ia_archiver
Disallow: /