66 lines
921 B
Plaintext
66 lines
921 B
Plaintext
# DictIA - Block all web crawlers and search engines
|
|
# This application contains private user data and should not be indexed
|
|
|
|
User-agent: *
|
|
Disallow: /
|
|
|
|
# Specific directives for major search engines
|
|
User-agent: Googlebot
|
|
Disallow: /
|
|
|
|
User-agent: Googlebot-Image
|
|
Disallow: /
|
|
|
|
User-agent: Bingbot
|
|
Disallow: /
|
|
|
|
User-agent: Slurp
|
|
Disallow: /
|
|
|
|
User-agent: DuckDuckBot
|
|
Disallow: /
|
|
|
|
User-agent: Baiduspider
|
|
Disallow: /
|
|
|
|
User-agent: YandexBot
|
|
Disallow: /
|
|
|
|
User-agent: ia_archiver
|
|
Disallow: /
|
|
|
|
# AI Crawlers
|
|
User-agent: GPTBot
|
|
Disallow: /
|
|
|
|
User-agent: ChatGPT-User
|
|
Disallow: /
|
|
|
|
User-agent: CCBot
|
|
Disallow: /
|
|
|
|
User-agent: anthropic-ai
|
|
Disallow: /
|
|
|
|
User-agent: Claude-Web
|
|
Disallow: /
|
|
|
|
User-agent: cohere-ai
|
|
Disallow: /
|
|
|
|
# Social Media Crawlers
|
|
User-agent: facebookexternalhit
|
|
Disallow: /
|
|
|
|
User-agent: Twitterbot
|
|
Disallow: /
|
|
|
|
User-agent: LinkedInBot
|
|
Disallow: /
|
|
|
|
User-agent: Slackbot
|
|
Disallow: /
|
|
|
|
User-agent: Discordbot
|
|
Disallow: /
|