##### 1. Trusted search engines
User-agent: Googlebot
Allow: /
Disallow: /signup           

User-agent: Bingbot
Allow: /
Disallow: /signup

User-agent: Yandex
Allow: /
Disallow: /signup
Crawl-delay: 2              

##### 2. Basic restrictions for all others
User-agent: *
Disallow: /signup
Crawl-delay: 10            # reduces request frequency

##### 3. Blocking aggressive SEO scanners
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot        # Majestic
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

##### 4. Neural networks and AI bots
# 4.1 "User" bots — minimal access (via direct links)
User-agent: ChatGPT-User
User-agent: ChatGPT-User/2.0
User-agent: Perplexity-User
User-agent: Claude-User
Allow: /
Crawl-delay: 5

# 4.2 AI search engines (completely blocked; remove Disallow if needed)
User-agent: PerplexityBot
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: DuckAssistBot
Disallow: /

# 4.3 Bots for training models — blocked
User-agent: GPTBot            # OpenAI
Disallow: /

User-agent: ClaudeBot         # Anthropic
User-agent: anthropic-ai
Disallow: /

User-agent: CCBot             # Common Crawl
Disallow: /

User-agent: Bytespider        # ByteDance
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: AI2Bot
Disallow: /

# 4.4 Extended crawlers from search engines (for generative responses)
User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

# ------------------ end of robots.txt file ------------------