# ============================================
# Rivers of Steel (ROS) - robots.txt
# Updated: March 2026
# Purpose: Preserve Yoast rules + block aggressive bots
# Based on actual access log data from rosteelprd
# ============================================

# ----- GOOD BOTS (allow these - they help your SEO) -----
User-agent: Googlebot
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

User-agent: Bingbot
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

User-agent: Slurp
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

User-agent: DuckDuckBot
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# ----- AMAZON BOT (the big offender - 20K+ requests) -----
User-agent: Amazonbot
Disallow: /

# ----- APPLE BOT (4,870 requests - not worth the server cost) -----
User-agent: Applebot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

# ----- SEO SCRAPER BOTS -----
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: YandexBot
Disallow: /

# ----- IMAGE / MEDIA CRAWLERS -----
User-agent: TinEye
Disallow: /

# ----- AI TRAINING BOTS -----
User-agent: Bytespider
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

# START YOAST BLOCK
# ---------------------------
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Crawl-delay: 5

# Block expensive event calendar views for remaining bots
Disallow: /events/photo/
Disallow: /events/week/
Disallow: /events/map/
Disallow: /events/month/

Sitemap: https://riversofsteel.com/sitemap_index.xml
# ---------------------------
# END YOAST BLOCK