# SwimBikeRun.ph - Ultimate SEO & Performance Optimized Robots.txt
# Version: Gold Standard | Last Updated: March 29, 2025

# ✅ Allow Googlebot to index core site content
User-agent: Googlebot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Sitemap: https://swimbikerun.ph/sitemap_index.xml

# ❌ Block BingBot completely (Too resource-intensive, not valuable)
User-agent: Bingbot
Disallow: /

# ❌ Block known aggressive crawlers to preserve bandwidth and server resources
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: Sogou
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: archive.org_bot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Bytespider
Disallow: /

# 🚫 Control specific bot activity to prevent excessive dynamic page hits
User-agent: meta-externalagent
Disallow: /calendar/
Allow: /

# 🔗 Allow social media crawlers for proper sharing previews
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

# 🔒 Block backend WordPress directories, but allow needed frontend assets
User-agent: *
Disallow: /cgi-bin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /xmlrpc.php
Disallow: /trackback/
Disallow: /search/
Disallow: /feed/
Disallow: /login/
Disallow: /signup/
Disallow: /author/
Disallow: /private/
Disallow: /tmp/

# 🛡 Block indexing of sensitive or irrelevant file types
User-agent: *
Disallow: /*.php$
Disallow: /*.cgi$
Disallow: /*.txt$
Disallow: /*.log$
Disallow: /*.zip$
Disallow: /*.sql$
Disallow: /*.json$
Disallow: /*.inc$
Disallow: /*.asp$
Disallow: /*.htm$
Disallow: /*.shtml$
Disallow: /*.pl$

# 🎯 Allow media, uploads, and AJAX functionality
Allow: /wp-content/uploads/
Allow: /wp-admin/admin-ajax.php

# 📍 Sitemap for structured crawling
Sitemap: https://swimbikerun.ph/sitemap_index.xml