# Electric Scouts - Robots.txt
# https://www.electricscouts.com

# Default rules for all crawlers
User-agent: *
Allow: /

# Public content paths
Allow: /electricity-rates/
Allow: /learn/
Allow: /providers/
Allow: /compare-rates
Allow: /bill-analyzer
Allow: /savings-calculator
Allow: /all-providers
Allow: /all-states
Allow: /all-cities
Allow: /learning-center
Allow: /renewable-energy
Allow: /renewable-compare-rates
Allow: /business-electricity
Allow: /business-compare-rates
Allow: /business-hub
Allow: /faq
Allow: /about-us
Allow: /home-concierge
Allow: /privacy-policy
Allow: /terms-of-service

# State electricity pages
Allow: /texas-electricity
Allow: /illinois-electricity
Allow: /ohio-electricity
Allow: /pennsylvania-electricity
Allow: /new-york-electricity
Allow: /new-jersey-electricity
Allow: /maryland-electricity
Allow: /massachusetts-electricity
Allow: /maine-electricity
Allow: /new-hampshire-electricity
Allow: /rhode-island-electricity
Allow: /connecticut-electricity

# Block admin, API, and internal areas
Disallow: /admin/
Disallow: /api/
Disallow: /go/
Disallow: /user-settings
Disallow: /business-quote-dashboard
Disallow: /not-found

# Block utility/meta pages that should not be indexed
Disallow: /robots
Disallow: /sitemap
Disallow: /sitemap-xml

# Block legacy query-param URLs (superseded by clean URLs)
Disallow: /city-rates
Disallow: /article-detail

# Block duplicate content pages
Disallow: /landing

# Sitemap location (canonical www domain)
Sitemap: https://www.electricscouts.com/sitemap.xml

# Google-specific rules
User-agent: Googlebot
Allow: /

# Bing-specific rules
User-agent: Bingbot
Allow: /
Crawl-delay: 2

# Block AI training crawlers
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /