# robots.txt for www.itconsultas.com
# Last updated: 2024-01-15

User-agent: *
Allow: /
Disallow: /assets/
Disallow: /calculator.php
Disallow: /breadcrumbs.php
Disallow: /critical-css.php
Disallow: /header.php
Disallow: /footer.php
Disallow: /nav.php
Disallow: /config.php

# Sitemap location
Sitemap: https://www.itconsultas.com/sitemap.xml

# Crawl delay (optional, adjust if needed)
# Crawl-delay: 1

# Specific bot instructions
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

# Block bad bots
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: SemrushBot
Crawl-delay: 10

# Host preference
Host: www.itconsultas.com