# 🤖 ROBOTS.TXT OTIMIZADO - 
# Última atualização: Agosto 2025
# Foco: SEO, Segurança, TODOS OS AIs PERMITIDOS

# ==========================================
# 📋 REGRAS GERAIS PARA TODOS OS BOTS
# ==========================================

User-agent: *

# ✅ RECURSOS ESSENCIAIS - Sempre permitidos
Allow: /wp-content/uploads/
Allow: /wp-includes/js/
Allow: /wp-content/themes/
Allow: *.css
Allow: *.js
Allow: *.png
Allow: *.jpg
Allow: *.jpeg
Allow: *.gif
Allow: *.svg
Allow: *.webp
Allow: *.pdf

# 🚫 WORDPRESS - Diretórios e arquivos sensíveis
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /xmlrpc.php
Disallow: /readme.html
Disallow: /license.txt
Disallow: /wp-login.php
Disallow: /cgi-bin/
Disallow: /tag/
Disallow: /tags/
Disallow: /uncategorized/
Disallow: */trackback
Disallow: /*/*?s=*

# 🚫 ECOMMERCE - Páginas privadas e checkout
Disallow: /loja/
Disallow: /carrinho/
Disallow: /cart/
Disallow: /checkout/
Disallow: /finalizar-compra/
Disallow: /minha-conta/
Disallow: /my-account/
Disallow: /conta/
Disallow: /verificacao-de-e-mail/

# 🚫 CONTEÚDO DUPLICADO - URLs problemáticas
Disallow: /search
Disallow: /?s=
Disallow: /page/
Disallow: /pagina/
Disallow: /?paged=
Disallow: /*/page/
Disallow: /*?page=
Disallow: /feed/
Disallow: /comments/feed/
Disallow: /rss
Disallow: /rss2
Disallow: /*/feed/
Disallow: /author/

# 🚫 PARÂMETROS - Evita URLs duplicadas
Disallow: /*?replytocom
Disallow: /*?orderby=
Disallow: /*?filter_
Disallow: /*?utm_
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?ref=
Disallow: /*?session=
Disallow: /*?add-to-cart=
Disallow: /*&
Disallow: */?add_to_wishlist=* 
Disallow: */?max_price*
Disallow: */?min_price*
Disallow: */politica-de-privacidade/*

# 🚫 ARQUIVOS SENSÍVEIS - Segurança
Disallow: /*.git$
Disallow: /*.sql$
Disallow: /*.tgz$
Disallow: /*.gz$
Disallow: /*.tar$
Disallow: /*.svn$
Disallow: /*.bz2$
Disallow: /*.log$
Disallow: /*.zip$
Disallow: /*.old$
Disallow: /cgi-bin/
Disallow: /wp-json/
Disallow: /trackback/
Disallow: /?rest_route=
Disallow: /*.php$
Disallow: /*.cgi$
Disallow: /*.inc$
Disallow: /*.xhtml$

# ==========================================
# 🔍 BOTS DE SEO - Configurações específicas
# ==========================================

# ✅ GOOGLEBOT - Recursos críticos sempre permitidos
User-Agent: Googlebot
Allow: /*.css$
Allow: /*.js$
Allow: /wp-content/uploads/
Allow: /wp-content/uploads/*
Allow: /wp-content/themes/
Allow: /wp-*.png          
Allow: /wp-*.jpg          
Allow: /wp-*.jpeg        
Allow: /wp-*.gif           
Allow: /wp-*.svg          
Allow: /wp-*.pdf
Allow: /wp-*.webp
Allow: /wp-content/*.js
Allow: /wp-content/*.css
Allow: /wp-includes/*.js
Allow: /wp-includes/*.css
Allow: /wp-admin/admin-ajax.php

# ⚖️ FERRAMENTAS SEO ESSENCIAIS

# ✅ MOZ - Permitido (dados únicos, menos usado pela concorrência)
User-agent: rogerbot
Allow: /

User-agent: dotbot
Allow: /

# ✅ AHREFS - Permitido (essencial para backlinks)
User-agent: AhrefsBot
Allow: /

# 🚫 SEMRUSH - Bloqueado (concorrência)
User-agent: SemrushBot
Disallow: /

# 🚫 BOTS PROBLEMÁTICOS - Bloqueados estrategicamente
User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-OCOB
Disallow: /

User-agent: SemrushBot-SWA
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: BLEXBot
Disallow: /

# ==========================================
# 🤖 BOTS DE IA ESSENCIAIS - TOP 9 PERMITIDOS
# ==========================================

# ✅ CHATGPT (OpenAI) - PERMITIDO
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

# ✅ SEARCHGPT (OpenAI Search) - PERMITIDO
User-agent: SearchGPT-User
Allow: /

User-agent: SearchGPT
Allow: /

# ✅ CLAUDE (Anthropic) - PERMITIDO
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

# ✅ GOOGLE GEMINI - PERMITIDO
User-agent: Google-Extended
Allow: /

User-agent: Gemini-Deep-Research
Allow: /

# ✅ META AI (Facebook/Instagram) - PERMITIDO
User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

# ✅ PERPLEXITY AI - PERMITIDO
User-agent: PerplexityBot
Allow: /

# ✅ COMMON CRAWL (Treina múltiplas AIs) - PERMITIDO
User-agent: CCBot
Allow: /

# ✅ GROK (xAI/Elon Musk) - PERMITIDO
User-agent: GrokBot
Allow: /

User-agent: XAI-Bot
Allow: /

User-agent: xAI-Grok
Allow: /

# ✅ MICROSOFT COPILOT - PERMITIDO
User-agent: Bingbot
Allow: /

# ==========================================
# 🚫 SCRAPERS MALICIOSOS - Sempre bloqueados
# ==========================================

# ⚠️ SCRAPERS GENÉRICOS
User-agent: Scrapy
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Wget
Disallow: /

User-agent: curl
Disallow: /

User-agent: libwww-perl
Disallow: /

User-agent: Python-urllib
Disallow: /

User-agent: Java
Disallow: /

User-agent: Jakarta
Disallow: /

# ⚠️ BOTS DE CONTEÚDO/SPAM - NOVOS ADICIONADOS
User-agent: SiteSnagger
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebWhacker
Disallow: /

User-agent: RealDownload
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: WebDownloader
Disallow: /

User-agent: BackStreet
Disallow: /

User-agent: PageNest
Disallow: /

# ⚠️ CRAWLERS AGRESSIVOS - NOVOS ADICIONADOS
User-agent: WebBandit
Disallow: /

User-agent: WebRipper
Disallow: /

User-agent: NetAnts
Disallow: /

User-agent: FlashGet
Disallow: /

User-agent: GetRight
Disallow: /

User-agent: GetWeb
Disallow: /

User-agent: Go-http-client
Disallow: /

User-agent: Go!Zilla
Disallow: /

User-agent: Download Demon
Disallow: /

User-agent: Download Wonder
Disallow: /

User-agent: ReGet
Disallow: /

User-agent: SuperHTTP
Disallow: /

User-agent: Surfbot
Disallow: /

User-agent: JetCar
Disallow: /

# ⚠️ BOTS AUTOMATIZADOS PROBLEMÁTICOS - APENAS OS MAIS ESPECÍFICOS
User-agent: Python-requests/1.0
Disallow: /

User-agent: Python-requests/2.0
Disallow: /

User-agent: urllib/1.0
Disallow: /

User-agent: urllib2/1.0
Disallow: /

# NOTA: Linguagens genéricas (PHP, Ruby, Node.js) foram removidas
# para evitar bloqueio de ferramentas legítimas e APIs oficiais

# ⚠️ BOTS DE EMAIL/SPAM - NOVOS ADICIONADOS
User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: EmailWolf
Disallow: /

User-agent: ExtractorPro
Disallow: /

User-agent: CherryPicker
Disallow: /

User-agent: EmailWalk
Disallow: /

User-agent: SpamBot
Disallow: /

# ⚠️ BOTS DE IMAGENS/MÍDIA - NOVOS ADICIONADOS
User-agent: psbot
Disallow: /

User-agent: ImageFetch
Disallow: /

User-agent: PictureFinder
Disallow: /

User-agent: MediaBot
Disallow: /

# ⚠️ COPIADORES DE SITE - NOVOS ADICIONADOS
User-agent: SiteSucker
Disallow: /

User-agent: WebSite-X
Disallow: /

User-agent: Website eXtractor
Disallow: /

User-agent: WebsiteExtractor
Disallow: /

User-agent: WebMaster
Disallow: /

User-agent: WebCapture
Disallow: /

User-agent: NetZip
Disallow: /

# ⚠️ BOTS MALICIOSOS CONHECIDOS - NOVOS
User-agent: PaperLiBot
Disallow: /

User-agent: SputnikBot
Disallow: /

User-agent: CCResearchBot
Disallow: /

User-agent: safeDNSbot
Disallow: /

User-agent: Nmap
Disallow: /

User-agent: masscan
Disallow: /

User-agent: zmap
Disallow: /

# ==========================================
# 🗺️ SITEMAP
# ==========================================

Sitemap: https://oniontv.site/sitemap_index.xml