# =============================================================================
# robots.txt — Brasil em Folhas
# Versão 2.0 — 11/09/2026
#
# Contexto: 674.849 URLs conhecidas, 33.796 indexadas (5%).
# 544.881 páginas foram rastreadas e recusadas pelo Google.
# Esta versão concentra o rastreamento no que ainda pode ranquear.
#
# Lembrete: Disallow NÃO desindexa. Para tirar do índice use noindex.
# =============================================================================


User-agent: *

Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-register.php
Disallow: /xmlrpc.php
Disallow: /wp-content/cache/
Disallow: /wp-content/upgrade/
Disallow: /wp-content/backup*/
Disallow: /readme.html
Disallow: /license.txt
Disallow: /bef-config.php
Disallow: /bef-duplicatas.php
Disallow: /bef-teste.php
Disallow: /bef-indexnow.php
Disallow: /bef-cache/

# Busca interna
Disallow: /?s=
Disallow: /search/
Disallow: /*?s=

# Parâmetros sem valor de indexação
Disallow: /*?replytocom=
Disallow: /*?attachment_id=
Disallow: /*?preview=true
Disallow: /*?p=
Disallow: /*?page_id=
Disallow: /*?orderby=
Disallow: /*?filter=
Disallow: /*/embed/
Disallow: /comments/feed/
Disallow: /*/trackback/
Disallow: /trackback/

# v2 — paginação profunda. Com 540 mil matérias, /page/2000/ e adiante são
# armadilha de rastreamento pura. As 10 primeiras páginas bastam para
# descoberta; o resto vem por sitemap e links internos.
Disallow: /page/
Allow: /page/1/
Allow: /page/2/
Allow: /page/3/
Allow: /page/4/
Allow: /page/5/
Allow: /page/6/
Allow: /page/7/
Allow: /page/8/
Allow: /page/9/
Allow: /page/10/

# Recursos de renderização — NUNCA bloquear
Allow: /wp-content/uploads/
Allow: /wp-includes/js/
Allow: /wp-content/themes/
Allow: /wp-content/plugins/
Allow: /*.css$
Allow: /*.js$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.webp$
Allow: /*.avif$
Allow: /*.gif$
Allow: /*.svg$

# --- FASE 2: só descomentar depois que as tags saírem do índice ------------
# Disallow: /tag/
# ---------------------------------------------------------------------------


# -----------------------------------------------------------------------------
# GOOGLE
# -----------------------------------------------------------------------------
User-agent: Googlebot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /?s=
Disallow: /search/
Disallow: /*?s=
Disallow: /*?replytocom=
Disallow: /*?attachment_id=
Disallow: /*?preview=true
Disallow: /*/embed/
Disallow: /comments/feed/
Disallow: /page/
Allow: /page/1/
Allow: /page/2/
Allow: /page/3/
Allow: /page/4/
Allow: /page/5/
Allow: /page/6/
Allow: /page/7/
Allow: /page/8/
Allow: /page/9/
Allow: /page/10/
Allow: /wp-content/uploads/
Allow: /*.css$
Allow: /*.js$

User-agent: Googlebot-News
Allow: /

User-agent: Googlebot-Image
Allow: /wp-content/uploads/
Disallow: /wp-admin/

User-agent: Googlebot-Video
Allow: /

User-agent: AdsBot-Google
Allow: /

User-agent: Google-InspectionTool
Allow: /

User-agent: Mediapartners-Google
Allow: /


# -----------------------------------------------------------------------------
# BING / COPILOT
# Crawl-delay reduzido: com 32 GB de RAM o servidor aguenta mais.
# -----------------------------------------------------------------------------
User-agent: Bingbot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /?s=
Disallow: /search/
Disallow: /*?replytocom=
Disallow: /*/embed/
Crawl-delay: 1

User-agent: msnbot
Disallow: /wp-admin/
Crawl-delay: 2


# -----------------------------------------------------------------------------
# REDES SOCIAIS E MENSAGEIROS — preview de link
# -----------------------------------------------------------------------------
User-agent: facebookexternalhit
Allow: /

User-agent: meta-externalfetcher
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: Discordbot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: Pinterestbot
Allow: /


# -----------------------------------------------------------------------------
# BOTS DE IA
# (a) busca com citação = tráfego  -> LIBERADO
# (b) fetch por usuário            -> LIBERADO
# (c) treinamento de modelo        -> BLOQUEADO
# -----------------------------------------------------------------------------

User-agent: OAI-SearchBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: MistralAI-Index
Allow: /

User-agent: meta-webindexer
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: Youbot
Disallow: /

# Google-Extended liberado: bloquear não tira do AI Overviews e custaria
# presença no Gemini.
User-agent: Google-Extended
Allow: /

User-agent: Applebot
Allow: /
Disallow: /wp-admin/


# -----------------------------------------------------------------------------
# CRAWLERS DE SEO E SCRAPERS — não trazem visitante
# -----------------------------------------------------------------------------
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: rogerbot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: SeekportBot
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: ZoominfoBot
Disallow: /

User-agent: serpstatbot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: SiteAuditBot
Disallow: /


# -----------------------------------------------------------------------------
# BUSCADORES REGIONAIS DE BAIXO RETORNO
# -----------------------------------------------------------------------------
User-agent: YandexBot
Crawl-delay: 5
Disallow: /wp-admin/
Disallow: /?s=

User-agent: Baiduspider
Crawl-delay: 10
Disallow: /wp-admin/
Disallow: /?s=

User-agent: Sogou
Disallow: /

User-agent: Slurp
Crawl-delay: 3
Disallow: /wp-admin/


# =============================================================================
# SITEMAPS
# =============================================================================
Sitemap: https://www.brasilemfolhas.com.br/sitemap.xml
Sitemap: https://www.brasilemfolhas.com.br/news-sitemap.php