# ============================================================
# ROBOTS.TXT OPTIMIZADO - URBANNEXT.NET
# Actualizado: 19 Noviembre 2025
# ============================================================

# Reglas generales para todos los bots
User-agent: *
Allow: /

# Bloquear acceso a directorios internos de WordPress
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Disallow: /wp-json/
Disallow: /xmlrpc.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /?s=
Disallow: /*?attachment_id=
Disallow: /tag/
Disallow: /feed/
Disallow: /trackback/
Disallow: /embed/

# Permitir acceso a CSS y JS (importante para SEO)
Allow: /wp-content/uploads/
Allow: /wp-includes/js/
Allow: /wp-includes/css/

# ============================================================
# BOTS AGRESIVOS - CRAWL DELAY Y RESTRICCIONES
# ============================================================

# AhrefsBot - Bot muy agresivo, limitar mucho
User-agent: AhrefsBot
Crawl-delay: 60
Disallow: /wp-content/uploads/videos/
Disallow: /wp-content/uploads/pdf_books/
Disallow: /feed/
Disallow: /tag/
Disallow: /category/

# SemrushBot - Bot de análisis SEO, limitar
User-agent: SemrushBot
Crawl-delay: 60
Disallow: /wp-content/uploads/videos/
Disallow: /wp-content/uploads/pdf_books/
Disallow: /feed/

# DotBot (Moz) - Bot de análisis, limitar
User-agent: DotBot
Crawl-delay: 60
Disallow: /wp-content/uploads/videos/
Disallow: /wp-content/uploads/pdf_books/

# MJ12bot (Majestic) - Bot agresivo
User-agent: MJ12bot
Crawl-delay: 120
Disallow: /wp-content/uploads/

# PetalBot - Bot chino, limitar
User-agent: PetalBot
Crawl-delay: 60
Disallow: /wp-content/uploads/videos/
Disallow: /wp-content/uploads/pdf_books/

# Sogou - Bot chino
User-agent: Sogou web spider
Crawl-delay: 60
Disallow: /wp-content/uploads/

# Bytespider (ByteDance/TikTok)
User-agent: Bytespider
Crawl-delay: 60
Disallow: /wp-content/uploads/videos/
Disallow: /wp-content/uploads/pdf_books/

# TikTokSpider - Bloquear completamente
User-agent: TikTokSpider
Disallow: /

# Baiduspider - Bot chino, permitir pero con delay
User-agent: Baiduspider
Crawl-delay: 30
Disallow: /wp-content/uploads/videos/
Disallow: /wp-content/uploads/pdf_books/

# ============================================================
# BOTS IMPORTANTES - PERMITIR ACCESO COMPLETO
# ============================================================

# Googlebot - IMPORTANTE: acceso completo
User-agent: Googlebot
Allow: /

# Googlebot para imágenes
User-agent: Googlebot-Image
Allow: /wp-content/uploads/

# Googlebot para videos
User-agent: Googlebot-Video
Allow: /wp-content/uploads/videos/

# Bingbot - Permitir acceso completo
User-agent: Bingbot
Allow: /

# ChatGPT / OpenAI - Permitir para indexación AI
User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: GPTBot
Allow: /

# Applebot - Permitir para Siri y búsquedas Apple
User-agent: Applebot
Allow: /

# Amazonbot - Permitir
User-agent: Amazonbot
Crawl-delay: 10
Allow: /

# Meta/Facebook - Permitir para compartir en redes sociales
User-agent: facebookexternalagent
Allow: /

User-agent: meta-externalagent
Allow: /

# Twitter - Permitir para compartir
User-agent: Twitterbot
Allow: /

# Pinterest - Permitir
User-agent: Pinterestbot
Allow: /

# LinkedIn - Permitir
User-agent: LinkedInBot
Allow: /

# ============================================================
# SCRAPERS Y BOTS MALICIOSOS - BLOQUEAR
# ============================================================

# Bloquear scrapers conocidos
User-agent: HTTrack
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: WebZip
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: WebEmailExtract
Disallow: /

# ============================================================
# SITEMAPS
# ============================================================

# Sitemap principal
Sitemap: https://urbannext.net/sitemap_index.xml
Sitemap: https://urbannext.net/sitemap.xml

# ============================================================
# NOTAS DE IMPLEMENTACIÓN
# ============================================================
#
# Crawl-delay valores:
# - 10-30 segundos: Bots buenos pero que queremos ralentizar un poco
# - 60 segundos: Bots agresivos que queremos limitar significativamente
# - 120+ segundos: Bots muy agresivos o poco importantes
#
# Este archivo debe subirse como: /robots.txt
# Verificar después en: https://urbannext.net/robots.txt
#
# Después de subir, verificar en:
# - Google Search Console → Prueba de robots.txt
# - https://www.google.com/webmasters/tools/robots-testing-tool
#
# ============================================================