seo.org Review

 User-agent: *
Crawl-Delay: 60
Disallow: /wp-content/
Disallow: /wp-icludes/
Disallow: /trackback/
Disallow: /wp-admin/
Disallow: /archives/
Disallow: /category/
Disallow: /tag/*
Disallow: /tag/
Disallow: /wp-*
Disallow: /login/
#Disallow: /*.js$
Disallow: /*.inc$
#Disallow: /*.css$
Disallow: /*.php$

# Desindexar carpetas que empiecen por wp-

Disallow: /wp-

# Permitir sitemap pero no las búsquedas.

Sitemap: https://seo.org/sitemap_index.xml
Disallow: /?s=
Disallow: /search

# Permitir Feed general para Google Blogsearch.
# Impedir que /permalink/feed/ sea indexado pues el feed de comentarios suele posicionarse antes de los post.
# Impedir URLs terminadas en /trackback/ que sirven como Trackback URI (contenido duplicado).

Allow: /feed/$
Disallow: /feed
Disallow: /comments/feed
Disallow: /*/feed/$
Disallow: /*/feed/rss/$
Disallow: /*/trackback/$
Disallow: /*/*/feed/$
Disallow: /*/*/feed/rss/$
Disallow: /*/*/trackback/$
Disallow: /*/*/*/feed/$
Disallow: /*/*/*/feed/rss/$
Disallow: /*/*/*/trackback/$

# Evita bloqueos de CSS y JS.

Allow: /*.js$
Allow: /*.css$

# Lista de bots que deberías permitir.

User-agent: All
Allow:/

User-agent: Googlebot-Image
Allow: /wp-content/uploads/

User-agent: Adsbot-Google
Allow: /

User-agent: Googlebot-Mobile
Allow: /

# Lista de bots que generan consultas abusivas aunque siguen las pautas del archivo robots.txt

User-agent: MSIECrawler
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Microsoft.URL.Control
Disallow: /

User-agent: libwww
Disallow: /

User-agent: duggmirror
Disallow: /

User-agent: ia_archiver
Disallow: /

# Slurp (Yahoo!), Noxtrum y el bot de MSN que suelen generar excesivas consultas.

User-agent: noxtrumbot
Crawl-delay: 50

User-agent: msnbot
Crawl-delay: 30

User-agent: Slurp
Crawl-delay: 10

HTTP/1.1 301 Moved Permanently
Date: Fri, 29 Aug 2025 04:48:31 GMT
Server: Apache
Expires: Fri, 29 Aug 2025 05:48:32 GMT
Cache-Control: max-age=3600
X-Redirect-By: WordPress
Upgrade: h2,h2c
Connection: Upgrade, Keep-Alive
Location: https://seo.org/
Vary: Accept-Encoding
Referrer-Policy: no-referrer-when-downgrade
Content-Length: 0
Keep-Alive: timeout=4, max=245
Content-Type: text/html; charset=UTF-8

HTTP/1.1 200 OK
Date: Fri, 29 Aug 2025 04:48:32 GMT
Server: Apache
Upgrade: h2
Connection: Upgrade, Keep-Alive
Last-Modified: Thu, 28 Aug 2025 23:22:27 GMT
Cache-Control: max-age=0
Expires: Fri, 29 Aug 2025 04:48:32 GMT
Vary: Accept-Encoding
Content-Encoding: gzip
Referrer-Policy: no-referrer-when-downgrade
Content-Length: 37344
Keep-Alive: timeout=4, max=245
Content-Type: text/html; charset=UTF-8