shsu.am Review

 # Allow all crawlers to index public pages
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Disallow: /wp-json/
Disallow: /xmlrpc.php
Disallow: /?s=
Disallow: /search/
Disallow: /cgi-bin/
Disallow: /trackback/
Disallow: /admin-ajax.php

User-agent: Googlebot
Allow: /

# Allow access to necessary admin-ajax.php for frontend plugins if needed (optional)
# Uncomment only if needed:
# Allow: /wp-admin/admin-ajax.php

# Block specific aggressive bots (example names, extend as needed)
User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

# Sitemap location (if you have one)
Sitemap: https://shsu.am/sitemap.xml

HTTP/1.1 200 OK
Date: Wed, 20 Aug 2025 19:50:17 GMT
Server: Apache
Cache-Control: no-cache
X-Frame-Options: SAMEORIGIN
Link: ; rel="https://api.w.org/"
Vary: Accept-Encoding,User-Agent
Content-Encoding: gzip
Keep-Alive: timeout=5, max=100
Connection: Keep-Alive
Transfer-Encoding: chunked
Content-Type: text/html; charset=UTF-8