mhsystem.org Review

 # This file provides instructions for well-behaved crawlers.
# Malicious bots are already handled by our .htaccess security layer.

# --- SECTION 1: ALLOWED AI & SEARCH ---
User-agent: Googlebot
User-agent: GoogleOther
User-agent: bingbot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
# Block the Time-Traveler loops, iCal exports, and heavy Map views
Disallow: /events/*?tribe-bar-date=
Disallow: /events/*?ical=
Disallow: /events/*?outlook-ical=
Disallow: /events/*/map/
Allow: /

User-agent: Applebot
User-agent: Baiduspider
User-agent: Baiduspider-render
Crawl-delay: 5
Allow: /

# --- SECTION 2: DISALLOWED AI TRAINING BOTS ---
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: CCBot
User-agent: DeepSeekBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Amazonbot
User-agent: Bytespider
User-agent: TikTokSpider
Disallow: /

# --- SECTION 3: SEO VAMPIRES ---
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: Rogerbot
User-agent: DotBot
User-agent: MJ12bot
User-agent: PetalBot
Disallow: /

# --- SECTION 4: GENERAL WORDPRESS RULES ---
User-agent: *
Crawl-delay: 10
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-includes/
Allow: /wp-includes/js/
Allow: /wp-includes/css/
Disallow: /wp-content/plugins/
Disallow: /readme.html
Disallow: /license.txt
# Block the calendar loops for general scrapers too
Disallow: /events/*?tribe-bar-date=
Disallow: /events/*?ical=
Disallow: /events/*/map/

Sitemap: https://www.mhsystem.org/sitemap.xml

HTTP/1.1 301 Moved Permanently
Server: nginx
Date: Mon, 06 Apr 2026 04:00:01 GMT
Content-Type: text/html; charset=iso-8859-1
Content-Length: 273
Connection: keep-alive
Keep-Alive: timeout=60
Location: https://www.mhsystem.org/

HTTP/1.1 200 OK
Server: nginx
Date: Mon, 06 Apr 2026 04:00:03 GMT
Content-Type: text/html; charset=UTF-8
Content-Length: 46439
Connection: keep-alive
Keep-Alive: timeout=60
Link: ; rel="https://api.w.org/", ; rel="alternate"; title="JSON"; type="application/json", ; rel=shortlink
X-ET-API-VERSION: v1
X-ET-API-ROOT: https://www.mhsystem.org/wp-json/tribe/tickets/v1/
X-ET-API-ORIGIN: https://www.mhsystem.org
X-TEC-API-VERSION: v1
X-TEC-API-ROOT: https://www.mhsystem.org/wp-json/tribe/events/v1/
X-TEC-API-ORIGIN: https://www.mhsystem.org
Vary: Accept-Encoding
Content-Encoding: gzip