mhsystem.org Review

 # This file provides instructions for well-behaved crawlers.
# Malicious bots are already handled by our .htaccess security layer.

# --- SECTION 1: ALLOWED AI & SEARCH ---
# We allow these so the health system is accurately represented in AI answers.

User-agent: Googlebot
User-agent: GoogleOther
User-agent: bingbot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
Allow: /
# Note: Google & OpenAI ignore Crawl-delay. 
# Manage their speed in Google Search Console / OpenAI dashboard.

User-agent: Applebot
User-agent: Baiduspider
User-agent: Baiduspider-render
Crawl-delay: 5
Allow: /

# --- SECTION 2: DISALLOWED AI TRAINING BOTS ---
# These bots harvest data for models without providing search value.
# Even though they are blocked in .htaccess, we list them here for "polite" compliance.

User-agent: GPTBot
User-agent: ClaudeBot
User-agent: CCBot
User-agent: DeepSeekBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Amazonbot
User-agent: Bytespider
User-agent: TikTokSpider
Disallow: /

# --- SECTION 3: SEO VAMPIRES ---
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: Rogerbot
User-agent: DotBot
User-agent: MJ12bot
User-agent: PetalBot
Disallow: /

# --- SECTION 4: GENERAL WORDPRESS RULES ---
User-agent: *
# A 20-second delay is very high; 5-10 is safer for general indexing.
Crawl-delay: 5
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /readme.html
Disallow: /license.txt

# Sitemap directive
Sitemap: https://www.mhsystem.org/sitemap.xml

HTTP/1.1 301 Moved Permanently
Server: nginx
Date: Sun, 01 Feb 2026 15:37:40 GMT
Content-Type: text/html; charset=UTF-8
Content-Length: 0
Connection: keep-alive
Keep-Alive: timeout=60
X-Redirect-By: WordPress - Really Simple Security
Location: https://mhsystem.org/
Cache-Control: no-cache, no-store, must-revalidate
Expires: 0
Pragma: no-cache

HTTP/1.1 200 OK
Server: nginx
Date: Sun, 01 Feb 2026 15:37:42 GMT
Content-Type: text/html; charset=UTF-8
Content-Length: 45874
Connection: keep-alive
Keep-Alive: timeout=60
Link: ; rel="https://api.w.org/", ; rel="alternate"; title="JSON"; type="application/json", ; rel=shortlink
X-ET-API-VERSION: v1
X-ET-API-ROOT: https://www.mhsystem.org/wp-json/tribe/tickets/v1/
X-ET-API-ORIGIN: https://www.mhsystem.org
X-TEC-API-VERSION: v1
X-TEC-API-ROOT: https://www.mhsystem.org/wp-json/tribe/events/v1/
X-TEC-API-ORIGIN: https://www.mhsystem.org
Cache-Control: no-cache, no-store, must-revalidate
Expires: 0
Vary: Accept-Encoding
Content-Encoding: gzip
Pragma: no-cache