mxtoolbox.com Review

 # General rules for all crawlers
User-agent: *
# Block access to API endpoints
Disallow: /api/
# Block access to premium or user-specific content
Disallow: /pro/
# Block access to email template resources
Disallow: /public/emailtemplates/
# Block direct access to the email header analysis tool
Disallow: /public/tools/emailheaders.aspx
# Block access to unsubscribe page to avoid unnecessary indexing
Disallow: /public/unsubscribe.aspx
# Block access to checkout pages to avoid indexing dynamic purchase flows
Disallow: /public/checkout/

# Block AhrefsBot from crawling problem diagnostic tools
User-agent: AhrefsBot
Disallow: /problem/

# Block dotbot (used by Dotcom-Monitor) from domain-specific health tools
User-agent: dotbot
Disallow: /domain/
Disallow: /emailhealth/

# Block Meta AI training bots from crawling any content
User-agent: meta-externalagent
Disallow: /SuperTool.aspx

HTTP/1.1 301 Moved Permanently
Server: CloudFront
Date: Sun, 24 Aug 2025 00:50:12 GMT
Content-Type: text/html
Content-Length: 167
Connection: keep-alive
Location: https://mxtoolbox.com/
X-Cache: Redirect from cloudfront
Via: 1.1 81f996ec256b4b15c47b23df66cf2372.cloudfront.net (CloudFront)
X-Amz-Cf-Pop: NRT12-C5
X-Amz-Cf-Id: w_2UeSD1o_e5t5LbQHEMSilIpmLIB7ynwPuG_xMDUXx9giTjiKcQag==

HTTP/1.1 200 OK
Content-Type: text/html; charset=utf-8
Transfer-Encoding: chunked
Connection: keep-alive
Date: Sun, 24 Aug 2025 00:50:13 GMT
X-AspNet-Version: 4.0.30319
Access-Control-Allow-Credentials: true
Cache-Control: public, max-age=300
Expires: Sun, 24 Aug 2025 00:55:13 GMT
Last-Modified: Sun, 24 Aug 2025 00:50:13 GMT
Vary: *
vary: accept-encoding
Server: Microsoft-IIS/10.0
X-Powered-By: UrlRewriter.NET 2.0.0
X-Powered-By: ASP.NET
Access-Control-Allow-Origin: *
content-encoding: gzip
X-Cache: Miss from cloudfront
Via: 1.1 77fa148eb46043c1163d401141317f10.cloudfront.net (CloudFront)
X-Amz-Cf-Pop: HKG62-P1
X-Amz-Cf-Id: oHcE8bn1Te4NPy7CFVOZHglFNT0xeaW7o0JDS5KDEMwTLbxYcXbjig==