myrobotdoll.com Review

 User-Agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

Disallow: /wp-content/uploads/wc-logs/
Disallow: /wp-content/uploads/woocommerce_transient_files/
Disallow: /wp-content/uploads/woocommerce_uploads/
Allow: /wp-content/uploads/

# WooCommerce transactional URLs
Disallow: /cart
Disallow: /checkout
Disallow: /my-account
Disallow: /*add-to-cart=*
Disallow: /*remove_item=*
Disallow: /*wc-ajax=*
Disallow: /*?wc-ajax=*

# Internal search and filter crawl traps
Disallow: /search
Disallow: /search*
Disallow: /*?s=*
Disallow: /*&s=*
Disallow: /*_=pf*
Disallow: /*pf_t_*
Disallow: /*filtering=1*
Disallow: /*filter_product_brand=*
Disallow: /*query-0-page=*
Disallow: /*mrd_r=*

# Tracking / duplicate URL parameters
Disallow: /*srsltid=*
Disallow: /*gclid=*
Disallow: /*fbclid=*
Disallow: /*utm_*

# Explicitly allow selected AI/LLM crawlers
User-Agent: CCBot
Allow: /

User-Agent: GPTBot
Allow: /

User-Agent: ClaudeBot
Allow: /

User-Agent: Claude-Web
Allow: /

User-Agent: Google-Extended
Allow: /

User-Agent: PerplexityBot
Allow: /

User-Agent: facebookexternalhit
Allow: /

User-Agent: YouBot
Allow: /

User-Agent: Applebot-Extended
Allow: /

# Do not allow aggressive non-essential crawlers
User-Agent: Bytespider
Disallow: /

User-Agent: SemrushBot
Disallow: /

User-Agent: PetalBot
Disallow: /

User-Agent: IbouBot
Disallow: /

User-Agent: Amazonbot
Disallow: /

Sitemap: https://www.myrobotdoll.com/sitemap_index.xml

HTTP/1.1 301 Moved Permanently
Date: Mon, 07 Sep 2026 03:19:22 GMT
Content-Type: text/html
Transfer-Encoding: chunked
Connection: keep-alive
Location: https://www.myrobotdoll.com/
CF-Ray: a372924c5c767958-NRT
CF-Cache-Status: EXPIRED
Server: cloudflare
Vary: accept-encoding
Cf-Apo-Via: origin,resnok
Speculation-Rules: "/cdn-cgi/speculation"
Server-Timing: cfCacheStatus;desc="EXPIRED"
Server-Timing: cfEdge;dur=557,cfOrigin;dur=0
Report-To: {"group":"cf-nel","max_age":604800,"endpoints":[{"url":"https://a.nel.cloudflare.com/report/v4?s=ao10TnLX88M6uO8qKpW12PBNa7%2F4QWqPI27buOcZriRnfYfcgXMSzBsCk%2FGSjjOgAL06EM%2Fu%2BlLBthBNP%2BCrT6A6a%2BwekfU%2BXZznyAlw7QkEr2grYOUFgtvKT47qjBw50Q%3D%3D"}]}
Nel: {"report_to":"cf-nel","success_fraction":0.0,"max_age":604800}
alt-svc: h3=":443"; ma=86400

HTTP/1.1 200 OK
Date: Mon, 07 Sep 2026 03:19:22 GMT
Content-Type: text/html; charset=UTF-8
Transfer-Encoding: chunked
Connection: keep-alive
CF-Ray: a37292502b0bcc19-NRT
CF-Cache-Status: HIT
Age: 36482
Last-Modified: Sun, 06 Sep 2026 21:24:23 GMT
Link: ; rel="https://api.w.org/", ; rel="alternate"; title="JSON"; type="application/json", ; rel=shortlink
Server: cloudflare
Strict-Transport-Security: max-age=31536000; includeSubDomains
Vary: Accept-Encoding
Cf-Apo-Via: tcache
cf-edge-cache: cache,platform=wordpress
referrer-policy: strict-origin-when-cross-origin
x-content-type-options: nosniff
Speculation-Rules: "/cdn-cgi/speculation"
Server-Timing: cfCacheStatus;desc="HIT"
Server-Timing: cfEdge;dur=29,cfOrigin;dur=0
Report-To: {"group":"cf-nel","max_age":604800,"endpoints":[{"url":"https://a.nel.cloudflare.com/report/v4?s=56DYIOttK55Q73um%2FEwZKabyOnBr0SKH19HOQZ0c3pbx1XqcQBuy3K1LcMH9hyKx6yj3lVeGtbJ5ahkj%2BakB68sJa1NRnP13p5Nlub1PHc7ImmblWqxLh%2BzIiAtaInCmugF2DlU%3D"}]}
Nel: {"report_to":"cf-nel","success_fraction":0.0,"max_age":604800}
Content-Encoding: br
alt-svc: h3=":443"; ma=86400