smiththompson.com Review

 User-agent: *
Disallow:

# OpenAI (ChatGPT, GPTBot)
User-agent: GPTBot
Disallow:

User-agent: ChatGPT-User
Disallow:

# Anthropic (Claude)
User-agent: ClaudeBot
Disallow:

# Google AI (Gemini, Bard)
User-agent: Google-Extended
Disallow:

# Perplexity AI
User-agent: PerplexityBot
Disallow:

# xAI / Grok (Twitter/X) — emerging agents
User-agent: xAIBot
Disallow:

User-agent: GrokBot
Disallow:

User-agent: Twitterbot
Disallow:

# Common Crawl (used by many LLMs)
User-agent: CCBot
Disallow:

# Amazon (used in AI/ML pipelines)
User-agent: Amazonbot
Disallow:

# Facebook / Meta AI
User-agent: facebookexternalhit
Disallow:

User-agent: Facebot
Disallow:

# DuckDuckGo AI
User-agent: DuckDuckBot
Disallow:

# Neeva (now part of Snowflake)
User-agent: NeevaBot
Disallow:

# You.com
User-agent: YouBot
Disallow:

# Andi AI
User-agent: andi-bot
Disallow:

# Phind (dev AI search engine)
User-agent: PhindBot
Disallow:

# Bing / Microsoft AI
User-agent: bingbot
Disallow:

User-agent: BingPreview
Disallow:

# Yandex AI / General crawler
User-agent: YandexBot
Disallow:

# Apple AI systems (Siri, Spotlight)
User-agent: Applebot
Disallow:

# AhrefsBot (used in AI/data SEO)
User-agent: AhrefsBot
Disallow:

Sitemap: https://smiththompson.com/sitemap_index.xml

HTTP/1.1 301 Moved Permanently
Server: nginx
Date: Tue, 26 Aug 2025 02:28:56 GMT
Content-Type: text/html
Content-Length: 162
Connection: keep-alive
Keep-Alive: timeout=20
Location: https://smiththompson.com/

HTTP/1.1 200 OK
Server: nginx
Date: Tue, 26 Aug 2025 02:28:57 GMT
Content-Type: text/html; charset=UTF-8
Transfer-Encoding: chunked
Connection: keep-alive
Keep-Alive: timeout=20
Vary: Accept-Encoding
Vary: Accept-Encoding
Vary: Accept-Encoding
Set-Cookie: STYXKEY_HTTP_REFERER=deleted; expires=Thu, 01-Jan-1970 00:00:01 GMT; Max-Age=0; path=/; secure; HttpOnly
Link: ; rel="https://api.w.org/"
Link: ; rel="alternate"; title="JSON"; type="application/json"
Link: ; rel=shortlink
X-Powered-By: WP Engine
X-Cacheable: SHORT
Vary: Accept-Encoding,Cookie
Cache-Control: max-age=600, must-revalidate
X-Cache: HIT: 13
X-Cache-Group: normal
Content-Encoding: br