sanguosha.com Review

 # ============================================================
# 三国杀官方网站 robots.txt 配置文件
# 适用域名：www.sanguosha.com / sanguosha.com
# 作用：规范通用搜索引擎、AI搜索/训练爬虫的爬取范围
# ============================================================

# ------------------------------
# 一、通用搜索引擎规则（所有默认爬虫）
# 适用：百度、谷歌、必应、360、搜狗、字节等通用搜索引擎爬虫
# ------------------------------
User-agent: *

# 禁止爬取接口与数据路径
Disallow: /api/

# 禁止爬取搜索结果与动态参数页面
Disallow: /news/search/
Disallow: /*?*s=
Disallow: /*?*keyword=

# 禁止爬取内部预览路径
Disallow: /preview/

# 禁止爬取历史兼容与系统接口路径
Disallow: /Home/
Disallow: /home/
Disallow: /healthz
Disallow: /error
Disallow: /404
Disallow: /transit/
Disallow: /version


# ------------------------------
# 二、AI搜索与AI训练爬虫专项规则
# 适用：OpenAI、Google、Anthropic、Perplexity等AI平台爬虫
# 规则：允许爬取公开资讯、武将、玩法等前台内容，禁止爬取所有非公开路径
# ------------------------------
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-Extended
User-agent: Bytespider

# AI 爬虫单独声明 Disallow，避免依赖“继承”造成语义歧义
Disallow: /api/
Disallow: /news/search/
Disallow: /*?*s=
Disallow: /*?*keyword=
Disallow: /preview/
Disallow: /Home/
Disallow: /home/
Disallow: /healthz
Disallow: /error
Disallow: /404
Disallow: /transit/
Disallow: /version
Disallow: /Register
Disallow: /NewRegister/


# ------------------------------
# 三、站点地图
# ------------------------------
Sitemap: https://www.sanguosha.com/sitemap.xml

HTTP/1.1 200 OK
content-encoding: gzip
content-type: text/html
server: istio-envoy
trace-id: f6f607b427e0bad131399c091a08203e
date: Tue, 10 Nov 2026 06:16:49 GMT
vary: Accept-Encoding
x-varnish: 1507822943 149777242
accept-ranges: bytes
x-varnish-cache: hit
req-cost-time: 1
req-arrive-time: 1794502616971
resp-start-time: 1794502616972
x-envoy-upstream-service-time: 0
set-cookie: acw_tc=180113d117945026169717135e2b00fdc840dae7af1c437625c72ac1eb5f28;path=/;HttpOnly;Max-Age=1800
X-Cache-Lookup: Cache Miss
X-Cache-Lookup: Hit From Upstream Cluster
Age: 211207
Content-Length: 24497
X-NWS-LOG-UUID: 1964204667814794749
Connection: keep-alive
X-Cache-Lookup: Cache Miss