arxiv.org Review

TitletagDescriptiontaglanguage
arXiv.org e-Print archive English
Alexarank
563
Ip adress151.101.131.42Nameserverns-cloud-d1.googledomains.com
ns-cloud-d4.googledomains.com
ns-cloud-d2.googledomains.com
ns-cloud-d3.googledomains.com
Status code200
robots.txt -http://arxiv.org/robots.txt
 # robots.txt for http://arxiv.org/ and mirror sites http://*.arxiv.org/
# Indiscriminate automated downloads from this site are not permitted
# See also: http://arxiv.org/help/robots

User-agent: *
Crawl-delay: 15
Allow: /archive
Allow: /year
Allow: /list
Allow: /abs
Allow: /pdf
Allow: /html
Allow: /catchup
Disallow: /user
Disallow: /e-print
Disallow: /src
Disallow: /ps
Disallow: /dvi
Disallow: /cookies
Disallow: /form
Disallow: /find
Disallow: /view
Disallow: /ftp
Disallow: /refs
Disallow: /cits
Disallow: /format
Disallow: /PS_cache
Disallow: /Stats
Disallow: /seek-and-destroy
Disallow: /IgnoreMe
Disallow: /oai2
Disallow: /auth
Disallow: /tb
Disallow: /tb-recent
Disallow: /trackback
Disallow: /prevnext
Disallow: /ct
Disallow: /api
Disallow: /search
Disallow: /set_author_id
Disallow: /show-email

User-agent: Googlebot
Allow: /archive
Allow: /year
Allow: /list
Allow: /abs
Allow: /pdf
Allow: /html
Allow: /catchup
Disallow: /user
Disallow: /e-print
Disallow: /src
Disallow: /ps
Disallow: /dvi
Disallow: /cookies
Disallow: /form
Disallow: /find
Disallow: /view
Disallow: /ftp
Disallow: /refs
Disallow: /cits
Disallow: /format
Disallow: /PS_cache
Disallow: /Stats
Disallow: /seek-and-destroy
Disallow: /IgnoreMe
Disallow: /oai2
Disallow: /auth
Disallow: /tb
Disallow: /tb-recent
Disallow: /trackback
Disallow: /prevnext
Disallow: /ct
Disallow: /api
Disallow: /search

User-agent: Yahoo! Slurp
Crawl-delay: 1
Allow: /archive
Allow: /year
Allow: /list
Allow: /abs
Allow: /pdf
Allow: /html
Allow: /catchup
Disallow: /user
Disallow: /e-print
Disallow: /src
Disallow: /ps
Disallow: /dvi
Disallow: /cookies
Disallow: /form
Disallow: /find
Disallow: /view
Disallow: /ftp
Disallow: /refs
Disallow: /cits
Disallow: /format
Disallow: /PS_cache
Disallow: /Stats
Disallow: /seek-and-destroy
Disallow: /IgnoreMe
Disallow: /oai2
Disallow: /auth
Disallow: /tb
Disallow: /tb-recent
Disallow: /trackback
Disallow: /prevnext
Disallow: /ct
Disallow: /api
Disallow: /search

User-agent: bingbot
# 2021-10-14 - removed crawl-delay for Bingbot. Needs to be re-added if there are any problems.
# 2021-10-26 - added back
Crawl-delay: 1
Allow: /archive
Allow: /year
Allow: /list
Allow: /abs
Allow: /pdf
Allow: /html
Allow: /catchup
Disallow: /user
Disallow: /e-print
Disallow: /src
Disallow: /ps
Disallow: /dvi
Disallow: /cookies
Disallow: /form
Disallow: /find
Disallow: /view
Disallow: /ftp
Disallow: /refs
Disallow: /cits
Disallow: /format
Disallow: /PS_cache
Disallow: /Stats
Disallow: /seek-and-destroy
Disallow: /IgnoreMe
Disallow: /oai2
Disallow: /auth
Disallow: /tb
Disallow: /tb-recent
Disallow: /trackback
Disallow: /prevnext
Disallow: /ct
Disallow: /api
Disallow: /search

User-agent: Baiduspider
Crawl-delay: 10
Allow: /archive
Allow: /year
Allow: /list
Allow: /abs
Allow: /pdf
Allow: /html
Allow: /catchup
Disallow: /user
Disallow: /e-print
Disallow: /src
Disallow: /ps
Disallow: /dvi
Disallow: /cookies
Disallow: /form
Disallow: /find
Disallow: /view
Disallow: /ftp
Disallow: /refs
Disallow: /cits
Disallow: /format
Disallow: /PS_cache
Disallow: /Stats
Disallow: /seek-and-destroy
Disallow: /IgnoreMe
Disallow: /oai2
Disallow: /auth
Disallow: /tb
Disallow: /tb-recent
Disallow: /trackback
Disallow: /prevnext
Disallow: /ct
Disallow: /api
Disallow: /search

User-agent: ToutiaoSpider
Crawl-delay: 10
Allow: /archive
Allow: /year
Allow: /list
Allow: /abs
Allow: /pdf
Allow: /html
Allow: /catchup
Disallow: /user
Disallow: /e-print
Disallow: /src
Disallow: /ps
Disallow: /dvi
Disallow: /cookies
Disallow: /form
Disallow: /find
Disallow: /view
Disallow: /ftp
Disallow: /refs
Disallow: /cits
Disallow: /format
Disallow: /PS_cache
Disallow: /Stats
Disallow: /seek-and-destroy
Disallow: /IgnoreMe
Disallow: /oai2
Disallow: /auth
Disallow: /tb
Disallow: /tb-recent
Disallow: /trackback
Disallow: /prevnext
Disallow: /ct
Disallow: /api
Disallow: /search

User-agent: SQUID_configured_as_described_at_/help/faq/cache
Crawl-delay: 10
Allow: /list
Allow: /abs
Allow: /pdf
Disallow: /archive
Disallow: /year
Disallow: /html
Disallow: /catchup
Disallow: /user
Disallow: /e-print
Disallow: /src
Disallow: /ps
Disallow: /dvi
Disallow: /cookies
Disallow: /form
Disallow: /find
Disallow: /view
Disallow: /ftp
Disallow: /refs
Disallow: /cits
Disallow: /format
Disallow: /PS_cache
Disallow: /Stats
Disallow: /seek-and-destroy
Disallow: /IgnoreMe
Disallow: /oai2
Disallow: /auth
Disallow: /tb
Disallow: /tb-recent
Disallow: /trackback
Disallow: /prevnext
Disallow: /ct
Disallow: /api
Disallow: /search

User-agent: YandexBot
Crawl-delay: 1
Allow: /archive
Allow: /year
Allow: /list
Allow: /abs
Allow: /pdf
Allow: /html
Allow: /catchup
Disallow: /e-print/
Disallow: /src/
Disallow: /ps/
Disallow: /psfigs/
Disallow: /dvi/
Disallow: /cookies/
Disallow: /form/
Disallow: /find/
Disallow: /view/
Disallow: /ftp/
Disallow: /refs/
Disallow: /cits/
Disallow: /format/
Disallow: /register
Disallow: /submit
Disallow: /replace
Disallow: /cross
Disallow: /jref
Disallow: /paper_passwd/
Disallow: /PS_cache/
Disallow: /Stats/
Disallow: /seek-and-destroy
Disallow: /IgnoreMe
Disallow: /uploads
Disallow: /oai2
Disallow: /auth
Disallow: /tb
Disallow: /tb-recent
Disallow: /trackback
Disallow: /prevnext
Disallow: /ct
Disallow: /api
Disallow: /search

User-agent: Applebot
Crawl-delay: 1
Allow: /archive
Allow: /year
Allow: /list
Allow: /abs
Allow: /pdf
Allow: /html
Allow: /catchup
Disallow: /user
Disallow: /e-print
Disallow: /src
Disallow: /ps
Disallow: /dvi
Disallow: /cookies
Disallow: /form
Disallow: /find
Disallow: /view
Disallow: /ftp
Disallow: /refs
Disallow: /cits
Disallow: /format
Disallow: /PS_cache
Disallow: /Stats
Disallow: /seek-and-destroy
Disallow: /IgnoreMe
Disallow: /oai2
Disallow: /auth
Disallow: /tb
Disallow: /tb/recent
Disallow: /tb-recent
Disallow: /trackback
Disallow: /prevnext
Disallow: /ct
Disallow: /api
Disallow: /search

User-agent: SemrushBot
Disallow: /
HTTP/1.1 200 OK
Connection: keep-alive
Content-Length: 44300
cache-control: max-age: 3600
content-type: text/html; charset=utf-8
content-security-policy: frame-ancestors 'none'
x-frame-options: SAMEORIGIN
x-cloud-trace-context: 0f191752646840c98b877332a82a0ca3
server: Google Frontend
via: 1.1 google, 1.1 google, 1.1 varnish, 1.1 varnish
Accept-Ranges: bytes
Date: Sat, 25 May 2024 17:16:42 GMT
Age: 18
X-Served-By: cache-lga13629-LGA, cache-tyo11930-TYO
X-Cache: HIT, HIT
X-Timer: S1716657403.756771,VS0,VE1

iframe