labrujadeoro.es Review

 User-agent: *
Disallow: /decimo_virtual/

Crawl-delay: 3600
Visit-time: 0100-0700
Request-rate: 3/5m


#Open list of web crawlers associated with AI companies
#https://github.com/ai-robots-txt/ai.robots.txt/blob/main/robots.txt

User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: aiHitBot
User-agent: Amazonbot
User-agent: anthropic-ai
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Brightbot 1.0
User-agent: Bytespider
User-agent: CCBot
User-agent: ChatGPT-User
User-agent: Claude-Web
User-agent: ClaudeBot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Cotoyogi
User-agent: Crawlspace
User-agent: Diffbot
User-agent: DuckAssistBot
User-agent: DuckDuckBot
User-agent: FacebookBot
User-agent: Factset_spyderbot
User-agent: FirecrawlAgent
User-agent: FriendlyCrawler
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
User-agent: GPTBot
User-agent: iaskspider/2.0
User-agent: ICC-Crawler
User-agent: ImagesiftBot
User-agent: img2dataset
User-agent: imgproxy
User-agent: ISSCyberRiskCrawler
User-agent: Kangaroo Bot
User-agent: meta-externalagent
User-agent: Meta-ExternalAgent
User-agent: meta-externalfetcher
User-agent: Meta-ExternalFetcher
User-agent: NovaAct
User-agent: OAI-SearchBot
User-agent: omgili
User-agent: omgilibot
User-agent: Operator
User-agent: PanguBot
User-agent: Perplexity-User
User-agent: PerplexityBot
User-agent: PetalBot
User-agent: Scrapy
User-agent: SemrushBot-OCOB
User-agent: SemrushBot-SWA
User-agent: Sidetrade indexer bot
User-agent: TikTokSpider
User-agent: Timpibot
User-agent: VelenPublicWebCrawler
User-agent: Webzio-Extended
User-agent: YouBot
Disallow: /

HTTP/1.1 301 Moved Permanently
Date: Mon, 07 Jul 2025 00:18:42 GMT
Content-Type: text/html; charset=ISO-8859-15
Content-Length: 0
Connection: keep-alive
Expires: Thu, 19 Nov 1981 08:52:00 GMT
Cache-Control: no-store, no-cache, must-revalidate, private
Pragma: no-cache
Set-Cookie: GadminID=349793e6d3a4231ebff0a52599131aff; path=/; HttpOnly
Location: https://www.labrujadeoro.es/
Age: 0
Server: HTTPd

HTTP/1.1 200 OK
Date: Mon, 07 Jul 2025 00:18:44 GMT
Content-Type: text/html; charset=ISO-8859-1
Content-Length: 35539
Connection: keep-alive
Expires: Thu, 19 Nov 1981 08:52:00 GMT
Strict-Transport-Security: max-age=10886400; includeSubDomains; preload
Vary: User-Agent,Accept-Encoding
Cache-Control: no-store, no-cache, must-revalidate, max-age=0, post-check=0, pre-check=0, private
Pragma: no-cache
x-frame-options: SAMEORIGIN
X-XSS-Protection: 1; mode=block
X-Content-Type-Options: nosniff
Set-Cookie: GadminID=73718d741853ccd2582f9ea5c1e20622; path=/; HttpOnly
Content-Encoding: gzip
Age: 0
Server: HTTPd
Accept-Ranges: bytes