dfwurbanrealty.com Review

 # Block Baidu
# http://help.baidu.com/question?prod_en=master&class=Baiduspider
User-agent: Baiduspider
Disallow: /

# Block Yandex
# https://yandex.com/support/webmaster/controlling-robot/robots-txt.xml
User-agent: Yandex
Disallow: /

# Block Redbot
# https://redbot.org/
User-agent: RED
Disallow: /

# Block Dotbot
# https://wowrack.org/
User-agent: dotbot
Disallow: /

# Block Semrush
# https://www.semrush.com/bot/
User-agent: SemrushBot
Disallow: /

# Block TrovitBot
# https://www.trovit.com/bot.html
User-agent: trovitBot
Disallow: /

# Block RogerBot
# https://moz.com/help/moz-procedures/crawlers/rogerbot
User-agent: rogerbot
Disallow: /

# Block SeekPort
# http://seekport.com/
User-agent: Seekport
Disallow: /

# Generic bot rules
User-agent: *
Crawl-delay: 5
Disallow: /backend
Disallow: /blog/*/share.html
Disallow: /blog/*/print.html
Disallow: /idx/inc/php/ajax/
Disallow: /idx/?refine
Disallow: /*/idx/?refine
Disallow: /idx/search/
Disallow: /*/idx/search/
Disallow: /idx/search.html
Disallow: /*/idx/search.html
Disallow: /idx/*/print.html
Disallow: /idx/*/map.html
Disallow: /idx/*/friend.html
Disallow: /idx/*/inquire.html
Disallow: /listing/*/inquire/
Disallow: /listing/*/friend/
Disallow: /listing/*/brochure/
Disallow: /listing/*/brochure
Disallow: /listing/*/mortgage/

HTTP/1.1 301 Moved Permanently
Date: Wed, 17 Aug 2022 07:09:30 GMT
Accept-Ranges: bytes
Content-Encoding: gzip
Transfer-Encoding: chunked
Content-Type: text/html; charset=iso-8859-1
Server: Apache
Location: http://www.dfwurbanrealty.com/
Cache-Control: max-age=0
Via: 1.1 google
Set-Cookie: GCLB=CIqO4M2Bxu-V8QE; path=/; HttpOnly; expires=Wed, 17-Aug-2022 07:19:30 GMT
X-HW: 1660720170.cds256.tk2.h2,1660720170.cds222.tk2.sc,1660720170.cds222.tk2.p
Access-Control-Allow-Origin: *
Connection: keep-alive

HTTP/1.1 301 Moved Permanently
Date: Wed, 17 Aug 2022 07:09:31 GMT
Cache-Control: no-store, no-cache, must-revalidate, post-check=0, pre-check=0
Content-Type: text/html; charset=UTF-8
Server: Apache
Expires: Thu, 19 Nov 1981 08:52:00 GMT
Pragma: no-cache
Set-Cookie: PHPSESSID=fffbd8fef55fa207475f57ac444a97bf; path=/
Location: https://www.dfwurbanrealty.com/
X-Content-Type-Options: nosniff
Content-Language: en
Via: 1.1 google
Set-Cookie: GCLB=CMn-sJzj1ICToAE; path=/; HttpOnly; expires=Wed, 17-Aug-2022 07:19:30 GMT
X-HW: 1660720170.cds219.tk2.h2,1660720170.cds222.tk2.sc,1660720171.cds222.tk2.p
Access-Control-Allow-Origin: *
Connection: keep-alive
Content-Length: 0

HTTP/1.1 200 OK
Date: Wed, 17 Aug 2022 07:09:32 GMT
Cache-Control: no-store, no-cache, must-revalidate, post-check=0, pre-check=0
Content-Encoding: gzip
Transfer-Encoding: chunked
Content-Type: text/html; charset=UTF-8
Server: Apache
Expires: Thu, 19 Nov 1981 08:52:00 GMT
Pragma: no-cache
Content-Language: en,en-us
Set-Cookie: PHPSESSID=f64b9b75eeef04fd886e69a9ed8dec87; path=/
Vary: Accept-Encoding
X-Content-Type-Options: nosniff
Via: 1.1 google
Set-Cookie: GCLB=CLG2jNSgmr7FKg; path=/; HttpOnly; expires=Wed, 17-Aug-2022 07:19:31 GMT
Alt-Svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000
X-HW: 1660720171.cds239.tk2.hn,1660720171.cds204.tk2.sc,1660720172.cds204.tk2.p
Access-Control-Allow-Origin: *
Connection: keep-alive