lesterbanks.com Review

 # Grant Robots Access
#######################################################################################
 
# This example allows all robots to visit all files because the wildcard "*" specifies all robots:
User-agent: *
Allow:
 
#To allow a single robot you would use the following:
#User-agent: Google
#Disallow:
 
#User-agent: *
#Disallow: /
 
# Deny Robots Access
#######################################################################################
 
# This example keeps all robots out:
#User-agent: *
#Disallow: /
 
# The next is an example that tells all crawlers not to enter into four directories of a website:
User-agent: *
Disallow: /cgi-bin/
Disallow: /images/
Disallow: /tmp/
Disallow: /private/

# Example that tells a specific crawler not to enter one specific directory:
#User-agent: BadBot
#Disallow: /private/

# Example that tells all crawlers not to enter one specific file called foo.html
#User-agent: *
#Disallow: /domains/example.com/html/

HTTP/1.1 200 OK
Date: Tue, 16 Aug 2022 18:45:21 GMT
Content-Type: text/html; charset=UTF-8
Transfer-Encoding: chunked
Connection: keep-alive
Vary: Accept-Encoding,User-Agent
Last-Modified: Mon, 15 Aug 2022 18:16:58 GMT
Cache-Control: max-age=0
Expires: Tue, 16 Aug 2022 18:45:21 GMT
CF-Cache-Status: DYNAMIC
Report-To: {"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v3?s=1rXODEmt2Ghoe2GhfObEVM6sbLgVu2t59xDDP%2BO3iVzVpJXk8TKAh5y%2FfQthsNNNFoSPu01msLKak8T5cTE0q1d5laT%2FvgAaqQbensvxDUIVonKMLCwdivzU7vbblmnLx0U%3D"}],"group":"cf-nel","max_age":604800}
NEL: {"success_fraction":0,"report_to":"cf-nel","max_age":604800}
Server: cloudflare
CF-RAY: 73bc5397694b1eab-NRT
Content-Encoding: gzip
alt-svc: h3=":443"; ma=86400, h3-29=":443"; ma=86400