buildersmart.in Review

 ## GENERAL SETTINGS
 
User-agent: archive.org_bot
Disallow: /
User-agent: ia_archiver
Disallow: /
User-agent: rogerbot
Disallow: /



## Enable robots.txt rules for all crawlers
User-agent: *

 
## Crawl-delay parameter: number of seconds to wait between successive requests to the same server.
## Set a custom crawl rate if you're experiencing traffic problems with your server.
Crawl-delay: 30
 
## Magento sitemap: uncomment and replace the URL to your Magento sitemap file
Sitemap: https://www.buildersmart.in/sitemap.xml
 
## DEVELOPMENT RELATED SETTINGS
 
## Do not crawl development files and folders: CVS, svn directories and dump files
Disallow: /CVS
Disallow: /*.svn$
Disallow: /*.idea$
Disallow: /*.sql$
Disallow: /*.tgz$
 
## GENERAL MAGENTO SETTINGS
 
## Do not crawl Magento admin page
Disallow: /bmpanel/
 
## Do not crawl common Magento technical folders
Disallow: /app/
Disallow: /downloader/
Disallow: /errors/
Disallow: /includes/
Disallow: /lib/
Disallow: /pkginfo/
Disallow: /shell/
Disallow: /var/
Disallow: /api/
Disallow: /CRMImages/
Disallow: /custom/
Disallow: /STORES/
Disallow: /staging/
Disallow: /tmp/
Disallow: /test/
Disallow: /powerbi/
Disallow: /Mobile-Detect/
Disallow: /media/captcha/
Disallow: /media/customer/
Disallow: /media/dhl/
Disallow: /media/downloadable/
Disallow: /media/import/
Disallow: /media/pdf/
Disallow: /media/sales/
Disallow: /media/tmp/
 
## Do not crawl common Magento files
Disallow: /api.php
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /error_log
Disallow: /get.php
Disallow: /install.php
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /README.txt
Disallow: /RELEASE_NOTES.txt
 
## MAGENTO SEO IMPROVEMENTS
 
## Do not crawl sub category pages that are sorted or filtered.
Disallow: /*?dir*
Disallow: /*?dir=desc
Disallow: /*?dir=asc
Disallow: /*?limit=all
Disallow: /*?mode*
Disallow: /*category=*
Disallow: /*aw_shopbybrand_brand=*
Disallow: /*price=*
Disallow: /*order=*
 
## Do not crawl 2-nd home page copy (example.com/index.php/). Uncomment it only if you activated Magento SEO URLs.
Disallow: /index.php/
 
## Do not crawl links with session IDs
Disallow: /*?SID=
 
## Do not crawl checkout and user account pages
Disallow: /checkout/
Disallow: /onestepcheckout/
Disallow: /customer/
Disallow: /customer/account/
Disallow: /customer/account/login/
 
## Do not crawl seach pages and not-SEO optimized catalog links
Disallow: /catalogsearch/
#Disallow: /search?*
#Disallow: /search/
Disallow: /*limit=*
Disallow: /catalog/product_compare/
Disallow: /catalog/category/view/
#Disallow: /catalog/product/view/

## Video module fixes
Disallow: /videos/search/*

## CUSTOM ADDED BY GRAHAM
Disallow: /store/index.php/

## SERVER SETTINGS
 
## Do not crawl common server technical folders and files
Disallow: /cgi-bin/
Disallow: /cleanup.php
Disallow: /apc.php
Disallow: /memcache.php
Disallow: /phpinfo.php
Disallow: /picture/view/*
Disallow: /form_key*
#Disallow: */google-code-prettify*

HTTP/1.1 302 Found
Date: Tue, 26 Aug 2025 00:14:46 GMT
Content-Type: text/html; charset=UTF-8
Transfer-Encoding: chunked
Connection: keep-alive
Server: cloudflare
Location: https://www.buildersmart.in/
Cache-Control: max-age=2592000
Expires: Thu, 25 Sep 2025 00:14:46 GMT
Vary: User-Agent
X-Content-Type-Options: nosniff
X-Xss-Protection: 1; mode=block
Access-Control-Allow-Origin: *
Access-Control-Allow-Methods: GET,PUT,POST,DELETE
Access-Control-Allow-Headers: Content-Type, Authorization
Nel: {"report_to":"cf-nel","success_fraction":0.0,"max_age":604800}
Report-To: {"group":"cf-nel","max_age":604800,"endpoints":[{"url":"https://a.nel.cloudflare.com/report/v4?s=YJqsQx4b1czjmh07TuyInEibNpNah3SWLjzGJFDCkTCyNLTpgHgz3v4Qst2vbkBnrpHRG%2B92k2K4%2FPZusAtdTmtHdvMQKx9Ihoe46YI%3D"}]}
Cf-Cache-Status: DYNAMIC
Speculation-Rules: "/cdn-cgi/speculation"
Server-Timing: cfCacheStatus;desc="DYNAMIC"
Server-Timing: cfEdge;dur=30,cfOrigin;dur=410
CF-RAY: 974f21816bd0fcbe-NRT
alt-svc: h3=":443"; ma=86400

HTTP/1.1 200 OK
Date: Tue, 26 Aug 2025 00:14:48 GMT
Content-Type: text/html; charset=UTF-8
Transfer-Encoding: chunked
Connection: keep-alive
Server: cloudflare
Expires: Thu, 19 Nov 1981 08:52:00 GMT
Cache-Control: no-store, no-cache, must-revalidate
Pragma: no-cache
X-Frame-Options: SAMEORIGIN
Vary: Accept-Encoding,User-Agent
Server-Timing: cfCacheStatus;desc="DYNAMIC"
Server-Timing: cfEdge;dur=14,cfOrigin;dur=2402
X-Content-Type-Options: nosniff
X-Xss-Protection: 1; mode=block
Access-Control-Allow-Origin: *
Access-Control-Allow-Methods: GET,PUT,POST,DELETE
Access-Control-Allow-Headers: Content-Type, Authorization
Nel: {"report_to":"cf-nel","success_fraction":0.0,"max_age":604800}
Report-To: {"group":"cf-nel","max_age":604800,"endpoints":[{"url":"https://a.nel.cloudflare.com/report/v4?s=GwHIpYUdpuoXhfJEBaAhwjIADV1%2BTkwryb0nYF1xQrF9NpOi7gY%2FJIul1%2FPn0bsvwMz4FB4R8cxjNyNmp%2Bog%2BseicUpwhHxKV20bpnwvz5bC638%3D"}]}
Cf-Cache-Status: DYNAMIC
Speculation-Rules: "/cdn-cgi/speculation"
Content-Encoding: br
Set-Cookie: frontend=n1e3j9eg8r4ksl7nih87vuiv63; HttpOnly; Path=/; Domain=www.buildersmart.in; Max-Age=86400; Expires=Wed, 27 Aug 2025 00:14:46 GMT
Set-Cookie: frontend_cid=8CIRPfpkAIbjAaNq; HttpOnly; Secure; Path=/; Domain=www.buildersmart.in; Max-Age=86400; Expires=Wed, 27 Aug 2025 00:14:46 GMT
CF-RAY: 974f21846a258cac-NRT
alt-svc: h3=":443"; ma=86400