hipstamp.com Review

 User-agent: *
Crawl-delay: 5
#Sitemap: %SITE_PATH%/sitemap.xml

## Do not crawl application folders
Disallow: /config/
Disallow: /library/
Disallow: /module/

## Do not crawl root files
Disallow: /cron.php
Disallow: /install.php
Disallow: /error_log

## Do not crawl pages with sorting or listings per page limit set
Disallow: /*?sort*
Disallow: /*?limit*
#Disallow: /*?parent_id*

## Do not crawl language change pages
Disallow: /*?lang=

## Do not crawl links with session ids
Disallow: /*?SID=

## Do not crawl the admin module
Disallow: /admin/

## Do not crawl the install module
Disallow: /install/

## Do not crawl selected app module pages
## - rss pages
Disallow: /rss/

## - advert related pages
Disallow: /advert/

## - play video action
Disallow: /play-video/

## Do not crawl selected members module pages
Disallow: /members-area/
Disallow: /members/
Disallow: /login-modal/
Disallow: /register-modal/
Disallow: /forgot-username/
Disallow: /forgot-password/

## Do not crawl selected listings module pages
## - bid/purchase/cart related pages
Disallow: /bid/
Disallow: /buy/
Disallow: /offer/
Disallow: /cart/
Disallow: /listings/cart/

## - search pages
Disallow: /search/
Disallow: /advanced-search/

## - sell pages
Disallow: /sell/

## - categories list pages - uncomment to disallow
#Disallow: /categories/

## Uncomment if you do not wish to allow Google and Bing to index your images
# User-agent: Googlebot-Image
# Disallow: /
# User-agent: msnbot-media
# Disallow: /
User-agent: AhrefsBot
Disallow:/
User-agent: YandexBot
Disallow:/

#Disallow: /*.php$

HTTP/1.1 307 Temporary Redirect
Location: https://hipstamp.com/
Date: Mon, 15 Aug 2022 01:31:04 GMT
Transfer-Encoding: chunked
Via: 1.1 google

HTTP/1.1 301 Moved Permanently
Location: https://www.hipstamp.com/
Date: Mon, 15 Aug 2022 01:31:04 GMT
Transfer-Encoding: chunked
Via: 1.1 google
Alt-Svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000

HTTP/1.1 200 OK
Set-Cookie: i18n_redirected=en; Path=/; Expires=Tue, 15 Aug 2023 01:31:04 GMT; SameSite=Lax
Set-Cookie: auth.strategy=hybridJwt; Path=/
ETag: "88a34-cHi3IPhH2PYW5JQI4/mJ4pXyF+c"
Content-Type: text/html; charset=utf-8
Accept-Ranges: none
Vary: Accept-Encoding
Content-Encoding: gzip
Date: Mon, 15 Aug 2022 01:31:04 GMT
Via: 1.1 google
Alt-Svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000
Transfer-Encoding: chunked