airnav.com Review

 Disallow: /cgi-bin/

# Teleport Pro is too dumb, rude and agressive
User-agent: Teleport Pro
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: CCBot
Crawl-delay: 120

User-agent: msnbot
Crawl-delay: 120

User-agent: Slurp
Crawl-delay: 120

User-agent: ScoutJet
Crawl-delay: 60

User-agent: Blekkobot
Crawl-delay: 60

User-agent: Yandex
Crawl-delay: 120

User-agent: HyperRobot
Disallow: /

User-agent: 008
Disallow: /

User-agent: Teoma
Crawl-delay: 120

# Voila bot is too agressive, and there is no real benefit in being listed in voila.com
User-agent: VoilaBot
Disallow: /

# iltrovatore bot is too agressive, and there is no real benefit in being listed in iltrovatore
User-agent: iltrovatore-Setaccio
Disallow: /

User-agent: diamondbot
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Baiduspider
User-agent: Baiduspider-video
User-agent: Baiduspider-image
Disallow: /

User-agent: sogou spider
Disallow: /

User-agent: YoudaoBot
Disallow: /

User-agent: EasouSpider
Disallow: /

User-agent: AhrefsBot
disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: proximic
Disallow: /

User-agent: *
Disallow: /iws/

User-agent: anthropic-ai
#Disallow: /
Crawl-delay: 240

User-agent: ClaudeBot
#Disallow: /
Crawl-delay: 240

User-agent: Applebot
Crawl-delay: 120

User-agent: AwarioBot
Crawl-delay: 120

User-agent: AwarioRssBot
User-agent: AwarioSmartBot
Crawl-delay: 120

HTTP/1.1 200 OK
Date: Sat, 23 Aug 2025 05:50:30 GMT
Server: Apache/1.3.37 (Unix) mod_perl/1.29 mod_ssl/2.8.28 OpenSSL/0.9.7e-p1
Content-Type: text/html
Set-Cookie: SESSION_ID=a2d9522551c86c880cdf4de9dca2e243; domain=.airnav.com; path=/
Keep-Alive: timeout=5, max=100
Connection: Keep-Alive
Transfer-Encoding: chunked