schoolies.com Review

 # www.schoolies.com Robots.txt Configuration
# Generated: April 19, 2026
# Purpose: AI Bot and Search Engine Crawler Control

# ======================
# General Web Crawlers
# ======================
User-agent: *
Allow: /
Disallow: /adminlogin/
Disallow: /User/payment/
Disallow: /User/login/
Disallow: /login/
Disallow: /pay/
Disallow: /check-out/
Disallow: /Booking/
Disallow: /bookeasy/
Disallow: /quote/
Disallow: /registrations/
Disallow: /obj/
Disallow: /bin/
Disallow: /App_GlobalResources/
Disallow: /Configs/
Crawl-delay: 1

# ======================
# AI Training Bots
# ======================

# OpenAI GPTBot
User-agent: GPTBot
Allow: /destinations/
Allow: /BuildingList.aspx
Allow: /BuildingDetails.aspx
Allow: /faq/
Allow: /Content.aspx
Allow: /Destinations.aspx
Disallow: /User/
Disallow: /login/
Disallow: /pay/
Disallow: /adminlogin/
Crawl-delay: 2

# Google Extended (Bard/Gemini Training)
User-agent: Google-Extended
Allow: /

# Common Crawl (used by many AI systems)
User-agent: CCBot
Allow: /

# Anthropic AI (Claude)
User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

# Cohere AI
User-agent: cohere-ai
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# ======================
# Social Media Crawlers
# ======================
# Allows link previews on TikTok, Instagram, and Facebook

# ByteDance (TikTok)
User-agent: Bytespider
Allow: /
Disallow: /User/
Disallow: /login/
Disallow: /pay/
Disallow: /adminlogin/

# Facebook/Meta (Instagram & Facebook)
User-agent: FacebookBot
Allow: /
Disallow: /User/
Disallow: /login/
Disallow: /pay/
Disallow: /adminlogin/

# ======================
# Sitemaps
# ======================
Sitemap: https://www.schoolies.com/sitemap.xml
Sitemap: https://www.schoolies.com/sitemap-destinations.xml
Sitemap: https://www.schoolies.com/sitemap-buildings.xml
Sitemap: https://www.schoolies.com/sitemap-content.xml

# ======================
# Crawl Rate Settings
# ======================
# Host: www.schoolies.com
# Crawl-delay: 1 second (reduce server load)

HTTP/1.1 301 Moved Permanently
Date: Sat, 18 Apr 2026 23:24:17 GMT
Content-Type: text/html; charset=UTF-8
Content-Length: 148
Connection: keep-alive
Location: http://www.schoolies.com/
Server: Microsoft-IIS/10.0
X-Powered-By: ASP.NET

HTTP/1.1 301 Moved Permanently
Date: Sat, 18 Apr 2026 23:24:17 GMT
Content-Type: text/html; charset=UTF-8
Content-Length: 149
Connection: keep-alive
Location: https://www.schoolies.com/
Server: Microsoft-IIS/10.0
X-Powered-By: ASP.NET

HTTP/1.1 200 OK
Date: Sat, 18 Apr 2026 23:24:18 GMT
Content-Type: text/html; charset=utf-8
Content-Length: 80325
Connection: keep-alive
Cache-Control: private
Server: Microsoft-IIS/10.0
X-AspNet-Version: 4.0.30319
X-Powered-By: ASP.NET