projectools.io Review

 # WordPress robots.txt - Control bot crawling with delays

# Google Ads bots - allow landing pages but control crawl speed
User-agent: AdsBot-Google
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Disallow: /wp-content/uploads/
Disallow: /?s=
Disallow: /*?s=
Disallow: /search/
Disallow: /xmlrpc.php
Disallow: /trackback/
Disallow: /feed/
Disallow: /comments/
Disallow: /author/
Disallow: /wp-json/
Disallow: /tag/
Disallow: /category/
Crawl-delay: 60

User-agent: AdsBot-Google-Mobile
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Disallow: /wp-content/uploads/
Disallow: /?s=
Disallow: /*?s=
Disallow: /search/
Disallow: /xmlrpc.php
Disallow: /trackback/
Disallow: /feed/
Disallow: /comments/
Disallow: /author/
Disallow: /wp-json/
Disallow: /tag/
Disallow: /category/
Crawl-delay: 60

User-agent: AdsBot-Google-Mobile-Apps
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Disallow: /wp-content/uploads/
Crawl-delay: 60

# Allow Googlebot-Image but limit to essential paths
User-agent: Googlebot-Image
Allow: /wp-content/uploads/
Disallow: /
Crawl-delay: 10

# Other Google bots - more restrictive
User-agent: GoogleOther
Disallow: /
Crawl-delay: 86400

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Site-Verification
Allow: /

# Bing/Microsoft bots with controlled crawling
User-agent: bingbot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Disallow: /?s=
Disallow: /*?s=
Disallow: /search/
Disallow: /xmlrpc.php
Disallow: /trackback/
Disallow: /feed/
Disallow: /comments/
Disallow: /author/
Disallow: /wp-json/
Crawl-delay: 10

User-agent: adidxbot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Disallow: /wp-content/uploads/
Crawl-delay: 10

# Block resource-intensive SEO bots completely
User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: YandexBot
Disallow: /

# Social media crawlers - allow but control
User-agent: facebookexternalhit
Allow: /
Crawl-delay: 10

User-agent: Twitterbot
Allow: /
Crawl-delay: 10

User-agent: Amazonbot
Disallow: /
Crawl-delay: 86400

# Default rule for other bots
User-agent: *
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Disallow: /?s=
Disallow: /*?s=
Disallow: /search/
Disallow: /xmlrpc.php
Disallow: /trackback/
Disallow: /feed/
Disallow: /comments/
Disallow: /author/
Disallow: /wp-json/
Crawl-delay: 60

# No sitemap needed for dynamic pages

HTTP/1.1 301 Moved Permanently
Server: nginx
Date: Sun, 15 Mar 2026 12:03:13 GMT
Content-Type: text/html
Content-Length: 162
Connection: keep-alive
Keep-Alive: timeout=20
Location: https://projectools.io/

HTTP/1.1 200 OK
Server: nginx
Date: Sun, 15 Mar 2026 12:03:13 GMT
Content-Type: text/html; charset=UTF-8
Transfer-Encoding: chunked
Connection: keep-alive
Keep-Alive: timeout=20
Vary: Accept-Encoding
Vary: Accept-Encoding
Vary: Accept-Encoding
Expires: Thu, 19 Nov 1981 08:52:00 GMT
Link: ; rel="https://api.w.org/"
Link: ; rel="alternate"; title="JSON"; type="application/json"
Link: ; rel=shortlink
Pragma: no-cache
X-Powered-By: WP Engine
X-Cacheable: SHORT
Vary: Accept-Encoding,Cookie
Cache-Control: max-age=600, must-revalidate
X-Cache: HIT: 1
X-Cache-Group: normal
Content-Encoding: br