# robots.txt file
# Generally speaking, don't disallow any image directories.
# update May 20 '18 -- Removed because Google needs access to /js to render pages! Don't disallow /js !!!
# update Mar 15 '20 -- Prior to this there was just 1 line ifateaps/app_data .. but we were getting a lot of garbage in site search ... added back many dirs
# update Jul 27 '20 -- removed full http://www.ifate.com* urls .. replaced with relative.
# update Jul 27 '20 -- Added user agent *
# update Oct 8 '20 -- Allow everything
# update Spring '23 -- disallow semrush, common crawl, majestic, ahrefs
# update April 24 -- Add delays to Proximic and grapeshot who are hammering the servers
# update August 29 2024 -- Added /oranum_click.php because these were getting crawled in GSC and it's just a click tracker
# update August 29 2024 -- Added /daily as this is all repeat content. The root horoscopes page is all that needs to be indexed.
# update August 29 2024 -- Added /numerology_reading.html as this is the 2nd page and can't be accessed directly without URL vars.
# update August 29 2024 -- Added /js/min/ifateprimary-min.js? This prevents the parameter at the end from being counted as a new file
# General rules
User-agent: *
Disallow: /oranum_click.php
Disallow: /daily
Disallow: /numerology_reading.html
Disallow: /js/min/ifateprimary-min.js?
Disallow: /i-ching-changes/iching-changes-headers/*.jpg
Disallow: /astrology-aspects/astrology-aspect-images/*.jpg
# Block specific bots
User-agent: SemrushBot
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: GPTBot
Disallow: /
# Set crawl delay for specific bots
User-agent: Proximic
Crawl-delay: 10
User-agent: grapeshot
Crawl-delay: 5