webmineral.com Review

 # robots.txt
# Robot-exclusion file for www.webmineral.com
# See: http://info.webcrawler.com/mak/projects/robots/norobots.html

User-agent: *

# ROOT DIRECTORY EXCLUSIONS FOLLOW
Disallow: /cgi-bin/
Disallow: /chem/
Disallow: /cindy/
Disallow: /crystal/
Disallow: /determin/
Disallow: /dana/
Disallow: /exclusions/
Disallow: /java/
Disallow: /jmol/
Disallow: /jpowd/
Disallow: /log/
Disallow: /sounds/
Disallow: /strunz/
Disallow: /MySQL/
Disallow: /minot/
Disallow: /news/
Disallow: /phpMyAdmin/
Disallow: /phpMyAdmin_old/
Disallow: /radioactivity/
Disallow: /references/
Disallow: /reflectivity/
Disallow: /sounds/
Disallow: /test/

# GLOBAL DIRECTORY EXCLUSIONS FOLLOW
Disallow: *cgi-bin/
Disallow: *chem/
Disallow: *cindy/
Disallow: *crystal/
Disallow: *dana/
Disallow: *determin/
Disallow: *exclusions/
Disallow: *java/
Disallow: *jmol/
Disallow: *jpowd/
Disallow: *log/
Disallow: *sounds/
Disallow: *strunz/
Disallow: *minot/
Disallow: *news/
Disallow: *phpMyAdmin/
Disallow: *phpMyAdmin_old/
Disallow: *radioactivity/
Disallow: *references/
Disallow: *reflectivity/
Disallow: *sounds/
Disallow: *test/

# SUB-DIRECTORY EXCLUSIONS FOLLOW
Disallow: /About/Info/
Disallow: /error.shtml

# File name EXCLUSIONS FOLLOW
Disallow: /Random-Text.htm

Disallow: /testads.shtml

User-agent: HTTrack
Disallow: /

User-agent: HTTrack 3.0x
Disallow: /

User-agent: Teleport Pro
Disallow: /

User-agent: Webdup
Disallow: /

User-agent: Wget
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: WebDownloader
Disallow: /

HTTP/1.1 200 OK
Date: Wed, 15 Jan 2025 00:23:49 GMT
Server: Apache
Accept-Ranges: bytes
Keep-Alive: timeout=5, max=100
Connection: Keep-Alive
Transfer-Encoding: chunked
Content-Type: text/html