# Make changes for all web spiders
User-agent: *
Disallow: /cetadmin
Disallow: /cetstats
Disallow: /dbase
Disallow: /sql