################################### # Detected Spider/Bot: None # # Headers Sent: # Content-Type: text/plain # Sitemap: http://www.degreesofzero.com/index.php?zc=xml;get=sitemap # Google - Most Important bot # Unfortunately a robots.txt will only stop it crawling certain urls, and NOT adding any # urls which it comes across into its index. So we're relying on a meta noindex tag. User-agent: Googlebot # Don't index mobile versions Disallow: /index.php?*;wap Disallow: /index.php?*;wap2 Disallow: /index.php?*;imode # Default Folders Disallow: /Languages/ Disallow: /Mail/ Disallow: /lib/ Disallow: /Plugins/ Disallow: /Sources/ Disallow: /Themes/ Disallow: /cache/ Disallow: /doz/ Disallow: /inas/ Disallow: /dwin/ # Default Actions Disallow: /index.php?zc=cp Disallow: /index.php?zc=login Disallow: /index.php?zc=logout Disallow: /index.php?zc=register Disallow: /index.php?zc=help Disallow: /index.php?zc=contact_form Disallow: /index.php?zc=forum;p=* # Now allow bits and then disallow bits Allow: /robots.txt$ Allow: /index.php$ # Bad bot - Often ignores robots.txt - Waste of bandwidth # Despite claiming on their website to be a search engine in development # I'm suspicious as to whether they are a harvester pretending to be SE User-agent: Twiceler Disallow: / User-Agent: W3C-checklink Disallow: / # Stop following PHPSESSID's User-Agent: MJ12bot Disallow: /index.php?PHPSESSID # Default Folders Disallow: /Languages/ Disallow: /Mail/ Disallow: /lib/ Disallow: /Plugins/ Disallow: /Sources/ Disallow: /Themes/ Disallow: /cache/ Disallow: /doz/ Disallow: /inas/ Disallow: /dwin/ # Default Actions Disallow: /index.php?zc=cp Disallow: /index.php?zc=login Disallow: /index.php?zc=logout Disallow: /index.php?zc=register Disallow: /index.php?zc=help Disallow: /index.php?zc=contact_form Disallow: /index.php?zc=forum;p=* # Catch all (remainder) # Will be followed by any bots other than ones identified above # Uses BASIC robots.txt directives without wildcards, end-anchors etc # So Spiders should understand these (including MSNBOT) User-agent: * # Default Folders Disallow: /Languages/ Disallow: /Mail/ Disallow: /lib/ Disallow: /Plugins/ Disallow: /Sources/ Disallow: /Themes/ Disallow: /cache/ Disallow: /doz/ Disallow: /inas/ Disallow: /dwin/ # Default Actions Disallow: /index.php?zc=cp Disallow: /index.php?zc=login Disallow: /index.php?zc=logout Disallow: /index.php?zc=register Disallow: /index.php?zc=help Disallow: /index.php?zc=contact_form Disallow: /index.php?zc=forum;p=*