# exclude help system from robots
User-agent: *
Disallow: /manual/ 
Disallow: /doc/ 
Disallow: /gif/
Disallow: /chilledbaby/music/
Disallow: /cgi-bin/hcount.cgi
Disallow: /phpBB2/profile.php
Disallow: /phpBB2/viewonline.php
Disallow: /phpBB2/privmsg.php
Disallow: /phpBB2/login.php
Disallow: /phpBB2/faq.php
Disallow: /phpBB2/templates/
Disallow: /phpBB2/groupcp.php
Disallow: /phpBB2/memberlist.php
Disallow: /phpBB3/
Disallow: /bna/phpBB3/
Crawl-delay: 500
# Slow down msnbot
User-agent: msnbot
Crawl-delay: 900
Disallow: /manual/ 
Disallow: /doc/ 
Disallow: /gif/
Disallow: /chilledbaby/music/
User-agent: linksmanager
Disallow: /
User-agent: TurnitinBot
Disallow: /
User-Agent: RufusBot
Disallow: /
# but allow htdig to index our doc-tree
User-agent: susedig
Disallow:
# disallow stress test
user-agent: stress-agent
Disallow: /