# robots.txt

# This hits us too fast, and doesn't get the 403 hint!
User-agent: grub-client
Disallow: /

ErrorDocument 400 /error.php
ErrorDocument 401 /error.php
ErrorDocument 403 /error.php
ErrorDocument 404 /error.php
ErrorDocument 500 /error.php

User-agent: *
Disallow: /catalog/
Disallow: /cgi-bin/
Disallow: /images/
Disallow: /bbclone/
Disallow: /cre/
Disallow: /templates/
Disallow: /toolbars/
Disallow: /stats/
Disallow: /pics/
Disallow: /internal/
Disallow: /audio/
Disallow: /video/
Disallow: /phpBB2/
Disallow: /_private/
Disallow: /php_uploads/
Disallow: /adverts/


# Misbehaving: requests much too fast:
User-agent: fast
Disallow: /

Disallow: /catalog/	  # new search engine
Disallow: /author/

Disallow: /cgi-bin/	  # old search engine

Disallow: /images/	  # images
Disallow: /pics/	  # images
Disallow: /internal/      # private
Disallow: /stats/	  # private