# robots.txt, v 1.0 2001/08/30  JJB
#
# This is a file retrieved by webwalkers a.k.a. spiders that 
# conform to a defacto standard.
#
# See <URL:http://www.robotstxt.org/wc/exclusion.html#robotstxt>
#
# Format is:
#       User-agent: <name of spider 1>
#       User-agent: <name of spider 2>
#       Disallow: <nothing> | <path>
#
#  The record starts with one or more User-agent lines, followed by one or more Disallow lines, 
#  as detailed below. Unrecognised headers are ignored.
#
#
#        For example: 
#        User-agent: webcrawler 
#        User-agent: infoseek 
#        Allow: /tmp/ok.html                               
#        Disallow: /tmp Disallow: /user/foo 
#
#    (Note, the allow command is not yet fully supported)
#
#
#    User-agent 
#      The value of this field is the name of the robot the record is describing access policy for. 
#      If more than one User-agent field is present the record describes an identical access policy
#      for more than one robot. At least one field needs to be present per record.
#
#      The robot should be liberal in interpreting this field. A case insensitive substring match of
#      the name without version information is recommended.
#
#      If the value is '*', the record describes the default access policy for any robot that has 
#      not matched any of the other records. It is not allowed to have multiple such records in 
#      the "/robots.txt" file.
#
#    Disallow 
#      The value of this field specifies a partial URL that is not to be visited. This can be a full 
#      path, or a partial path; any URL that starts with this value will not be retrieved. For example:
#        Disallow: /help  disallows /help/index.html and /help.html 
#        Disallow: /help/ disallows /help/index.html but allows /help.html. 
#      Any empty value, indicates that all URLs can be retrieved. At least one Disallow field needs 
#      to be present in a record.
#
#
#


# Disallow all Robots
User-agent: *
#Disallow:   /
Disallow:   /cgi-bin
Disallow:   /search
Disallow:   /agents
Disallow:   /Cgi-bin
Disallow:   /Search
Disallow:   /Agents
Disallow:   /CGI-BIN
Disallow:   /SEARCH
Disallow:   /AGENTS
Disallow:   /Personal/storage

#User-agent: Mozilla/3.01 (hotwired-test/0.1)
#User-agent: Slurp
#User-agent: Scooter
#User-agent: Ultraseek
#User-agent: smallbear
#User-agent: GoogleBot


# The robots below are very rude

User-agent: Teleport Pro
Disallow:/