User-agent: * # the following applies to all robots
Crawl-delay: 60 # wait 60 sec between sccuessive requests
Request-rate: 1/60 # one request every 60 sec allowed by robot
Disallow: /cgi-bin/  # prevents access to a directory by a compliant robot 
#Disallow: /  # prevents access to ALL pages by robots by a compliant robot