User-agent: * # applies to all robots Disallow: /webstat # disallow indexing of directory pages Disallow: /cgi-bin/adcbrow.pl