# robots.txt for http://www.example.com/ # User-agent: Slurp # Crawl-delay: 20 User-agent: Slurp Disallow: / User-agent: Inktomi Slurp Disallow: / User-agent: * Disallow: /email-addresses/ Disallow: /cgi-bin/ Disallow: /logs/ Disallow: /guestbook/ Disallow: /secure/ Disallow: /stats/ Disallow: /scripts/ Disallow: https:// Disallow: *.zip$