#User-agent: ia_archiver #Disallow: /content/ User-agent: * # directed to all spiders Disallow: /cgi-bin/ Disallow: /content/ Disallow: /images/ Disallow: /whois/