User-agent: * Allow: /css/ Allow: /js/ Allow: /images/ # block only sensitive/private dirs Disallow: /admin/ Disallow: /cgi-bin/ Disallow: /tmp/