User-agent: * # list folders robots are not allowed to index Disallow: /cgi-bin/ Disallow: /images/ Disallow: /webmail/ Disallow: /shop/ Disallow: /wjrdeals/ Disallow: /wjrstores/ Disallow: /coupons/ Disallow: /cpns/ Disallow: /help/ # # The code below can be used to disallow access to all parts of your site just for our bot. User-Agent: RufusBot Disallow: / # User-agent: Googlebot-Image #Disallow:/ # list specific files robots are not allowed to index Disallow: /default.html Disallow: /default.php # # # End of robots.txt file