#Disallow folders and files User-agent: * Disallow: /css Disallow: /images Disallow: /en/administrator Disallow: /shell Disallow: /usage Disallow: /webalizer Disallow: /cms Disallow: /feed Disallow: /ads user-agent: httrack #Disallow: #Google Wildcards User-agent: googlebot Disallow: *.cgi