User-agent: * # Dev sites disallow all # Disallow: / # Live sites disallow specific files we dont want crawled # plus add sitemap Disallow: /admin/ Disallow: /_privatefiles/ Disallow: /dbv/ Disallow: /includes/ Sitemap: http://www.cairnsslane.co.nz/sitemap.xml