# For all robots
User-agent: *
Disallow: /login                # Block all access to the login page
Disallow: /MyAccount/           # Block all access to my account pages
Disallow: /myaccount/
Disallow: /checkout             # Block all access to the checkout pages
Disallow: /unregisteredCheckout
Disallow: /basket
Request-rate: 1/3               # maximum rate is one page every 3 seconds
Visit-time: 0400-0845           # only visit between 04:00 and 08:45 UTC

# Block CazoodleBot as it does not present correct accept content headers
User-agent: CazoodleBot
Disallow: /

# Block MJ12bot as it is just noise
User-agent: MJ12bot
Disallow: /

# Block dotbot as it cannot parse base urls properly
User-agent: dotbot/1.0
Disallow: /

# Block Gigabot
User-agent: Gigabot
Disallow: /
			
Sitemap: http://www.babiesrus.co.uk/sitemap-babies-1.xml.gz