#START: User-Agent: *
User-Agent: *
#start: disallow directory indexing
Disallow: /arqvmtgw/
Disallow: /bpre/
Disallow: /cache/
Disallow: /common/
Disallow: /docs/
Disallow: /download/
Disallow: /editors/
Disallow: /email/
Disallow: /extras/
Disallow: /googlecheckout/
Disallow: /images/
Disallow: /includes/
Disallow: /mail/
Disallow: /media/
Disallow: /pub/
Disallow: /rss/
Disallow: /site/
#end: disallow directory indexing
Sitemap: http://shop.wbweb.co.uk/sitemap.xml
#start: disallow file indexing
Disallow: /favicon.ico
Disallow: /shop.rss
#end: disallow file indexing
#END: User-Agent: *
#START: User-Agent: Sphider2
User-Agent: Sphider2
#start: disallow directory indexing
Disallow: /arqvmtgw/
Disallow: /bpre/
Disallow: /cache/
Disallow: /common/
Disallow: /docs/
Disallow: /download/
Disallow: /editors/
Disallow: /email/
Disallow: /extras/
Disallow: /googlecheckout/
Disallow: /images/
Disallow: /includes/
Disallow: /mail/
Disallow: /media/
Disallow: /pub/
Disallow: /rss/
Disallow: /site/
#end: disallow directory indexing
Sitemap: http://shop.wbweb.co.uk/sitemap.xml
#start: disallow file indexing
Disallow: /favicon.ico
Disallow: /shop.rss
#end: disallow file indexing
#END: User-Agent: Sphider2