# Note: see http://www.robotstxt.org/wc/norobots.html # poorly behaved crawlers that pass junk data to us - go away! User-agent: snap.com beta crawler v0 Disallow: / # Note: google supports some extensions to robots.txt # "*" matches any sequence of characters, $ indicates end of a name #User-agent: Googlebot # general rules User-agent: * Disallow: /PO/ Disallow: /pope/ Disallow: /mp/ Disallow: /mypointsuk/ Disallow: /sape/ Disallow: /mp_tlbx/ Disallow: /DV2U/ Disallow: /affiliate Disallow: /info Disallow: /emp/gate/ Disallow: /emp/i/ Disallow: /emp/a/ Disallow: /emp/u/velocity.do Disallow: /emp/u/mad.do Disallow: /emp/u/ma.do Disallow: /emp/u/ad.do Disallow: /emp/u/a.do Disallow: /emp/u/ps.do Disallow: /emp/u/c/ Disallow: /emp/u/h/ Disallow: /emp/u/t/ #Disallow: /emp/u/redemptionCatalog.vm