# http://www.robotstxt.org/wc/exclusion.html#robotstxt

User-agent: *
Disallow: /pub/mirror
Disallow: /pub/adsm
Disallow: /pub/antivirus
Disallow: /pub/campus
Disallow: /pub/debian
Disallow: /pub/debian-non-US
Disallow: /pub/debian-security
Disallow: /pub/internet-drafts
Disallow: /pub/kernel
Disallow: /pub/linux
Disallow: /pub/redhat
Disallow: /pub/redhat-addons
Disallow: /pub/redhat-contrib
Disallow: /pub/redhat-updates
Disallow: /pub/rfc
Disallow: /pub/sunbird
Disallow: /pub/sunsite
Disallow: /pub/suse
Disallow: /pub/tsm

# http://validator.w3.org/docs/checklink.html#bot
User-Agent: W3C-checklink
Disallow: