User-agent: *
Disallow: /401
Disallow: /403
Disallow: /404
Disallow: /500
Disallow: /css
Disallow: /inc
Disallow: /shr

User-agent: AboutUsBot
User-agent: ACONTBOT
User-agent: Amfibibot
User-agent: antibot
User-agent: appie
User-agent: Aqua_Products
User-agent: asterias
Disallow: /

User-agent: b2w/0.1
User-agent: BackDoorBot*
User-agent: baiduspider
User-agent: BecomeBot
User-agent: Bilbo
User-agent: BlowFish
User-agent: Bookmark search tool
User-agent: BotALot
# User-agent: Browsershots
User-agent: BruinBot
User-agent: btbot
User-agent: BuiltBotTough
User-agent: Bullseye*
User-agent: BunnySlippers
Disallow: /

User-agent: Cegbfeieh
User-agent: cfetch
User-agent: CheeseBot
User-agent: CherryPicker*
User-agent: Clustera Crawler*
User-agent: CoolBot
User-agent: CopyRightCheck
User-agent: cosmos
User-agent: Cowbot
User-agent: Crescent*
User-agent: CyberPatrol
User-agent: CydralSpider
Disallow: /

User-agent: dcbspider
User-agent: discobot
User-agent: DittoSpyder
User-agent: dotbot
User-agent: Download Ninja
User-agent: Dumbot
Disallow: /

User-agent: EmailCollector
User-agent: EmailSiphon
User-agent: EmailWolf
User-agent: EnaBot*
User-agent: EroCrawler
User-agent: Euripbot
User-agent: Exabot
User-agent: ExtractorPro
Disallow: /

User-agent: FairAd Client
User-agent: Fetch
User-agent: FindLinks
User-agent: Firefly*
User-agent: Flaming AttackBot
User-agent: flatlandbot
User-agent: FollowSite*
User-agent: Foobot
Disallow: /

User-agent: Gaisbot
User-agent: Generic
User-agent: GetRight*
User-agent: Georgios
User-agent: Gigabot
User-agent: GoForIt*
User-agent: gonzo*
User-agent: great-plains-web-spider
User-agent: gridBOT
User-agent: grub
User-agent: grub-client
Disallow: /

User-agent: HappyFunBot*
User-agent: Harvest*
User-agent: Haste
User-agent: HenryTheMiragoRobot
User-agent: Heritrix
User-agent: hloader
User-agent: HooWWWer
User-agent: http://www.almaden.ibm.com/cs/crawler
User-agent: httplib
User-agent: HTTrack*
User-agent: humanlinks
Disallow: /

User-agent: ia_archiver*
User-agent: iCCrawler
User-agent: IconSurf
User-agent: InfoNaviRobot
User-agent: IRLbot
User-agent: Iron33*
User-agent: IXE Crawler
Disallow: /

User-agent: JemmaTheTourist
User-agent: JennyBot
User-agent: Jetbot
Disallow: /

User-agent: Kenjin Spider
User-agent: Keyword Density*
User-agent: KFSW-Bot
User-agent: KnowItAll
Disallow: /

User-agent: Lachesis
User-agent: larbin
User-agent: LexiBot
User-agent: libWeb/clsHTTP
User-agent: libwww
User-agent: LinkextractorPro
User-agent: linko
User-agent: LinkScan*
User-agent: LinkWalker
User-agent: LNSpiderguy
User-agent: LocalcomBot
User-agent: looksmart
User-agent: LucidMedia*
User-agent: lwp-trivial
Disallow: /

User-agent: Mata Hari
User-agent: Mediapartners-Google
User-agent: MentorMate Spider
User-agent: Microsoft URL Control*
User-agent: MIIxpc*
User-agent: mirago
User-agent: Mister PiX
User-agent: MMCrawler
User-agent: moget*
User-agent: Molbsy
User-agent: Moni
User-agent: MSIECrawler
User-agent: MyEngines-Bot
Disallow: /

User-agent: NaverBot*
User-agent: Net Attache*
User-agent: NetAnts
User-agent: NetMechanic
User-agent: NetResearchServer*
User-agent: NetSeer*
User-agent: NICErsPRO
User-agent: NimbleCrawler*
User-agent: nnn/ttt*
User-agent: Nokia6680*
User-agent: NPBot*
User-agent: NPT*
User-agent: Nutch*
Disallow: /

User-agent: ObjectsSearch*
User-agent: Offline Explorer*
User-agent: Openbot
User-agent: Openfind*
User-agent: Oracle Ultra Search
Disallow: /

User-agent: page_prefetcher
User-agent: penthesilea*
User-agent: PerMan
User-agent: PhpDig*
User-agent: PRCrawler*
User-agent: ProPowerBot*
User-agent: ProWebWalker
User-agent: psbot
User-agent: Python-urllib
Disallow: /

User-agent: QuepasaCreep
User-agent: QueryN Metasearch
Disallow: /

User-agent: Radiation Retriever*
User-agent: RepoMonkey*
User-agent: RMA
User-agent: RPT-HTTPClient
Disallow: /

User-agent: SBIder
User-agent: searchpreview
User-agent: Searchspider
User-agent: shelob*
User-agent: Shim-Crawler
User-agent: sitecheck.internetseer.com
User-agent: SiteSnagger*
User-agent: sna
User-agent: sohu-search
User-agent: SpankBot
User-agent: spanner
User-agent: Speedy
User-agent: Spider_ Monkey
User-agent: Spinne
User-agent: Spock*
User-agent: Steeler
User-agent: SuperGet
User-agent: suzuran
User-agent: Szukacz*
Disallow: /

User-agent: TAMU_CS_IRL_CRAWLER
User-agent: Teemer*
User-agent: Teleport*
User-agent: Telesoft
User-agent: The Intraformant
User-agent: TheNomad
User-agent: thesubot
User-agent: TightTwatBot
User-agent: toCrawl/UrlDispatcher
User-agent: Trend Micro WebSurf Booster*
User-agent: True_Robot*
User-agent: turingos
User-agent: TurnitinBot
User-agent: TutorGig
User-agent: twiceler
Disallow: /

User-agent: Ultraseek
User-agent: URL Control
User-agent: URL_Spider_Pro
User-agent: URLy Warning
User-agent: USyd-NLP-Spider
Disallow: /

User-agent: Vagabondo
User-agent: VCI*
User-agent: verticalman
User-agent: vilainrobot
User-agent: VIREL
User-agent: vscooter  
User-agent: VSE/1.0
Disallow: /

User-agent: Web Image Collector
User-agent: WebAuto
User-agent: WebBandit*
User-agent: WebCapture*
User-agent: WebClient 
User-agent: WebCopier*
User-agent: WebEnhancer
User-agent: WebmasterWorldForumBot
User-agent: WebReaper
User-agent: WebSauger* 
User-agent: Website Quester
User-agent: Webster Pro
User-agent: WebStripper
User-agent: Webwhacker*
User-agent: WinWebBot 
User-agent: WebZip*
User-agent: Wget*
User-agent: WWW-Collector-E
Disallow: /

User-agent: yacybot
User-agent: Yahoo-MMAudVid
User-agent: Yahoo-MMCrawler
User-agent: YebolBot
Disallow: /

User-agent: Zao
User-agent: zermelo*
User-agent: Zeus*
User-agent: ZipppBot
Disallow: /