# No Indexing User-Agent: * Disallow: /*do= Disallow: /*idx= Disallow: /doku.php?id=talk Disallow: /?id=talk: Disallow: /?idx=talk Disallow: /*id=talk: Disallow: /*idx=talk Disallow: /talk: Disallow: /talk:* Disallow: /Kommentar: Disallow: /Kommentar:* Disallow: /Wiki:Users Disallow: /Wiki:Users* Disallow: /doku.php?id=Wiki-Testplatz:Wiki-Testplatz Disallow: /*id=Wiki-Testplatz:Wiki-Testplatz Disallow: /Wiki-Testplatz:Wiki-Testplatz Disallow: /doku.php?id=Admin:UserList Disallow: /?id=Admin:UserList Disallow: /*id=Admin:UserList Disallow: /Admin:UserList Disallow: /doku.php?idx=Admin Disallow: /?idx=Admin Disallow: /*idx=Admin Disallow: /Admin: Disallow: /doku.php?id=Diverses:Diskussion Disallow: /*id=Diverses:Diskussion Disallow: /Diverses:Diskussion Disallow: /doku.php?id=spamtrap Disallow: /*id=spamtrap Disallow: /spamtrap Disallow: /doku.php?id=Upload Disallow: /?id=Upload Disallow: /*id=Upload Disallow: /*idx=Upload Disallow: /Upload Disallow: /lib/* Disallow: /*install.php Disallow: /*mediamanager.php Disallow: /doku.php?id=403.shtml Disallow: /*id=403.shtml Disallow: /403.shtml User-Agent: Googlebot Allow: /Translation:Translation$ User-Agent: * Disallow: /Translation: Disallow: /Translation:* # No MailCollectors User-Agent: Titan Disallow: / User-Agent: EmailCollector Disallow: / User-Agent: EmailSiphon Disallow: / User-Agent: EmailWolf Disallow: / User-Agent: ExtractorPro Disallow: / User-Agent: WebmasterCoffee Disallow: / User-Agent: StackRambler Disallow: / User-Agent: LemurProject Disallow: / User-Agent: Nutch Disallow: / User-Agent: Charlotte Disallow: / User-Agent: Yanga Disallow: / User-Agent: Exabot Disallow: / User-Agent: DoCoMo Disallow: / User-Agent: Baiduspider Disallow: / User-Agent: Naverbot Disallow: / User-Agent: Naver Disallow: / User-Agent: Yeti Disallow: / User-Agent: sogou Disallow: / User-Agent: sogou spider Disallow: / User-Agent: yacybot Disallow: / User-Agent: YodaoBot Disallow: / User-Agent: QihooBot Disallow: / User-Agent: daumoa Disallow: / User-Agent: CazoodleBot Disallow: / User-Agent: SindiceBot Disallow: / User-Agent: BecomeJPBot Disallow: / User-Agent: disco Disallow: / User-Agent: psbot Disallow: / User-Agent: 008 Disallow: / User-Agent: 80legs Disallow: / User-Agent: SnapPreviewBot Disallow: / User-Agent: MJ12bot Disallow: / User-Agent: twiceler Disallow: / User-Agent: Speedy Disallow: / User-Agent: discobot Disallow: / User-Agent: spbot Disallow: / User-Agent: cityreview Disallow: / User-Agent: search17 Disallow: / User-Agent: aiHitBot Disallow: / User-Agent: Eurobot Disallow: / User-Agent: WebAlta Disallow: / User-Agent: MrCarlito Disallow: / User-Agent: WebDataCentreBot Disallow: / User-Agent: dotbot Disallow: / User-Agent: SimilarPages Disallow: / User-Agent: ssearch_bot Disallow: / User-Agent: SBIder Disallow: / User-Agent: kalooga Disallow: / User-Agent: msnbot-media Disallow: / User-Agent: gigabot Disallow: / User-Agent: archive.org_bot Disallow: / User-Agent: archive.org Disallow: / User-Agent: ia_archiver Disallow: / User-Agent: Linguee Disallow: / User-Agent: facebookexternalhit Disallow: / #User-agent: OOZBOT/SETOOZBOT/oozbot/setoozbot #Disallow: / # Please note: There are a lot of pages on this site, and there are # some misbehaved spiders out there that go _way_ too fast. If you're # irresponsible, your access to the site may be blocked. # advertising-related bots: User-Agent: Mediapartners-Google* Disallow: / # Crawlers that are kind enough to obey, but which we'd rather not have # unless they're feeding search engines. User-Agent: UbiCrawler Disallow: / User-Agent: DOC Disallow: / User-Agent: Zao Disallow: / # Some bots are known to be trouble, particularly those designed to copy # entire sites. # Please obey robots.txt! User-Agent: sitecheck.internetseer.com Disallow: / User-Agent: Zealbot Disallow: / User-Agent: MSIECrawler Disallow: / User-Agent: SiteSnagger Disallow: / User-Agent: WebStripper Disallow: / User-Agent: WebCopier Disallow: / User-Agent: Fetch Disallow: / User-Agent: Offline Explorer Disallow: / User-Agent: Teleport Disallow: / User-Agent: TeleportPro Disallow: / User-Agent: WebZIP Disallow: / User-Agent: linko Disallow: / User-Agent: HTTrack Disallow: / User-Agent: Microsoft.URL.Control Disallow: / User-Agent: Xenu Disallow: / User-Agent: larbin Disallow: / User-Agent: libwww Disallow: / User-Agent: ZyBORG Disallow: / User-Agent: Download Ninja Disallow: / # Sorry, wget in its recursive mode is a frequent problem. # Please read the man page and use it properly; there is a # --wait option you can use to set the delay between hits, # for instance. User-Agent: wget Disallow: / # The 'grub' distributed client has been *very* poorly behaved. User-Agent: grub-client Disallow: / # Doesn't follow robots.txt anyway, but... User-Agent: k2spider Disallow: / # Hits many times per second, not acceptable # http://www.nameprotect.com/botinfo.html User-Agent: NPBot Disallow: / # A capture bot, downloads gazillions of pages with no public benefit # http://www.webreaper.net/ User-Agent: WebReaper Disallow: / ## *at least* 1 second please. #Crawl-delay: 1