# robots.txt file for foxriver.net # 240125 # Violations by unruly agents, bots, crawlers, spiders, and sponsors are reported and/or firewalled # Request-rate: 1/10 User-agent: * Crawl-delay: 10 User-agent: * Request-rate: 1/10 User-agent: 007AC9 Disallow: / User-agent: 008 Disallow: / User-agent: 08_800w_web1 Disallow: / User-agent: 123peoplebot Disallow: / User-agent: 8legs Disallow: / User-agent: Aboundex Disallow: / User-agent: Accoona-AI-Agent Disallow: / User-agent: adidxbot Disallow: / User-agent: AdnormCrawler Disallow: / User-agent: AdnormCrawler www.adnorm.com Disallow: / User-agent: AdsBot Disallow: / User-agent: AdsBot-Google Disallow: / User-agent: AdvBot Disallow: / User-agent: AESOP_com_SpiderMan Disallow: / User-agent: aftenie.com Disallow: / User-agent: ah-ha.com Disallow: / User-agent: ah-ha.com crawler Disallow: / User-agent: AhrefsBot Disallow: / User-agent: aiHitBot Disallow: / User-agent: aiHitBot-BP Disallow: / User-agent: AJCrawler Disallow: / User-agent: aiohttp Disallow: / User-agent: aipbot Disallow: / User-agent: Alexa Disallow: / User-agent: Alexibot Disallow: / User-agent: al_viewer Disallow: / User-agent: Aloofly Disallow: / User-agent: AlphaBot Disallow: / User-agent: Amazonbot Disallow: / User-agent: Amfibibot Disallow: / User-agent: Anarchy99 Disallow: / User-agent: AnswerChase PROve 3.0 Disallow: / User-agent: antbot Disallow: / User-agent: antibot-V1.1 Disallow: / User-agent: AOL_Daniel_Clark_Spider Disallow: / User-agent: Applebot Disallow: / User-agent: Arachnida Web Crawler Disallow: / User-agent: archive.org_bot Disallow: / User-agent: ascribeBot Disallow: / User-agent: asearcher Disallow: / User-agent: ASPSeek/1.2.8 Disallow: / User-agent: ASR Ranking Technology/Spider/Crawler Disallow: / User-agent: asterias Disallow: / User-agent: awcheckBot Disallow: / User-agent: Babya Discoverer Disallow: / User-agent: BackDoorBot/1.0 Disallow: / User-agent: BacklinkCrawler Disallow: / User-agent: BaiDuSpider Disallow: / User-agent: Baiduspider+ Disallow: / User-agent: Baiduspider+(+http://www.baidu.com/search/spider.htm) Disallow: / User-agent: Balihoo Disallow: / User-agent: Barkrowler Disallow: / User-agent: baypup Disallow: / User-agent: bbbot Disallow: / User-agent: BeamFall Disallow: / User-agent: BecomeBot Disallow: / User-agent: bemash Disallow: / User-agent: BeNosey Mohawk Crawler Disallow: / User-agent: Bigsearch.ca Disallow: / User-agent: binance.com Disallow: / User-agent: binlar_2.6.3 Disallow: / User-agent: BIXOCRAWLER Disallow: / User-agent: BLABLUBB Disallow: / User-agent: Black Hole Disallow: / User-agent: BlackBerry9000 Disallow: / User-agent: Blekkobot Disallow: / User-agent: BLEXBot Disallow: / User-agent: BlitzBOT Disallow: / User-agent: blogtop.us Disallow: / User-agent: blogtop.us crawler Disallow: / User-agent: BlowFish/1.0 Disallow: / User-agent: BlueRobot Disallow: / User-agent: boitho.com-dc Disallow: / User-agent: BoogleBot Disallow: / User-agent: borchuk Disallow: / User-agent: bot.wsowner.com Disallow: / User-agent: BotALot Disallow: / User-agent: BPImageWalker Disallow: / User-agent: BruinBot Disallow: / User-agent: BUbiNG Disallow: / User-agent: BublupBot Disallow: / User-agent: BuiltBotTough Disallow: / User-agent: Bullseye/1.0 Disallow: / User-agent: Butterfly Disallow: / User-agent: bumblebee Disallow: / User-agent: bumblebee@relevare.com Disallow: / User-agent: BunnySlippers Disallow: / User-agent: BusinessBot Disallow: / User-agent: BusinessSeek.biz_Directory Disallow: / User-agent: c3.sh Disallow: / User-agent: ca-crawler Disallow: / User-agent: Cabot Disallow: / User-agent: caizhi Disallow: / User-agent: CakePHP Disallow: / User-agent: calculon spider/0.1.0 Disallow: / User-agent: CamontSpider Disallow: / User-agent: CareerBot Disallow: / User-agent: CatchBot Disallow: / User-agent: CATExplorador Disallow: / User-agent: CazoodleBot Disallow: / User-agent: CC-rget Disallow: / User-agent: ccubee Disallow: / User-agent: CCBot Disallow: / User-agent: CensysInspect Disallow: / User-agent: CentiverseBot Disallow: / User-agent: Cegbfeieh Disallow: / User-agent: CFNetwork Disallow: / User-agent: Charlotte Disallow: / User-agent: CheckMarkNetwork Disallow: / User-agent: checks.panopta.com Disallow: / User-agent: CheeseBot Disallow: / User-agent: CherryPicker Disallow: / User-agent: CherryPickerSE Disallow: / User-agent: CherryPickerElite Disallow: / User-agent: Chilkat Disallow: / User-agent: CipaCrawler Disallow: / User-agent: cipherkey Disallow: / User-agent: CirrusExplorer Disallow: / User-agent: CJNetworkQuality Disallow: / User-agent: Clarabot Disallow: / User-agent: ClariaBot Disallow: / User-agent: classbot Disallow: / User-agent: Claymont.com Disallow: / User-agent: Cliqzbot Disallow: / User-agent: coccoc Disallow: / User-agent: Colly Disallow: / User-agent: Comodo HTTPS Crawler Disallow: / User-agent: Comodo-Webinspector-Crawler Disallow: / User-agent: CompSpyBot Disallow: / User-agent: Computer_and_Automation_Research_Institute_Crawler Disallow: / User-agent: Contacts-Crawler Disallow: / User-agent: ConveraCrawler Disallow: / User-agent: ConveraMultiMediaCrawler/0.1 Disallow: / User-agent: CopyRightCheck Disallow: / User-agent: core-project Disallow: / User-agent: cosmos Disallow: / User-agent: cowbot Disallow: / User-agent: cpp-httplib Disallow: / User-agent: CPython Disallow: / User-agent: CrawlBot Disallow: / User-agent: Crawler Disallow: / User-agent: crawler4j Disallow: / User-agent: CRAZYWEBCRAWLER Disallow: / User-agent: CrazyWebCrawler-Spider Disallow: / User-agent: Crescent Disallow: / User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0 Disallow: / User-agent: CslBot Disallow: / User-agent: CukBot Disallow: / User-agent: curl Disallow: / User-agent: cuwhois Disallow: / User-agent: Dalvik Disallow: / User-agent: Darwin Disallow: / User-agent: DataCha0s Disallow: / User-agent: DataForSeoBot Disallow: / User-agent: DataFountains Disallow: / User-agent: DataparkSearch Disallow: / User-agent: Dataprovider Disallow: / User-agent: Dataprovider.com Disallow: / User-agent: Daum Disallow: / User-agent: Daumoa Disallow: / User-agent: DBot Disallow: / User-agent: DBLBot Disallow: / User-agent: DCPBot/1.2 Disallow: / User-agent: Debian 1.1-1 Disallow: / User-agent: del.icio.us-thumbnails Disallow: / User-agent: DeuSu Disallow: / User-agent: DF Bot Disallow: / User-agent: DialogSearch.com Disallow: / User-agent: DiamondBot Disallow: / User-agent: DigExt Disallow: / User-agent: Digincore crawler bot Disallow: / User-agent: Digincore bot Disallow: / User-agent: Digincore crawler bot Disallow: / User-agent: disco Disallow: / User-agent: discobot Disallow: / User-agent: discoverybot Disallow: / User-agent: Dispatch Disallow: / User-agent: DittoSpyder Disallow: / User-agent: dloader Disallow: / User-agent: dloader(NaverRobot) Disallow: / User-agent: DnBCrawler-Analytics Disallow: / User-agent: DNSMine Disallow: / User-agent: doccheckbot Disallow: / User-agent: DoCoMo Disallow: / User-agent: DomainAppender Disallow: / User-agent: domainsbot Disallow: / User-agent: Domain Re-Animator Bot Disallow: / User-agent: DomainCrawler Disallow: / User-agent: Domain Re-Animator Bot Disallow: / User-agent: DomainTaggingbot Disallow: / User-agent: DotBot Disallow: / User-agent: Dow Jones Searchbot Disallow: / User-agent: DownloadBot Disallow: / User-agent: DTS Agent Disallow: / #User-agent: DuckDuckBot #Disallow: / User-agent: Dulance bot Disallow: / User-agent: EasyDL Disallow: / User-agent: EC2LinkFinder Disallow: / User-agent: elefent Disallow: / User-agent: ELB-HealthChecker Disallow: / User-agent: EmailCollector Disallow: / User-agent: EMail Exractor Disallow: / User-agent: EmailSiphon Disallow: / User-agent: EmailWolf Disallow: / User-agent: Embarcadero Disallow: / User-agent: Embarcadero URI Client Disallow: / User-agent: Embedly Disallow: / User-agent: emefgebot Disallow: / User-agent: emeraldshield.com webbot Disallow: / User-agent: Enolyst Crawler Disallow: / User-agent: eripp Disallow: / User-agent: EroCrawler Disallow: / User-agent: Eurobot Disallow: / User-agent: Exabot Disallow: / User-agent: exactseek.com Disallow: / User-agent: ExactSeekCrawler Disallow: / User-agent: Exalead Disallow: / User-agent: Exalead NG/MimeLive Client Disallow: / User-agent: exooba Disallow: / User-agent: exooba crawler Disallow: / User-agent: Expanse Disallow: / User-agent: Experibot-v3 Disallow: / User-agent: ExtractorPro Disallow: / User-agent: Ezooms Disallow: / User-agent: facebookexternalhit Disallow: / User-agent: facebookscraper Disallow: / User-agent: FacetEngine Disallow: / User-agent: fast Disallow: / User-agent: FAST Enterprise Crawler/6.4 Disallow: / User-agent: FAST-WebCrawler Disallow: / User-agent: fasthttp Disallow: / User-agent: FatBoyKimComBot Disallow: / User-agent: Faxobot Disallow: / User-agent: GPTBot Disallow: / User-agent: GRequests Disallow: / User-agent: GuzzleHttp Disallow: / User-agent: Hivemind Disallow: / User-agent: http://www.faxo.com Disallow: / User-agent: FDSE Disallow: / User-agent: feedfetcher Disallow: / User-agent: feedfinder Disallow: / User-agent: Feosey Mohk Crawler Disallow: / User-agent: Fetcher Disallow: / User-agent: FHScan Core Disallow: / User-agent: fidget-spinner-bot Disallow: / User-agent: findlinks Disallow: / User-agent: flatlandbot Disallow: / User-agent: FlightDeckReportsBot Disallow: / User-agent: Fluid Disallow: / User-agent: Foobot Disallow: / User-agent: fr-crawler Disallow: / User-agent: FreeWebMonitoring SiteChecker Disallow: / User-agent: fujilabolx5 Disallow: / User-agent: FusionBot Disallow: / User-agent: FyberSpider Disallow: / User-agent: Gaisbot Disallow: / User-agent: GarlikCrawler Disallow: / User-agent: GD CRAWLER Disallow: / User-agent: Gecko 20020913 Disallow: / User-agent: genieBot Disallow: / User-agent: Genrepsoft Disallow: / User-agent: GeoHasher Disallow: / User-agent: gh-index-bot/Nutch-1.0 Disallow: / User-agent: gigabaz Disallow: / User-agent: gigaBazV11.3 Disallow: / User-agent: gigablast.com/spider.html Disallow: / User-agent: GigablastOpenSource Disallow: / User-agent: Gigabot Disallow: / User-agent: Gigamega.bot Disallow: / User-agent: GIGRIB Disallow: / User-agent: Gimme60bot Disallow: / User-agent: GingerCrawler Disallow: / User-agent: Girafabot Disallow: / User-agent: Gluten Free Crawler Disallow: / User-agent: Go-http-client Disallow: / User-agent: Go 1.1 package http Disallow: / User-agent: gobyus Disallow: / User-agent: GOFORITBOT Disallow: / User-agent: Googlebot-Image Disallow: / User-agent: Googlebot-News Disallow: / User-agent: Googlebot-Video Disallow: / User-agent: goose Disallow: / User-agent: Gowikibot Disallow: / User-agent: grabber Disallow: / User-agent: GrapeshotCrawler Disallow: / User-agent: Grasshopper Disallow: / User-agent: great-plains-web-spider Disallow: / User-agent: grub Disallow: / User-agent: grub-client Disallow: / User-agent: gulliver Disallow: / User-agent: Gulliver/1.3 Disallow: / User-agent: Gulper Disallow: / User-agent: GurujiBot Disallow: / User-agent: Harvest Disallow: / User-agent: Harvest-NG Disallow: / User-agent: Harvest-NG 1.0.2 Disallow: / User-agent: Harvest-NG Example Disallow: / User-agent: HEADMasterSEO Disallow: / User-agent: HELLO Crawler Disallow: / User-agent: Heratrix Disallow: / User-agent: Heretrix Disallow: / User-agent: Heritix Disallow: / User-agent: Heritrix Disallow: / User-agent: hloader Disallow: / User-agent: holmes Disallow: / User-agent: HoneyBee Disallow: / User-agent: HostTracker Disallow: / User-agent: HPL Disallow: / User-agent: HubSpot Crawler Disallow: / User-agent: hunter Disallow: / User-agent: htdig Disallow: / User-agent: http-kit Disallow: / User-agent: httplib Disallow: / User-agent: HTTP_Request2 Disallow: / User-agent: humanlinks Disallow: / User-agent: HyPeR Disallow: / User-agent: ia_archiver Disallow: / User-agent: Ibot Disallow: / User-agent: ICC-Crawler Disallow: / User-agent: IconSurf Disallow: / User-agent: Ideare+-+SignSite/1.2 Disallow: / User-agent: idmarch Disallow: / User-agent: ilial Disallow: / User-agent: Iltrovatore-Setaccio Disallow: / User-agent: imbot Disallow: / User-agent: ImplisenseBot Disallow: / User-agent: IonCrawl Disallow: / User-agent: IncyWincy Disallow: / User-agent: IndeedBot Disallow: / User-agent: Indy Library Disallow: / User-agent: InfoNaviRobot Disallow: / User-agent: InfoSeek Sidewinder/1.0A Disallow: / User-agent: IntegromeDB Disallow: / User-agent: intelium_bot Disallow: / User-agent: InternetMeasurement Disallow: / User-agent: InternetSeer Disallow: / User-agent: InternetSeer.com Disallow: / User-agent: iPhone; CPU iPhone OS 6_0 like Mac OS X Disallow: / User-agent: IPiumBot Disallow: / User-agent: ips-agent Disallow: / User-agent: ip-web-crawler.com Disallow: / User-agent: Isidorus Disallow: / User-agent: IstellaBot Disallow: / User-agent: izsearch.com Disallow: / User-agent: Jack Disallow: / User-agent: Jakarta Commons-HttpClient Disallow: / User-agent: Java Disallow: / User-agent: Java1.3.0 Disallow: / User-agent: Java1.4.0 Disallow: / User-agent: Java1.4.0_02 Disallow: / User-agent: Java/1.6.0_04 Disallow: / User-agent: JennyBot Disallow: / User-agent: JetBot/1.0 Disallow: / User-agent: Jigsaw Disallow: / User-agent: Kavam Disallow: / User-agent: Kenjin Spider Disallow: / User-agent: Keyword Density/0.9 Disallow: / User-agent: KickFire Disallow: / User-agent: kmky-not-a-bot Disallow: / User-agent: KnowItAll Disallow: / User-agent: KomodiaBot Disallow: / User-agent: Krugle/ Disallow: / User-agent: Krugle/Krugle Disallow: / User-agent: Kryptos Logic Telltale Disallow: / User-agent: kuloko Disallow: / User-agent: kuloko-bot/0.2 Disallow: / User-agent: Kyoto-Crawler Disallow: / User-agent: Kyoto-Tohoku-Crawler Disallow: / User-agent: Lachesis Disallow: / User-agent: lanshanbot Disallow: / User-agent: Larbin Disallow: / User-agent: larbin_2.5.9 Disallow: / User-agent: Larbin2.6.2 Disallow: / User-agent: larbin2.6.3@unspecified.mail Disallow: / User-agent: larbin_2.6.2 Disallow: / User-agent: larbin_2.6.3 Disallow: / User-agent: LegalX Deep Analyzer Disallow: / User-agent: LexiBot Disallow: / User-agent: libcurl Disallow: / User-agent: libcurl 7.9.1 Disallow: / User-agent: libWeb/clsHTTP Disallow: / User-agent: libwww Disallow: / User-agent: libwww-perl Disallow: / User-agent: LightspeedSystemsCrawler Disallow: / User-agent: lincobot Disallow: / User-agent: Linguee Bot Disallow: / User-agent: LinkChecker Disallow: / User-agent: linkdexbot Disallow: / User-agent: LinkextractorPro Disallow: / User-agent: LinkedInBot/1.0 Disallow: / User-agent: linko/0.1 libwww-perl/5.66 Disallow: / User-agent: LinkpadBot Disallow: / User-agent: LinkScan Disallow: / User-agent: LinkScan/8.1a Unix Disallow: / User-agent: LinkStats Bot Disallow: / User-agent: LinkWalker Disallow: / User-agent: Lipperhey Disallow: / User-agent: Lipperhey-Kaus-Australis Disallow: / User-agent: Lipperhey SEO Service Disallow: / User-agent: Lipperhey Spider Disallow: / User-agent: listfender Disallow: / User-agent: LocalcomBot Disallow: / User-agent: LNSpiderguy Disallow: / User-agent: Lsearch/sondeur Disallow: / User-agent: LSSRocketCrawler Disallow: / User-agent: ltx71 Disallow: / User-agent: lwp Disallow: / User-agent: LWP::Simple Disallow: / User-agent: lwp-trivial Disallow: / User-agent: Mag Reiting Disallow: / User-agent: Magic Browser Disallow: / User-agent: MagicBot Disallow: / User-agent: mahonie Disallow: / User-agent: Mail.Ru Disallow: / User-agent: Mail.RU_Bot Disallow: / User-agent: Marvin v0.5 Disallow: / User-agent: MarkWatch/1.0 Disallow: / User-agent: masscan Disallow: / User-agent: Mata Hari Disallow: / User-agent: MauiBot Disallow: / User-agent: meanpathbot Disallow: / User-agent: MegaIndex Disallow: / User-agent: MegaIndex.ru Disallow: / User-agent: memoryBot Disallow: / User-agent: metacarta Disallow: / User-agent: MetaIntelligence Disallow: / User-agent: Metaspinner Disallow: / User-agent: mfibot Disallow: / User-agent: MIA Disallow: / User-agent: MIA DEV Disallow: / User-agent: Microsoft URL Control Disallow: / User-agent: MIIxpc Disallow: / User-agent: MIIxpc/4.2 Disallow: / User-agent: Missigua Locator Disallow: / User-agent: Missigua Locator 1.9 Disallow: / User-agent: Mister PiX Disallow: / User-agent: MixrankBot Disallow: / User-agent: MnoGoSearch Disallow: / User-agent: MJ12bot Disallow: / User-agent: MnoGoSearch/3.2.32 Disallow: / User-agent: moget Disallow: / User-agent: MojeekBot Disallow: / User-agent: Mojolicious Disallow: / User-agent: Morfeus Fucking Scanner Disallow: / User-agent: MOTIbot Disallow: / User-agent: Mozilla/4.0 (compatible; http://search.thunderstone.com/texis/websearch/about.html) Disallow: / User-agent: Mozilla/5.0 (compatible; evc-batch/2.0) Disallow: / User-agent: MQBOT Disallow: / User-agent: MSIECrawler Disallow: / User-agent: msnbot-media Disallow: / User-agent: MSRBOT Disallow: / User-agent: MTRobot Disallow: / User-agent: My User Agent Disallow: / User-agent: MyNutchSpider Disallow: / User-agent: myseosnapshot Disallow: / User-agent: Muenster University of Applied Sciences; +http://fb02itsscan.fh-muenster.de Disallow: / User-agent: muhstik Disallow: / User-agent: muhstik-scan Disallow: / User-agent: MXT Disallow: / User-agent: myse Disallow: / User-agent: nabot Disallow: / User-agent: NaverBot Disallow: / User-agent: NAMEPROTECT Disallow: / User-agent: NCBot Disallow: / User-agent: Neevabot Disallow: / User-agent: NerdyBot Disallow: / User-agent: NetAnts Disallow: / User-agent: NetcraftSurveyAgent Disallow: / User-agent: netEstate Disallow: / User-agent: netEstate NE Crawler Disallow: / User-agent: NetID.com Disallow: / User-agent: NetID.com Bot 1.0 Disallow: / User-agent: NetMechanic Disallow: / User-agent: NetNoseCrawler Disallow: / User-agent: NetNoseCrawler 2.0 Disallow: / User-agent: Net Probe Disallow: / User-agent: NetResearchServer Disallow: / User-agent: netscan.lekus.su Disallow: / User-agent: NetSeer Disallow: / User-agent: NetSrcherP Disallow: / User-agent: NetSrcherP/0.2.3 Disallow: / User-agent: Networking4all Disallow: / User-agent: Networking4all Bot Disallow: / User-agent: NextGenSearchBot Disallow: / User-agent: NG Disallow: / User-agent: nicebot Disallow: / User-agent: NimbleCrawler Disallow: / User-agent: NimbleCrawler 1.12 Disallow: / User-agent: Nimbostratus-Bot Disallow: / User-agent: NICErsPRO Disallow: / User-agent: NIS Nutch Spider Disallow: / User-agent: nlpproject.info Disallow: / User-agent: node-fetch Disallow: / User-agent: noyona_0_1 Disallow: / User-agent: noxtrumbot Disallow: / User-agent: NPBot Disallow: / User-agent: NRLCorpusBuilder Disallow: / User-agent: nrsbot Disallow: / User-agent: Nu_tch-1.0-dev Disallow: / User-agent: nu_tch-princeton Disallow: / User-agent: Nutch Disallow: / User-agent: Nutch-0.9-dev Disallow: / User-agent: Nutch-1.0-dev Disallow: / User-agent: NutchOrg Disallow: / User-agent: NutchCVS/0.03-dev Disallow: / User-agent: newspaper Disallow: / User-agent: oBot Disallow: / User-agent: Offline Explorer Disallow: / User-agent: OmniExplorer_Bot Disallow: / User-agent: OnetSzukaj Disallow: / User-agent: OOZBOT Disallow: / User-agent: openbot Disallow: / User-agent: Openfind Disallow: / User-agent: OpenfosBot Disallow: / User-agent: openwebspider Disallow: / User-agent: OrangeBot Disallow: / User-agent: OrgProbe Disallow: / User-agent: OutclicksBot Disallow: / User-agent: OutfoxBot Disallow: / User-agent: Owler Disallow: / User-agent: page_prefetcher Disallow: / User-agent: page_test Disallow: / User-agent: PageAnalyzer Disallow: / User-agent: PagePeeker.com Disallow: / User-agent: PagesInventory Disallow: / User-agent: Pagespeed Disallow: / User-agent: PageThing Disallow: / User-agent: Pangalactic Gargleblaster Disallow: / User-agent: panscient Disallow: / User-agent: panscient.com Disallow: / User-agent: pantest Disallow: / User-agent: Pcore-HTTP Disallow: / User-agent: PEERbot Disallow: / User-agent: Peter Wang Disallow: / User-agent: PHPCrawl Disallow: / User-agent: Pingdom Disallow: / User-agent: Pingdom GIGRIB Disallow: / User-agent: PinkPad_Free Disallow: / User-agent: Pinterest Disallow: / User-agent: Pinterestbot Disallow: / User-agent: PiplBot Disallow: / User-agent: Pixray-Seeker Disallow: / User-agent: Plukkie Disallow: / User-agent: Pompos Disallow: / User-agent: PrivacyAwareBot Disallow: / User-agent: prlog Disallow: / User-agent: probethenet Disallow: / User-agent: probethenet.com scanner Disallow: / User-agent: Program Shareware 1.0.3 Disallow: / User-agent: Project 25499 Scanner Disallow: / User-agent: ProPowerBot/2.14 Disallow: / User-agent: ProWebWalker Disallow: / User-agent: proximic Disallow: / User-agent: psbot Disallow: / User-agent: psycheclone Disallow: / User-agent: puf Disallow: / User-agent: Purebot Disallow: / User-agent: Pycurl Disallow: / User-agent: python-requests Disallow: / User-agent: Python-urllib Disallow: / User-agent: QACC browser Disallow: / User-agent: QueryN Metasearch Disallow: / User-agent: Quick-Crawler Disallow: / User-agent: Qwantify Disallow: / User-agent: rabaz Disallow: / User-agent: rabot Disallow: / User-agent: RageComics Disallow: / User-agent: RAMPyBot Disallow: / User-agent: RAMPyBot - www.giveRAMP.com/0.1 Disallow: / User-agent: Raven Disallow: / User-agent: Raven1.1.3 Disallow: / User-agent: Re-re Studio Disallow: / User-agent: ReactorNetty Disallow: / User-agent: REAP Disallow: / User-agent: REAP-crawler Nutch Disallow: / User-agent: redback Disallow: / User-agent: RepoMonkey Disallow: / User-agent: RepoMonkey Bait & Tackle/v1.01 Disallow: / User-agent: research-scanner Disallow: / User-agent: research-spider Disallow: / User-agent: ResearchProject Disallow: / User-agent: rest-client Disallow: / User-agent: RestSharp Disallow: / User-agent: RiceComputerArchitecture Disallow: / User-agent: RiceComputerArchitecture Crawler Disallow: / User-agent: rico/0.1 Disallow: / User-agent: Riddler Disallow: / User-agent: RMA Disallow: / User-agent: Robotgenius Disallow: / User-agent: Robotgenius crawler Disallow: / User-agent: roboto Disallow: / User-agent: rogerbot Disallow: / User-agent: RPT-HTTPClient/0.3-3 Disallow: / User-agent: Ruby Disallow: / User-agent: Ruby, link_thumbnailer Disallow: / User-agent: Runet-Research-Crawler Disallow: / User-agent: SafeDNSBot Disallow: / User-agent: SalesIntelligent Disallow: / User-agent: SBIder Disallow: / User-agent: Scanbot Disallow: / User-agent: ScoutAbout Disallow: / User-agent: ScoutJet Disallow: / User-agent: Scrapy Disallow: / User-agent: Screaming Frog SEO Spider Disallow: / User-agent: ScreenerBot Disallow: / User-agent: ScreenerBot Crawler Beta 2.0 Disallow: / User-agent: search.ch Disallow: / User-agent: search.marginalia.nu Disallow: / User-agent: SearchDaimon.com-dc Disallow: / User-agent: SearchmetricsBot Disallow: / User-agent: Seekport Crawler Disallow: / User-agent: SeekportBot Disallow: / User-agent: sees.co Disallow: / User-agent: sees.co bot Disallow: / User-agent: semalt Disallow: / User-agent: semanticdiscovery Disallow: / User-agent: SemanticJuice Disallow: / User-agent: SemrushBot Disallow: / User-agent: Sensis Web Crawler Disallow: / User-agent: SEOkicks-Robot Disallow: / User-Agent: SEOlyt Disallow: / User-agent: seoscanners.net Disallow: / User-agent: serpstatbot Disallow: / User-agent: SETOOZBOT Disallow: / User-agent: SeznamBot Disallow: / User-agent: Sgrunt Disallow: / User-agent: SheenBot Disallow: / User-agent: shelob Disallow: / User-agent: shelob v1.0 Disallow: / User-agent: sherlock Disallow: / User-agent: Shim-Crawler Disallow: / User-agent: ShrinkTheWeb.com Disallow: / User-agent: ShrinkTheWeb.com Crawler v1.0 Disallow: / User-agent: SimilarPages Disallow: / User-agent: SISTRIX Disallow: / User-agent: SISTRIX Crawler Disallow: / User-agent: SiteBot Disallow: / User-agent: sitecheck Disallow: / User-agent: sitecheck.internetseer.com Disallow: / User-agent: sitechecker Disallow: / User-agent: SiteExplorer Disallow: / User-agent: siteintel.net Disallow: / User-agent: siteintel.net Bot Disallow: / User-agent: SiteLockSpider Disallow: / User-agent: SiteOne-Crawler Disallow: / User-agent: SiteSnagger Disallow: / User-agent: Skywalker Disallow: / User-agent: SlySearch Disallow: / User-agent: smart.apnoti.com Robot/v1.34 (http://smart.apnoti.com/bot) Disallow: / User-agent: SMEALSearch-Bot Disallow: / User-agent: SMTBot Disallow: / User-agent: Snapbot Disallow: / User-agent: SnapPreviewBot Disallow: / User-agent: sogou Disallow: / User-agent: Sogou web spider Disallow: / User-agent: sohu-search Disallow: / User-agent: SolomonoBot Disallow: / User-agent: Sosospider Disallow: / User-agent: spbot Disallow: / User-agent: SpankBot Disallow: / User-agent: spanner Disallow: / User-agent: speedfind ramBot xtreme 8.1 Disallow: / User-agent: Speedy Disallow: / User-agent: SpeedySpider Disallow: / User-agent: Speedy Spider Disallow: / User-agent: sphider Disallow: / User-agent: Spijdar Disallow: / User-agent: Sproose Disallow: / User-agent: Spyder disallow:/ User-agent: Spyder/Nutch-2.1 disallow:/ User-agent: Sqworm Disallow: / User-agent: ssearch_bot Disallow: / User-agent: StackRambler Disallow: / User-agent: Steeler Disallow: / User-agent: STINGbot Disallow: / User-agent: Stratagems Kumo Disallow: / User-agent: Struct.it Disallow: / User-agent: StructuredWeb Disallow: / User-agent: StructuredWeb Agent Disallow: / User-agent: StumbleUpon Disallow: / User-agent: sufog Disallow: / User-agent: suggybot Disallow: / User-agent: SuperPagesUrlVerifyBot Disallow: / User-agent: SurdotlyBot Disallow: / User-agent: SurveyBot Disallow: / User-agent: Symfony BrowserKit Disallow: / User-agent: Synapse Disallow: / User-agent: sysscan Disallow: / User-agent: Szukacz Disallow: / User-agent: Tagoobot Disallow: / User-agent: tags2dir.net Disallow: / User-agent: Taiga Disallow: / User-agent: Taiga web spider Disallow: / User-agent: tbot-nutch Disallow: / User-agent: Teemer Disallow: / User-agent: TeeRaidBot Disallow: / User-agent: Teleport Disallow: / User-agent: TeleportPro Disallow: / User-agent: Telesoft Disallow: / User-agent: Telesphoreo Disallow: / User-agent: Teradex Mapper Disallow: / User-agent: test-bot Disallow: / User-agent: TestBot Disallow: / User-agent: TestCrawler Disallow: / User-agent: testingforex.com Disallow: / User-agent: textdigger Disallow: / User-agent: The Knowledge AI Disallow: / User-agent: TheNomad Disallow: / User-agent: ThumbSniper Disallow: / User-agent: T-H-U-N-D-E-R-S-T-O-N-E Disallow: / User-agent: TightTwatBot Disallow: / User-agent: Tipiweb Disallow: / User-agent: Titan Disallow: / User-agent: TMCrawler Disallow: / User-agent: TMM Crawler Disallow: / User-agent: toCrawl/UrlDispatcher Disallow: / User-agent: Toplistbot Disallow: / User-agent: Topsy Disallow: / User-agent: TosCrawler Disallow: / User-agent: Toweyabot Disallow: / User-agent: TprAdsTxtCrawler Disallow: / User-agent: trafilatura Disallow: / User-agent: True_Robot Disallow: / User-agent: TScoutBot Disallow: / User-agent: turingos Disallow: / User-agent: TurnitinBot Disallow: / User-agent: TweetmemeBot Disallow: / User-agent: TwengaBot Disallow: / User-agent: Twiceler Disallow: / User-agent: Twitturly Disallow: / User-agent: u12Bot Disallow: / User-agent: UCmore Disallow: / User-agent: UK Crawler Disallow: / User-agent: Uptimebot Disallow: / User-agent: URLAppendBot Disallow: / User-agent: URLy Warning Disallow: / User-agent: URL_Spider_SQL/1.0 Disallow: / User-agent: UXCrawlerBot Disallow: / User-agent: Vagabondo Disallow: / User-agent: Valuationbot Disallow: / User-agent: VCI Disallow: / User-agent: VCI WebViewer VCI WebViewer Win32 Disallow: / User-agent: vebidoobot Disallow: / User-agent: verticalman Disallow: / User-agent: verzamelgids.nl Disallow: / User-agent: Virusdie crawler Disallow: / User-agent: VisBot Disallow: / User-agent: VisiStat Bot Disallow: / User-agent: visionutils Disallow: / User-agent: visual ramBot xtreme 7.0 Disallow: / User-agent: VoilaBot Disallow: / User-agent: voltron Disallow: / User-agent: voyager Disallow: / User-agent: VSE Disallow: / User-agent: VSE/1.0 Disallow: / User-agent: vuhuvBot Disallow: / User-agent: Waterfall Robot Disallow: / User-agent: wauuu engine Disallow: / User-agent: wbdbot Disallow: / User-agent: WBSearchBot Disallow: / User-agent: Web front page analyser. Disallow: / User-agent: Web Image Collector Disallow: / User-agent: WebAlta Crawler/2.0 Disallow: / User-agent: WebAuto Disallow: / User-agent: WebBandit Disallow: / User-agent: Webclipping.com Disallow: / User-agent: WebCopier Disallow: / User-agent: WebEnhancer Disallow: / User-agent: WebFilter Disallow: / User-agent: WebFilter Robot 1.0 Disallow: / User-agent: WebHistoryBot Disallow: / User-agent: WebKit Disallow: / User-agent: WebRACE/1.1 Disallow: / User-agent: WebSauger Disallow: / User-agent: websays Disallow: / User-agent: Website Quester Disallow: / User-agent: WEBSITEtheWEB.COM Disallow: / User-agent: Webster Pro Disallow: / User-agent: WebStripper Disallow: / User-agent: WebTarantula.com Crawler Disallow: / User-agent: WebZip Disallow: / User-agent: Wegtam Crawler Disallow: / User-agent: Wells Search II Disallow: / User-agent: Wget Disallow: / User-agent: WikiDo Disallow: / User-agent: WinInet Test Disallow: / User-agent: WhatWeb Disallow: / User-agent: whiteiexpres Disallow: / User-agent: whizbang Disallow: / User-agent: Who.is Bot Disallow: / User-agent: wonderbot Disallow: / User-agent: woriobot Disallow: / User-agent: WorldLight Disallow: / User-agent: Wotbot Disallow: / User-agent: Wotbox Disallow: / User-agent: wsr-agent Disallow: / User-agent: WWW-Collector-E Disallow: / User-agent: WWW-Mechanize Disallow: / User-agent: www.integromedb.org/Crawler Disallow: / User-agent: www.probethenet.com scanner Disallow: / User-agent: www.socialayer.com Agent 0.1 Disallow: / User-agent: wwwster Disallow: / User-agent: wwwtype Disallow: / User-agent: WyoBot Disallow: / User-agent: Xenu Disallow: / User-agent: Xenu Link Sleuth Disallow: / User-agent: Xenu's User-agent: Xenu's Link Sleuth Disallow: / User-agent: XoviBot Disallow: / User-agent: xpymep.exe Disallow: / User-agent: Y!J-ASR/0.1 crawler Disallow: / User-agent: yacybot Disallow: / User-agent: Yahoo! Slurp China Disallow: / User-agent: YahooMobileMessenger Disallow: / User-agent: Yandex Disallow: / User-agent: Yandexbot Disallow: / User-agent: Yanga Disallow: / User-agent: Yanga WorldSearch Bot Disallow: / User-agent: Yeti Disallow: / User-agent: YisouSpider Disallow: / User-agent: YodaoBot Disallow: / User-agent: Yowedo Disallow: / User-agent: YowedoBot Disallow: / User-agent: YPBot Disallow: / User-agent: ZaldomoSearchBot Disallow: / User-agent: Zauba Crawler Disallow: / User-agent: ZemlyaCrawl Disallow: / User-agent: Zeus Disallow: / User-agent: Zeus 32297 Webster Pro V2.9 Win32 Disallow: / User-agent: Zeus 60359 Webster Pro V2.9 Win32 Disallow: / User-Agent: zgrab Disallow: / User-agent: ZoominfoBot Disallow: / User-agent: ZSEBOT Disallow: / User-agent: ZyBorg Disallow: / User-agent: * Crawl-delay: 5 # please hit me no more than once in 5 sec