# RobotInclude.txt # Robot web browsers Configuration List # This list is prepared for the Analog Log Analysis Program but may be adapted for other applications # # Current Status # Updated: 5/3/2007 # ROBOTINCLUDE entries: 668 # # Instructions: Save file as RobotInclude.txt # In your analog configuration file, add the line: # # CONFIGFILE RobotInclude.txt # # This file and other analog resources may be found at: # http://www.owen.vanderbilt.edu/mike.shor/diversions/analog/ # # Maintained by Mike Shor # http://www.owen.vanderbilt.edu/mike.shor/ # mike.shor AT owen.vanderbilt.edu # # ROBOTINCLUDE "???" # GetBot ROBOTINCLUDE "?Surf15a*" ROBOTINCLUDE "16274.345.67.23 WebWasher 3.0" ROBOTINCLUDE "AbachoBOT*" # Abacho.com ROBOTINCLUDE "ABCdatos BotLink/*" ROBOTINCLUDE "Accoona-AI-Agent*" ROBOTINCLUDE "AcoiRobot" ROBOTINCLUDE "Acoon?Robot" ROBOTINCLUDE "AESOP_com_SpiderMan" ROBOTINCLUDE "Ahoy! The Homepage Finder*" ROBOTINCLUDE "AITCSRobot/*" # HI (HTML Index) Search ROBOTINCLUDE "AITCSRobot/1.1" # HI (HTML Index) Search ROBOTINCLUDE "Aladin/3*" ROBOTINCLUDE "AlkalineBOT*" ROBOTINCLUDE "*almaden.ibm.*" ROBOTINCLUDE "AmigaVoyager" ROBOTINCLUDE "AnthillV*" ROBOTINCLUDE "A-Online?Search" ROBOTINCLUDE "Aport" ROBOTINCLUDE "appie/*" # Walhello appie ROBOTINCLUDE "appie/1.1" # Walhello appie ROBOTINCLUDE "Arachnoidea*" # Euroseek ROBOTINCLUDE "Arachnophilia*" ROBOTINCLUDE "Araneo/*" ROBOTINCLUDE "AraybOt/*" ROBOTINCLUDE "ArchitextSpider*" # Excite ROBOTINCLUDE "arks/*" # arks ROBOTINCLUDE "arks/1.0" # arks ROBOTINCLUDE "*Ask Jeeves/Teoma*" # Ask.com / Askjeeves.com ROBOTINCLUDE "*Ask*Jeeves.*" # Ask.com / Askjeeves.com ROBOTINCLUDE "ASpider/0.09" # ASpider (Associative Spider) ROBOTINCLUDE "ASPSeek*" ROBOTINCLUDE "*AspTear?1.*" ROBOTINCLUDE "ASSORT/*" ROBOTINCLUDE "AtlantisSearch" # Atlantis ROBOTINCLUDE "ATN_Worldwide*" # ATN Worldwide ROBOTINCLUDE "Atomz/*" # Atomz.com Search Robot ROBOTINCLUDE "Atomz/1.0" # Atomz.com Search Robot ROBOTINCLUDE "AURESYS/*" # AURESYS ROBOTINCLUDE "AURESYS/1.0" # AURESYS ROBOTINCLUDE "aWapClient*" # Skymob.com ROBOTINCLUDE "BackRub/*.*" # BackRub ROBOTINCLUDE "*BACS?http://www.ba.be*" ROBOTINCLUDE "Baiduspider*" # BaiDu ROBOTINCLUDE "BaiDuSpider" # BaiDu ROBOTINCLUDE "BaySpider" ROBOTINCLUDE "bbot/*" # BBot ROBOTINCLUDE "bbot/0.100" # BBot ROBOTINCLUDE "*BecomeBot*" ROBOTINCLUDE "Big Brother*" # Big Brother ROBOTINCLUDE "Bizbot003" ROBOTINCLUDE "BizBot04 kirk.overleaf.com" ROBOTINCLUDE "Bjaaland/*" # Bjaaland ROBOTINCLUDE "Bjaaland/0.5" # Bjaaland ROBOTINCLUDE "*Black Widow v*" # BlackWidow ROBOTINCLUDE "BlackWidow*" # BlackWidow ROBOTINCLUDE "borg-bot/*" # Borg-Bot ROBOTINCLUDE "borg-bot/0.9" # Borg-Bot ROBOTINCLUDE "Bot mailto:craftbot@yahoo.com" ROBOTINCLUDE "BoxSeaBot/*" # BoxSeaBot ROBOTINCLUDE "BoxSeaBot/0.5 (http://boxsea.com/crawler)" # BoxSeaBot ROBOTINCLUDE "BSDSeek*" ROBOTINCLUDE "BSpider/1.0 libwww-perl/0.40" # BSpider ROBOTINCLUDE "BullsEye*" ROBOTINCLUDE "bumblebee@*" ROBOTINCLUDE "bumblebee@relevare.com" ROBOTINCLUDE "BunnySlippers" ROBOTINCLUDE "CACTVS Chemistry Spider" # CACTVS Chemistry Spider ROBOTINCLUDE "Calif/*" # Calif ROBOTINCLUDE "Calif/0.6 (kosarev@tnps.net; http://www.tnps.dp.ua)" # Calif ROBOTINCLUDE "CaliforniaBrownSpider" ROBOTINCLUDE "Checkbot/x.xx LWP/*" # Checkbot ROBOTINCLUDE "Checkbot/x.xx LWP/5.x" # Checkbot ROBOTINCLUDE "cIeNcIaFiCcIoN.nEt Spider (http://www.cienciaficcion.net)" # cIeNcIaFiCcIoN.nEt ROBOTINCLUDE "CMC/*" # CMC/0.01 ROBOTINCLUDE "CMC/0.01" # CMC/0.01 ROBOTINCLUDE "combine/*" # Combine System ROBOTINCLUDE "combine/0.0" # Combine System ROBOTINCLUDE "ComputingSite Robi/*" # ComputingSite Robi/1.0 ROBOTINCLUDE "ComputingSite Robi/1.0 (robi@computingsite.com)" # ComputingSite Robi/1.0 ROBOTINCLUDE "Confuzzledbot/X.X (+http://www.confuzzled.lu/bot/)*" # ConfuzzledBot ROBOTINCLUDE "CoolBot*" # CoolBot ROBOTINCLUDE "cosmos/*" # XYLEME Robot ROBOTINCLUDE "cosmos/0.3" # XYLEME Robot ROBOTINCLUDE "crawler@alexa*" # Alexa ROBOTINCLUDE "*crawler_admin@bigfoot.com*" # BigFoot ROBOTINCLUDE "CrawlPaper/n.n.n (Windows n)*" # WallPaper (alias crawlpaper) ROBOTINCLUDE "CSE@IITBombay" ROBOTINCLUDE "Cuam*" ROBOTINCLUDE "Cusco/*" # Cusco ROBOTINCLUDE "Cusco/3.2" # Cusco ROBOTINCLUDE "Custom?Spider*" ROBOTINCLUDE "CyberSpyder (amant@www.cyberspyder.com)" # CyberSpyder Link Test ROBOTINCLUDE "CyberSpyder/*" # CyberSpyder Link Test ROBOTINCLUDE "CyberSpyder/2.1" # CyberSpyder Link Test ROBOTINCLUDE "CydralSpider/X.X (Cydral Web Image Search; http://www.cydral.com/)" # CydralSpider ROBOTINCLUDE "DataFountains/DMOZ*" # DataFountains/DMOZ Downloader ROBOTINCLUDE "DaviesBot*" ROBOTINCLUDE "dbDig*" ROBOTINCLUDE "DesertRealm.com;*" # Desert Realm Spider ROBOTINCLUDE "DesertRealm.com; 0.2; [J];" # Desert Realm Spider ROBOTINCLUDE "Deweb/*" ROBOTINCLUDE "Deweb/1.01" # DeWeb(c) Katalog/Index ROBOTINCLUDE "Die Blinde Kuh*" # Die Blinde Kuh ROBOTINCLUDE "dienstspider/1.0" # DienstSpider ROBOTINCLUDE "Digger/*" # Digger ROBOTINCLUDE "Digger/1.0 JDK/1.3.0" # Digger ROBOTINCLUDE "Digimarc CGIReader/*" # Digimarc Marcspider/CGI ROBOTINCLUDE "Digimarc CGIReader/1.0" # Digimarc Marcspider/CGI ROBOTINCLUDE "Digimarc WebReader/*" # Digimarc MarcSpider ROBOTINCLUDE "Digimarc WebReader/1.2" # Digimarc MarcSpider ROBOTINCLUDE "DigOut4U*" ROBOTINCLUDE "DIIbot*" # Digital Integrity Robot ROBOTINCLUDE "dlw3robot/x.y (in TclX by http://hplyot.obspm.fr/~dl/)*" # Tcl W3 Robot ROBOTINCLUDE "DNAbot/*" # DNAbot ROBOTINCLUDE "DNAbot/1.0" # DNAbot ROBOTINCLUDE "DoCoMo*" ROBOTINCLUDE "DragonBot/*" # DragonBot ROBOTINCLUDE "DragonBot/1.0 libwww/5.0" # DragonBot ROBOTINCLUDE "Duppies*" # Phantom ROBOTINCLUDE "DWCP/*" # DWCP (Dridus' Web Cataloging Project) ROBOTINCLUDE "DWCP/2.0" # DWCP (Dridus' Web Cataloging Project) ROBOTINCLUDE "EbiNess/*" # EbiNess ROBOTINCLUDE "EbiNess/0.01a" # EbiNess ROBOTINCLUDE "EchO!*" ROBOTINCLUDE "EI*Net/0.1 libwww/0.1" ROBOTINCLUDE "*eidetica.com/spider*" ROBOTINCLUDE "EIT-Link-Verifier-Robot/*" # EIT Link Verifier Robot ROBOTINCLUDE "EIT-Link-Verifier-Robot/0.2" # EIT Link Verifier Robot ROBOTINCLUDE "elfinbot*" # ELFINBOT ROBOTINCLUDE "Emacs-w3/v*" # Emacs-w3 Search Engine ROBOTINCLUDE "EmailSiphon*" ROBOTINCLUDE "EmailWolf*" ROBOTINCLUDE "EmailWolf 1.00" ROBOTINCLUDE "EMC Spider" # ananzi ROBOTINCLUDE "EnigmaBot" ROBOTINCLUDE "Enterprise_Search/*" ROBOTINCLUDE "envolk[ITS]spider*" # Envolk Web Search robot ROBOTINCLUDE "EroCrawler" ROBOTINCLUDE "esculapio/*" # esculapio ROBOTINCLUDE "esculapio/1.1" # esculapio ROBOTINCLUDE "ESIRover v*" # FetchRover ROBOTINCLUDE "ESIRover v1.0" # FetchRover ROBOTINCLUDE "ESISmartSpider/2.0" # Smart Spider ROBOTINCLUDE "e-SocietyRobot*" ROBOTINCLUDE "esther*" # Esther ROBOTINCLUDE "Evliya Celebi v*" # Evliya Celebi ROBOTINCLUDE "Evliya Celebi v0.151 - http://ilker.ulak.net.tr" # Evliya Celebi ROBOTINCLUDE "Excalibur*" ROBOTINCLUDE "explorersearch*" # nzexplorer ROBOTINCLUDE "*EZResult*" ROBOTINCLUDE "FAST Enterprise Crawler/*" ROBOTINCLUDE "FAST MetaWeb*" # Fast/AllTheWeb ROBOTINCLUDE "*FastCrawler*" # Fast/AllTheWeb ROBOTINCLUDE "FastCrawler 3.0.X (crawler@1klik.dk) - http://www.1klik.dk" # FastCrawler ROBOTINCLUDE "*FAST-WebCrawler*" # Fast/AllTheWeb ROBOTINCLUDE "*FDSE robot*" ROBOTINCLUDE "*FDSE?robot*" ROBOTINCLUDE "FelixIDE/*" # Felix IDE ROBOTINCLUDE "FelixIDE/1.0" # Felix IDE ROBOTINCLUDE "fido/*" # fido ROBOTINCLUDE "fido/0.9 Harvest/1.4.pl2" # fido ROBOTINCLUDE "Firefly/*" # Firefly ROBOTINCLUDE "Fish-Search-Robot*" # Fish search ROBOTINCLUDE "*Fluffy?the?spider*" ROBOTINCLUDE "*flunky*" ROBOTINCLUDE "flunky crawler_admin@bigfoot.com" ROBOTINCLUDE "*fouineur v*" ROBOTINCLUDE "Freecrawl*" # Freecrawl ROBOTINCLUDE "FunnelWeb-*" # FunnelWeb ROBOTINCLUDE "FunnelWeb-1.0" # FunnelWeb ROBOTINCLUDE "GAIS*Robot*" # seed.net.tw search ROBOTINCLUDE "GaisLab*" # gais.cs.ccu.edu.tw ROBOTINCLUDE "GaisLab data gatherer*" # gais.cs.ccu.edu.tw ROBOTINCLUDE "gammaSpider xxxxxxx ()/" # gammaSpider, FocusedCrawler ROBOTINCLUDE "gazz/*" # gazz ROBOTINCLUDE "gazz/1.0" # gazz ROBOTINCLUDE "GB2 LinkChecker" ROBOTINCLUDE "gcreep/*" # GCreep ROBOTINCLUDE "gcreep/1.0" # GCreep ROBOTINCLUDE "geckobot*" ROBOTINCLUDE "gestaltIconoclast/1.0 libwww-FM/2.17" # Popular Iconoclast ROBOTINCLUDE "GetURL.rexx v*" # GetURL ROBOTINCLUDE "GetURL.rexx v1.05" # GetURL ROBOTINCLUDE "gigabaz*" ROBOTINCLUDE "Gigabot/*" ROBOTINCLUDE "GNODSPIDER*" # www.gnod.net ROBOTINCLUDE "Golem/*" # Golem ROBOTINCLUDE "Golem/1.1" # Golem ROBOTINCLUDE "Googlebot*" # Google-Yahoo ROBOTINCLUDE "googlebot*" # Google ROBOTINCLUDE "Googlebot/2*" # Googlebot # ROBOTINCLUDE "Googlebot/2.X (+http://www.googlebot.com/bot.html)" # Googlebot ROBOTINCLUDE "*gossamer-threads*" # Directory Management program ROBOTINCLUDE "grabber*" # Direct Hit Grabber ROBOTINCLUDE "griffon/*" # Griffon ROBOTINCLUDE "griffon/1.0" # Griffon ROBOTINCLUDE "Gromit/*" # Gromit ROBOTINCLUDE "Gromit/1.0" # Gromit ROBOTINCLUDE "Gulliver*" # Northernlight ROBOTINCLUDE "Gulliver/*" # Northern Light Gulliver ROBOTINCLUDE "Gulliver/1.1" # Northern Light Gulliver ROBOTINCLUDE "Gulper Web Bot*" # Gulper Bot ROBOTINCLUDE "Gulper Web Bot 0.2.4 (www.ecsl.cs.sunysb.edu/~maxim/cgi-bin/Link/GulperBot)" # Gulper Bot ROBOTINCLUDE "Hämähäkki/*" # Hämähäkki ROBOTINCLUDE "Hämähäkki/0.2" # Hämähäkki ROBOTINCLUDE "HappyBot (gserver.kw.net)" ROBOTINCLUDE "Harvest/*" ROBOTINCLUDE "Harvest-NG*" ROBOTINCLUDE "havIndex/X.xx[bxx]*" # havIndex ROBOTINCLUDE "Hazel's Ferret Web hopper*" # Wild Ferret Web Hopper #1, #2, #3 ROBOTINCLUDE "heraSpider" ROBOTINCLUDE "Hippias/*" ROBOTINCLUDE "HKU WWW Robot*" # HKU WWW Octopus ROBOTINCLUDE "HomePageSearch*" ROBOTINCLUDE "Hometown Spider Pro" # Hometown Spider Pro ROBOTINCLUDE "htdig/*" # ht://Dig ROBOTINCLUDE "htdig/3.1.0b2" # ht://Dig ROBOTINCLUDE "HTMLgobble v*" # HTMLgobble ROBOTINCLUDE "HTMLgobble v2.2" # HTMLgobble ROBOTINCLUDE "http://www.sygol.com*" # Sygol ROBOTINCLUDE "I Robot*" # I, Robot ROBOTINCLUDE "I Robot 0.4 (irobot@chaos.dk)" # I, Robot ROBOTINCLUDE "ia_archiver*" # Alexa ROBOTINCLUDE "IAGENT/*" # IntelliAgent ROBOTINCLUDE "IAGENT/1.0" # IntelliAgent ROBOTINCLUDE "iajaBot/*" # iajaBot ROBOTINCLUDE "iajaBot/0.1" # iajaBot ROBOTINCLUDE "IBM_Planetwide*" # IBM_Planetwide ROBOTINCLUDE "Ibot/1.0 libwww-perl/0.40 " ROBOTINCLUDE "IlTrovatore*" # Italian Iltrovatore-Setaccio search engine ROBOTINCLUDE "image.kapsi.net/*" # image.kapsi.net ROBOTINCLUDE "image.kapsi.net/1.0" # image.kapsi.net ROBOTINCLUDE "IncyWincy/*" # IncyWincy ROBOTINCLUDE "IncyWincy/1.0b1" # IncyWincy ROBOTINCLUDE "Inet?library*" ROBOTINCLUDE "InfoNaviRobot*" # fujitsu.co.jp ROBOTINCLUDE "Informant*" # Informant ROBOTINCLUDE "InfoSeek Robot*" # InfoSeek Robot 1.0 ROBOTINCLUDE "InfoSeek Robot 1.0" # InfoSeek Robot 1.0 ROBOTINCLUDE "Infoseek?Sidewinder*" # Infoseek Sidewinder ROBOTINCLUDE "InfoSpiders/0.1" # InfoSpiders ROBOTINCLUDE "INGRID/*" # Ingrid ROBOTINCLUDE "INGRID/0.1" # Ingrid ROBOTINCLUDE "Inktomi?Search*" # Inktomi ROBOTINCLUDE "inktomisearch*" # Inktomi ROBOTINCLUDE "inspectorwww/*" # Inspector Web ROBOTINCLUDE "inspectorwww/1.0 http://www.greenpac.com/inspectorwww.html" # Inspector Web ROBOTINCLUDE "Internet Cruiser Robot/*" # Internet Cruiser Robot ROBOTINCLUDE "Internet Cruiser Robot/2.1" # Internet Cruiser Robot ROBOTINCLUDE "Internet?Ninja*" ROBOTINCLUDE "*InternetSeer.com*" # InternetSeer ROBOTINCLUDE "Iron33/*" # Iron33 ROBOTINCLUDE "Iron33/0.0" # Iron33 ROBOTINCLUDE "IsraeliSearch/*" # Israeli-search ROBOTINCLUDE "IsraeliSearch/1.0" # Israeli-search ROBOTINCLUDE "ITI Spider" ROBOTINCLUDE "Java*" ROBOTINCLUDE "JavaBee*" # JavaBee ROBOTINCLUDE "JavaBee" # JavaBee ROBOTINCLUDE "JBot*" # JBot Java Web Robot ROBOTINCLUDE "JBot (but can be changed by the user)" # JBot Java Web Robot ROBOTINCLUDE "JCrawler/0.2" # JCrawler ROBOTINCLUDE "Jigsaw*" # W3C CSS code ROBOTINCLUDE "JoBo*" # JoBo Java Web Robot ROBOTINCLUDE "JoBo (can be modified by the user)" # JoBo Java Web Robot ROBOTINCLUDE "Jobot/*" # Jobot ROBOTINCLUDE "Jobot/0.1alpha libwww-perl/4.0" # Jobot ROBOTINCLUDE "JoeBot/x.x*" # JoeBot ROBOTINCLUDE "JubiiRobot/version#*" # The Jubii Indexing Robot ROBOTINCLUDE "jumpstation*" # JumpStation ROBOTINCLUDE "Katipo/*" # Katipo ROBOTINCLUDE "Katipo/1.0" # Katipo ROBOTINCLUDE "KDD-Explorer/*" # KDD-Explorer ROBOTINCLUDE "KDD-Explorer/0.1" # KDD-Explorer ROBOTINCLUDE "KE_1.0*" ROBOTINCLUDE "Kenjin Spider" ROBOTINCLUDE "Kenjin?Spider*" ROBOTINCLUDE "KIT-Fireball/*" # KIT-Fireball (fireball.de) ROBOTINCLUDE "KIT-Fireball/2.0 libwww/5.0a" # KIT-Fireball (fireball.de) ROBOTINCLUDE "KO_Yappo_Robot/*" # KO_Yappo_Robot ROBOTINCLUDE "KO_Yappo_Robot/1.0.4(http://yappo.com/info/robot.html)" # KO_Yappo_Robot ROBOTINCLUDE "kulturarw3*" # www.kulturarw3.kb.se Heritage project ROBOTINCLUDE "LabelGrab/*" # LabelGrabber ROBOTINCLUDE "LabelGrab/1.1" # LabelGrabber ROBOTINCLUDE "larbin*" # larbin ROBOTINCLUDE "*larbin*" # www.merl.com ROBOTINCLUDE "larbin (+mail)*" # larbin # ROBOTINCLUDE "larbin_2.2.0 (crawl@compete.com)" # larbin ROBOTINCLUDE "LB-Crawler/1.0" ROBOTINCLUDE "LECodeChecker*" # linkexchange.com ROBOTINCLUDE "legs*" # legs ROBOTINCLUDE "LEIA/*" ROBOTINCLUDE "LexiBot*" ROBOTINCLUDE "Libertech-Rover www.libertech.com?" ROBOTINCLUDE "libWeb*" ROBOTINCLUDE "libwww-perl-*" # Sift ROBOTINCLUDE "libwww-perl-5.41" # Sift ROBOTINCLUDE "Linkidator/*" # Link Validator ROBOTINCLUDE "Linkidator/0.93" # Link Validator ROBOTINCLUDE "linklooker" ROBOTINCLUDE "LinkProver*" ROBOTINCLUDE "*Links2Go?Similarity*" ROBOTINCLUDE "LinkScan*" ROBOTINCLUDE "LinkScan Server/*" ROBOTINCLUDE "LinkScan Server/5.5 | LinkScan Workstation/5.5" # LinkScan ROBOTINCLUDE "LinkWalker*" # LinkWalker ROBOTINCLUDE "Lockon/xxxxx*" # Lockon ROBOTINCLUDE "logo.gif crawler" # logo.gif Crawler ROBOTINCLUDE "LWP*" # Perl ROBOTINCLUDE "LWP::*" # Perl ROBOTINCLUDE "lwp-trivial*" ROBOTINCLUDE "Lycos/x.x*" # Lycos ROBOTINCLUDE "Lycos_Spider*" # Lycos ROBOTINCLUDE "M/*" # Mattie ROBOTINCLUDE "M/3.8" # Mattie ROBOTINCLUDE "Magpie/*" # Magpie # ROBOTINCLUDE "Magpie/1.0" # Magpie ROBOTINCLUDE "Mariner/*" ROBOTINCLUDE "MARS SV" ROBOTINCLUDE "Marvin*" # marvin/infoseek ROBOTINCLUDE "marvin/infoseek (marvin-team@webseek.de)*" # marvin/infoseek ROBOTINCLUDE "MediaCrawler*" ROBOTINCLUDE "MediaFox/x.y*" # MediaFox ROBOTINCLUDE "MediaSearch/*" ROBOTINCLUDE "Merritt/1.0" ROBOTINCLUDE "MerzScope*" # MerzScope ROBOTINCLUDE "MetaGer*" # MetaGer-LinkChecker ROBOTINCLUDE "MFC_Tear_Sample" ROBOTINCLUDE "Microsoft URL Control 5.01.4319" ROBOTINCLUDE "Microsoft URL Control 6.00.8862" ROBOTINCLUDE "MindCrawler" # MindCrawler ROBOTINCLUDE "*MiragoRobot*" ROBOTINCLUDE "Mitsu" ROBOTINCLUDE "moget/*" # moget ROBOTINCLUDE "moget/1.0" # moget ROBOTINCLUDE "MOMspider/1.00 libwww-perl/0.40" # MOMspider ROBOTINCLUDE "Monster/vX.X.X -*" # Monster ROBOTINCLUDE "Motor/*" # Motor ROBOTINCLUDE "Motor/0.2" # Motor ROBOTINCLUDE "mouse.house/*" # spider_monkey ROBOTINCLUDE "mouse.house/7.1" # spider_monkey ROBOTINCLUDE "Mozilla 3.01 PBWF (Win95)" # Imagelock ROBOTINCLUDE "Mozilla/2.0 (compatible fouineur v2.0; fouineur.9bit.qc.ca)" # Fouineur ROBOTINCLUDE "Mozilla/2.0 (compatible; Ask Jeeves/Teoma)" # AskJeeves ROBOTINCLUDE "Mozilla/3.0 (Black Widow v1.1.0; Linux 2.0.27; Dec 31 1997 12:25:00" # TACH Black Widow ROBOTINCLUDE "Mozilla/3.01 (compatible;)" # bright.net caching robot ROBOTINCLUDE "Mozilla/4.0 (compatible: FDSE robot)" # Fluid Dynamics Search Engine robot ROBOTINCLUDE "Mozilla/4.0 (compatible: Pimptrain's robot)" # Pimptrain.com's robot ROBOTINCLUDE "Mozilla/4.0 (compatible; ChristCrawler.com*" # ChristCrawler.com ROBOTINCLUDE "Mozilla/4.0 (compatible; SpiderView 1.0;unix)" # SpiderView(tm) ROBOTINCLUDE "Mozilla/4.0 (Sleek Spider/1.2)" # Sleek ROBOTINCLUDE "*Mozilla@somewhere.com*" ROBOTINCLUDE "*MS?Search?4.??Robot*" ROBOTINCLUDE "msnbot*" # MSNBot ROBOTINCLUDE "MSNBOT/*" # MSNBot ROBOTINCLUDE "msnbot/*" # MSNBot # ROBOTINCLUDE "MSNBOT/0.1 (http://search.msn.com/msnbot.htm)" # MSNBot ROBOTINCLUDE "msnbot-media*" # MSNBot ROBOTINCLUDE "MultiText/*" ROBOTINCLUDE "Muninn/*" # Muninn ROBOTINCLUDE "Muninn/0.1 libwww-perl-5.76 (http://people.freenet.de/Muninn/)" # Muninn ROBOTINCLUDE "*MuscatFerret*" # Euroferret ROBOTINCLUDE "MuscatFerret/*" # Muscat Ferret ROBOTINCLUDE "MwdSearch/*" # Mwd.Search ROBOTINCLUDE "MwdSearch/0.1" # Mwd.Search ROBOTINCLUDE "MyCNNSpider" ROBOTINCLUDE "nabot*" ROBOTINCLUDE "NationalDirectory*" ROBOTINCLUDE "NaverBot-*" ROBOTINCLUDE "NDSpider/1.5" # NDSpider ROBOTINCLUDE "NEC?Research?Agent*" ROBOTINCLUDE "NEC-MeshExplorer*" # NEC-MeshExplorer ROBOTINCLUDE "Nederland.zoek*" # Nederland.zoek ROBOTINCLUDE "NetCarta CyberPilot Pro*" # NetCarta WebMap Engine ROBOTINCLUDE "*Netcraft?Web?Server*" ROBOTINCLUDE "NetMechanic*" # NetMechanic ROBOTINCLUDE "NetScoop/*" # NetScoop ROBOTINCLUDE "NetScoop/1.0 libwww/5.0a" # NetScoop ROBOTINCLUDE "NetZippy" ROBOTINCLUDE "NeuralBot*" ROBOTINCLUDE "newscan-online/*" # newscan-online ROBOTINCLUDE "newscan-online/1.1" # newscan-online ROBOTINCLUDE "*NEWT*ActiveX*" # Webcollector ROBOTINCLUDE "NHSEWalker/*" # NHSE Web Forager ROBOTINCLUDE "NHSEWalker/3.0" # NHSE Web Forager ROBOTINCLUDE "no*" # WebMirror/Spry Wizard Robot/Hyper-Decontextualizer/The Web Wombat ROBOTINCLUDE "Noago*" ROBOTINCLUDE "Nokia7110" ROBOTINCLUDE "Nomad-V2.x*" # Nomad ROBOTINCLUDE "None*" # Bloodhound ROBOTINCLUDE "none" # WebZinger ROBOTINCLUDE "NorthStar*" # The NorthStar Robot ROBOTINCLUDE "NOSEYBOLLOCKS V1.0" ROBOTINCLUDE "not available" # Search.Aus-AU.COM ROBOTINCLUDE "noxtrumbot*" ROBOTINCLUDE "NPBot*" ROBOTINCLUDE "nttdirectory_robot*" ROBOTINCLUDE "Nutscrape" ROBOTINCLUDE "ObjectsSearch/*" # ObjectsSearch ROBOTINCLUDE "ObjectsSearch/0.01" # ObjectsSearch ROBOTINCLUDE "oBot?((*" ROBOTINCLUDE "Occam/*" # Occam ROBOTINCLUDE "Occam/1.0" # Occam ROBOTINCLUDE "Ocelli/*" ROBOTINCLUDE "OGspider" ROBOTINCLUDE "OliverPerry*" ROBOTINCLUDE "OmniExplorer*" # OmniExplorer_Bot ROBOTINCLUDE "OntoSpider/1.0 libwww-perl/5.65" # OntoSpider ROBOTINCLUDE "*Openfind*" # Openfind.com.tw ROBOTINCLUDE "Openfind data gatherer*" # Openfind data gatherer ROBOTINCLUDE "Openfind data gatherer@Openbot" # Openfind data gatherer ROBOTINCLUDE "OpenTextSiteCrawler*" ROBOTINCLUDE "Orbsearch/*" # Orb Search ROBOTINCLUDE "Orbsearch/1.0" # Orb Search ROBOTINCLUDE "PackRat/*" # Pack Rat ROBOTINCLUDE "PackRat/1.0" # Pack Rat ROBOTINCLUDE "PageBoy/*" # PageBoy ROBOTINCLUDE "PageBoy/1.0" # PageBoy ROBOTINCLUDE "parallelContextFocusCrawler1.1" ROBOTINCLUDE "ParaSite/*" # ParaSite (www.ianett.com) ROBOTINCLUDE "ParaSite/0.21 (http://www.ianett.com/parasite/)" # ParaSite (www.ianett.com) ROBOTINCLUDE "Patric/*" # Patric ROBOTINCLUDE "Patric/0.01a" # Patric ROBOTINCLUDE "*PBWF*" ROBOTINCLUDE "Peregrinator-Mathematics/*" # The Peregrinator ROBOTINCLUDE "Peregrinator-Mathematics/0.7" # The Peregrinator ROBOTINCLUDE "PerlCrawler/1.0 Xavatoria/2.0" # PerlCrawler 1.0 ROBOTINCLUDE "PGP-KA/*" # PGP Key Agent ROBOTINCLUDE "PGP-KA/1.2" # PGP Key Agent ROBOTINCLUDE "phpdig/x.x.x*" # PhpDig ROBOTINCLUDE "PicoSearch*" ROBOTINCLUDE "PiltdownMan/*" ROBOTINCLUDE "PiltdownMan/1.0 profitnet@myezmail.com" # PiltdownMan ROBOTINCLUDE "*Pimptrain's robot*" ROBOTINCLUDE "PingALink*" ROBOTINCLUDE "Pioneer*" # Pioneer ROBOTINCLUDE "PJspider*" ROBOTINCLUDE "PlantyNet_WebRobot*" ROBOTINCLUDE "PlumtreeWebAccessor/*" # PlumtreeWebAccessor ROBOTINCLUDE "PlumtreeWebAccessor/0.9" # PlumtreeWebAccessor ROBOTINCLUDE "Pompos*" ROBOTINCLUDE "Poppi/*" # Poppi ROBOTINCLUDE "Poppi/1.0" # Poppi ROBOTINCLUDE "PortalBSpider/1.0 (spider@portalb.com)" # PortalB Spider ROBOTINCLUDE "PortalJuice.com/*" # Portal Juice Spider ROBOTINCLUDE "PortalJuice.com/4.0" # Portal Juice Spider ROBOTINCLUDE "processor/0.0ALPHA libwww-perl/0.20" ROBOTINCLUDE "psbot/*" # psbot picsearch ROBOTINCLUDE "psbot/0.X (+http://www.picsearch.com/bot.html)" # psbot picsearch ROBOTINCLUDE "PycURL/*" ROBOTINCLUDE "PycURL/7.15.5" ROBOTINCLUDE "QueryN?Metasearch*" ROBOTINCLUDE "*QXW03018)*" # wespe.de ROBOTINCLUDE "*ramBot*" ROBOTINCLUDE "Raven-v2*" # Raven Search ROBOTINCLUDE "RepoMonkey Bait & Tackle" ROBOTINCLUDE "Resume Robot*" # Resume Robot ROBOTINCLUDE "Resume Robot" # Resume Robot ROBOTINCLUDE "RHCS/*" # RoadHouse Crawling System ROBOTINCLUDE "RHCS/1.0a" # RoadHouse Crawling System ROBOTINCLUDE "RixBot (http://www.oops-as.no/rix/)" # RixBot ROBOTINCLUDE "Road Runner: ImageScape Robot (lim@cs.leidenuniv.nl)*" # Road Runner: The ImageScape Robot ROBOTINCLUDE "Robbie/*" # Robbie the Robot ROBOTINCLUDE "Robbie/0.1" # Robbie the Robot ROBOTINCLUDE "Robo Crawler*" ROBOTINCLUDE "RoboCrawl (http://www.canadiancontent.net)*" ROBOTINCLUDE "Robofox v*" # RoboCrawl Spider ROBOTINCLUDE "Robofox v2.0" # RoboFox ROBOTINCLUDE "Robot du CRIM*" # Robot Francoroute ROBOTINCLUDE "Robot du CRIM 1.0a" # Robot Francoroute ROBOTINCLUDE "Robot@SuperSnooper.Com*" ROBOTINCLUDE "Robozilla/*" # Robozilla ROBOTINCLUDE "Robozilla/1.0" # Robozilla ROBOTINCLUDE "root/*" # Web Core / Roots ROBOTINCLUDE "root/0.1" # Web Core / Roots ROBOTINCLUDE "Rotondo/*" ROBOTINCLUDE "Roverbot*" # Roverbot ROBOTINCLUDE "RRC*" ROBOTINCLUDE "RuLeS/*" # RuLeS ROBOTINCLUDE "RuLeS/1.0 libwww/4.0" # RuLeS ROBOTINCLUDE "SafetyNet Robot*" # SafetyNet Robot ROBOTINCLUDE "SafetyNet Robot 0.1*" # SafetyNet Robot ROBOTINCLUDE "SandBot" ROBOTINCLUDE "SBIder*" # SiteSell.com ROBOTINCLUDE "Scooter*" # Altavista ROBOTINCLUDE "Scooter/2.0 G.R.A.B. V1.1.0" # Scooter ROBOTINCLUDE "ScoutAbout*" # www.about.com ROBOTINCLUDE "*search.ch?V*" ROBOTINCLUDE "SearchExpress?Spider*" ROBOTINCLUDE "searchprocess/*" # SearchProcess ROBOTINCLUDE "searchprocess/0.9" # SearchProcess ROBOTINCLUDE "SearchSpider.com*" ROBOTINCLUDE "Seekbot*" ROBOTINCLUDE "Seekbot/1*" ROBOTINCLUDE "Senrigan/xxxxxx*" # Senrigan ROBOTINCLUDE "SG-Scout*" # SG-Scout ROBOTINCLUDE "Shagseeker at http://www.shagseek.com /*" # ShagSeeker ROBOTINCLUDE "Shagseeker at http://www.shagseek.com /1.0" # ShagSeeker ROBOTINCLUDE "Shai'Hulud*" # Shai'Hulud ROBOTINCLUDE "Shai'Hulud" # Shai'Hulud ROBOTINCLUDE "*sharp-info-agent v*" ROBOTINCLUDE "SimBot/*" # Simmany Robot Ver1.0 ROBOTINCLUDE "SimBot/1.0" # Simmany Robot Ver1.0 ROBOTINCLUDE "Site Valet*" # Site Valet ROBOTINCLUDE "*Site?Server?3.??Robot*" ROBOTINCLUDE "sitecheck.internetseer.com" # InternetSeer ROBOTINCLUDE "SiteTech-Rover*" # SiteTech-Rover ROBOTINCLUDE "Slarp/*" ROBOTINCLUDE "SLCrawler" # SLCrawler ROBOTINCLUDE "Slider_Search*" ROBOTINCLUDE "SlowBot" ROBOTINCLUDE "*Slurp*" # Hotbot-Lycos-Yahoo ROBOTINCLUDE "Slurp/2.0" # Inktomi Slurp ROBOTINCLUDE "SlySearch (slysearch@slysearch.com)" ROBOTINCLUDE "SlySearch?(*" ROBOTINCLUDE "Snooper/b97_01" # Snooper ROBOTINCLUDE "Sohu*" # the Sohu Agent ... Sohu Spider ... Asian Search Robot ROBOTINCLUDE "Solbot/*" # Solbot ROBOTINCLUDE "Solbot/1.0 LWP/5.07" # Solbot ROBOTINCLUDE "SpaceBison" ROBOTINCLUDE "Speedy Spider ( http://www.entireweb.com/speedy.html )" # Speedy Spider ROBOTINCLUDE "spider.yellopet*" ROBOTINCLUDE "spider.yellopet.com www.yellopet.com" ROBOTINCLUDE "SpiderBot/1.0" # SpiderBot ROBOTINCLUDE "SpiderKU/*" ROBOTINCLUDE "spiderline/3.1.3" # Spiderline Crawler ROBOTINCLUDE "SpiderMan 1.0" # SpiderMan ROBOTINCLUDE "Spinne/*" ROBOTINCLUDE "spyder3.microsys.com" ROBOTINCLUDE "Sqworm/*" ROBOTINCLUDE "ssearcher100*" # Site Searcher ROBOTINCLUDE "StatFetcher/1.0" ROBOTINCLUDE "straight FLASH!! GetterroboPlus*" # GetterroboPlus Puu ROBOTINCLUDE "straight FLASH!! GetterroboPlus 1.5" # GetterroboPlus Puu ROBOTINCLUDE "suke/*.*" # Suke ROBOTINCLUDE "SummyCrawler" ROBOTINCLUDE "suntek/*" # suntek search engine ROBOTINCLUDE "suntek/1.0" # suntek search engine ROBOTINCLUDE "Surfbot" ROBOTINCLUDE "SurferF3*" ROBOTINCLUDE "Surfnomore Spider v1.1" ROBOTINCLUDE "swbot*" ROBOTINCLUDE "SwishSpider" ROBOTINCLUDE "SyncBot*" ROBOTINCLUDE "Szukacz/1*" ROBOTINCLUDE "Tarantula/*" # Tarantula ROBOTINCLUDE "Tarantula/1.0" # Tarantula ROBOTINCLUDE "tarspider" # tarspider ROBOTINCLUDE "TeacherSoft/1.0 libwww/2.17" ROBOTINCLUDE "TechBOT*" # TechBOT ROBOTINCLUDE "TechBOT" # TechBOT ROBOTINCLUDE "Templeton/*" # Templeton ROBOTINCLUDE "teomaagent*" # Teoma ROBOTINCLUDE "teomaagent crawler admin@teoma.com" # Teoma ROBOTINCLUDE "The Intraformant" ROBOTINCLUDE "*T-H-U-N-D-E-R-S-T-O-N-E*" ROBOTINCLUDE "TITAN/*" # TITAN ROBOTINCLUDE "TITAN/0.1" # TITAN ROBOTINCLUDE "TitIn/*" # TitIn ROBOTINCLUDE "TitIn/0.2" # TitIn ROBOTINCLUDE "tivraSpider*" # www.tivra.com ROBOTINCLUDE "TJG/Spider*" ROBOTINCLUDE "TLSpider/1.1" # TLSpider ROBOTINCLUDE "Toutatis*" ROBOTINCLUDE "Toutatis 2.5 2 (http:*" ROBOTINCLUDE "Trampelpfad*" # www.trampelpfad.ch Webkatalog ROBOTINCLUDE "*TrueRobot*" ROBOTINCLUDE "TurnitinBot/*" ROBOTINCLUDE "TV33_Mercator_1 1.0" ROBOTINCLUDE "UCSD-Crawler" # UCSD Crawl ROBOTINCLUDE "UdmSearch*" # UdmSearch, mnoGoSearch search engine software ROBOTINCLUDE "UdmSearch/2.1.1" # UdmSearch ROBOTINCLUDE "UIowaCrawler" ROBOTINCLUDE "Ultraseek*" # Ultraseek ROBOTINCLUDE "uptimebot*" # UptimeBot ROBOTINCLUDE "URL Indexer" ROBOTINCLUDE "URL Spider Pro" # URL Spider Pro ROBOTINCLUDE "urlck/*" # URL Check ROBOTINCLUDE "urlck/1.2.3" # URL Check ROBOTINCLUDE "*URL-Spider*" ROBOTINCLUDE "User-Agent: Mozilla/*" ROBOTINCLUDE "User-Agent: Mozilla/4.0 (compatible; sharp-info-agent v1.0; )" # Internet Shinchakubin ROBOTINCLUDE "*usww.com-Spider-*" ROBOTINCLUDE "*Vagabondo*" ROBOTINCLUDE "Valkyrie/*" # Valkyrie ROBOTINCLUDE "Valkyrie/1.0 libwww-perl/0.40" # Valkyrie ROBOTINCLUDE "Verticrawl*" # Verticrawl ROBOTINCLUDE "Verticrawlbot*" ROBOTINCLUDE "verzamelgids.nl*" ROBOTINCLUDE "Victoria/*" # Victoria ROBOTINCLUDE "Victoria/1.0" # Victoria ROBOTINCLUDE "vision-search/*" # vision-search ROBOTINCLUDE "vision-search/3.0'" # vision-search ROBOTINCLUDE "void-bot/*" # void-bot ROBOTINCLUDE "void-bot/0.1 (bot@void.be; http://www.void.be/)" # void-bot ROBOTINCLUDE "*VoilaBot*" ROBOTINCLUDE "Voyager/*" # Voyager ROBOTINCLUDE "Voyager/0.0" # Voyager ROBOTINCLUDE "VWbot_K/*" # VWbot ROBOTINCLUDE "VWbot_K/4.2" # VWbot ROBOTINCLUDE "w@pSpider/xxx (unix) by wap4.com" # w@pSpider by wap4.com ROBOTINCLUDE "W3C_Validator*" # W3C HTML-Code Validator ROBOTINCLUDE "w3index*" # The NWI Robot ROBOTINCLUDE "W3M2/x.xxx*" # W3M2 ROBOTINCLUDE "w3mir*" # w3mir ROBOTINCLUDE "web robot PEGASUS*" # pegasus ROBOTINCLUDE "Web?Link?Validator*" ROBOTINCLUDE "WebBandit/*" # WebBandit Web Spider ROBOTINCLUDE "WebBandit/1.0" # WebBandit Web Spider ROBOTINCLUDE "WebCatcher/*" # WebCatcher ROBOTINCLUDE "WebCatcher/1.0" # WebCatcher ROBOTINCLUDE "WebCopy/(version)" # WebCopy ROBOTINCLUDE "WebCopy/(version*)" # WebCopy ROBOTINCLUDE "WebFetcher/*" # webfetcher ROBOTINCLUDE "WebFetcher/0.8*" # webfetcher ROBOTINCLUDE "*Webinator*" ROBOTINCLUDE "weblayers/*" # weblayers ROBOTINCLUDE "weblayers/0.0" # weblayers ROBOTINCLUDE "WebLinker/*" # WebLinker ROBOTINCLUDE "WebLinker/0.0 libwww-perl/0.1" # WebLinker ROBOTINCLUDE "WebMoose/*" # The Web Moose ROBOTINCLUDE "WebMoose/0.0.0000" # The Web Moose ROBOTINCLUDE "WebQuest/*" # WebQuest ROBOTINCLUDE "WebQuest/1.0" # WebQuest ROBOTINCLUDE "WebReaper [webreaper@otway.com]*" ROBOTINCLUDE "webs@recruit.co.jp*" ROBOTINCLUDE "WebSauger 1.20b" ROBOTINCLUDE "Webspinne/*" ROBOTINCLUDE "webvac/*" # WebVac ROBOTINCLUDE "webvac/1.0" # WebVac ROBOTINCLUDE "webwalk*" # webwalk ROBOTINCLUDE "WebWalker/*" # WebWalker ROBOTINCLUDE "WebWalker/1.10" # WebWalker ROBOTINCLUDE "WebWatch*" ROBOTINCLUDE "Wget/*" # Wget ROBOTINCLUDE "Wget/?.*" # Wget ROBOTINCLUDE "Wget/1.4.0" # Wget ROBOTINCLUDE "whatUseek_winona*" # whatUseek Winona ROBOTINCLUDE "whatUseek_winona/3.0" # whatUseek Winona ROBOTINCLUDE "WhoWhere Robot" ROBOTINCLUDE "WIBBLE WOBBLE" ROBOTINCLUDE "Willow?Internet?Crawler" ROBOTINCLUDE "wired-digital-newsbot/*" # Wired Digital ROBOTINCLUDE "wired-digital-newsbot/1.5" # Wired Digital ROBOTINCLUDE "WISEbot*" # www.koreawisenut.com ROBOTINCLUDE "WiseWire*" # Wisewire.com domain checker ROBOTINCLUDE "wlm-*" ROBOTINCLUDE "wlm-1.1" # Weblog Monitor ROBOTINCLUDE "wobot/1.0 from 206.214.202.45" ROBOTINCLUDE "WOLP/*" # WebStolperer ROBOTINCLUDE "WOLP/1.0 mda/1.0" # WebStolperer ROBOTINCLUDE "Wusage/*" ROBOTINCLUDE "WWW Collector" ROBOTINCLUDE "www.freeloader.com." ROBOTINCLUDE "*www.galaxy.com;?www*" ROBOTINCLUDE "WWWC/*" # WWWC Other ROBOTINCLUDE "WWWC/0.25 (Win95)" # WWWC Ver 0.2.5 ROBOTINCLUDE "WWWWanderer v*" # the World Wide Web Wanderer ROBOTINCLUDE "Xenu*" # Xenu Link Sleuth ROBOTINCLUDE "XGET/*" # XGET ROBOTINCLUDE "xyro_(xcrawler*" ROBOTINCLUDE "yes*" ROBOTINCLUDE "ZBot/*" ROBOTINCLUDE "Zeus ThemeSite Viewer Webster Pro V2.9 Win32" ROBOTINCLUDE "Ziggy The Clown From Hell!!" ROBOTINCLUDE "Zippy?v2.*" ROBOTINCLUDE "*ZyBorg*" # wisenut.com