forked from fnando/browser
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathbots.yml
143 lines (143 loc) · 4.43 KB
/
bots.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
AdsBot-Google: "Google Adwords"
AhrefsBot: "Ahrefs backlinks research tool"
alexa: "Alexa Crawler"
Apache-HttpClient: "Java http library"
ApacheBench: "ApacheBench (ab)"
AppEngine-Google: "Google App Engine"
AsyncHttpClient: "Java http and WebSocket client library"
awe.sm: "Awe.sm URL expander"
Baidu: "Chinese search engine"
Baiduspider: "Chinese search engine"
bingbot: "Microsoft bing bot"
bitlybot: "bit.ly bot"
[email protected]: "Linkfluence bot"
BuiBui-CheckBot: "email: buibui[at]dadapro[dot]com"
Butterfly: "Topsy Labs"
buzztalk: "buzztalk"
check_http: "Nagios monitor"
CMRadar/0.1: "CMRadar/0.1"
ColdFusion: "ColdFusion http library"
Crowsnest: "Crowsnest"
Crowsnest: "crowsnest.tv news portal bot"
curl: "curl unix CLI http client"
developers.google.com/+/web/snippet/: "Google Plus"
DigitalPersona Fingerprint Software: "HP Fingerprint scanner"
DotBot: "Dot Bot"
Embedly: "Embedly"
EOAAgent: "EOAAgent"
EventMachine HttpClient: "Ruby http library"
EvriNid: "Evri bot"
Exabot: "Exalead's bot"
ExaleadCloudView: "ExaleadCloudView"
facebookexternalhit: "facebook bot"
FeedBurner: "RSS bot"
Feedfetcher-Google: "Google Feedfetcher"
FlipboardProxy: "FlipboardProxy"
Genieo: "Genieo Web filter bot"
getprismatic.com: "getprismatic.com"
Gigabot: "Gigabot spider"
Go http package: "Go http library"
Google Web Preview: "Google Instant Previews crawler"
Googlebot: "Google spider"
Hatena::Bookmark: "Hatena::Bookmark"
heritrix: "heritrix"
HTMLParser: "HTMLParser"
HTTP_Request2: "HTTP_Request2"
HTTPClient: "HTTPClient"
https://developers.google.com/+/web/snippet: "Google+ Snippet Fetcher"
HubSpot Connect: "HubSpot Connect"
ia_archiver(OS-Wayback): "ia_archiver(OS-Wayback)"
iCoreService: "iCoreService"
InAGist: "URL resolver"
Insieve Bot: "Insieve Bot"
InsieveBot: "InsieveBot"
Instapaper: "Instapaper"
jack: "jack"
Jakarta Commons-HttpClient: "Jakarta Commons-HttpClient"
Jakarta Commons: "Jakarta Commons HttpClient"
Java: "Generic Java http library"
Jetslide: "Jetslide"
JS-Kit: "URL resolver"
Kimengi/nineconnections.com: "Kimengi/nineconnections.com"
Kimengi: "nineconnections.com bot"
knows.is: "knows.is"
kraken: "kraken"
Laconica: "Laconica"
libwww-perl: "Perl client-server library loved by script kids"
linkdexbot: "Linkdex Bot"
LinkedInBot: "LinkedIn"
LinksCrawler: "LinksCrawler"
Lipperhey: "Lipperhey"
Livelapbot: "Livelapbot"
LoadTimeBot: "Load Time Bot"
LongURL: "URL expander service"
Lumibot: "Lumibot"
lwp-trivial: "Another Perl library loved by script kids"
Mail.RU_Bot: "Mail.ru Bot"
Mediapartners-Google: "Google Adsense bot"
MetaURI: "MetaURI"
MetaURI: "URI meta information extractor bot"
MFE_expand: "Mcafee spider"
MJ12bot: "Majestic-12 spider"
MojeekBot: "Mojeek UK search crawler"
msnbot: "Microsoft bot"
Netvibes: "Personalized dashboard bot"
NewRelicPinger: "NewRelic monitor"
newsme: "Link analyzer"
newsme: "newsme"
NING: "NING - Yet Another Twitter Swarmer"
Nutch: "Apache search spider"
Panopta: "Monitoring service"
PaperLiBot: "PaperLi content curation service"
PaperLiBot: "PaperLi is another content curation service"
peerindex: "peerindex"
PercolateCrawler: "PercolateCrawler"
PhantomJS: "PhantomJS"
Pingdom: "Pingdom monitoring"
PycURL: "Python http library"
Python-httplib2: "Python-httplib2"
python-requests: "Python http library"
python-requests: "python-requests"
Python-urllib: "Python http library"
QuickLook: "QuickLook"
Readability: "Readability"
RebelMouse: "RebelMouse"
redditbot: "Reddit Bot"
Riddler: "Riddler Bot"
rogerbot: "SeoMoz spider"
Ruby/1.9.3: "Ruby/1.9.3"
Ruby: "Ruby"
Scrapy: "Scrapy"
SemrushBot: "SEO analysis bot"
ShortLinkTranslate: "Link shortener"
ShowyouBot: "Showyou iOS app spider"
Siege: "Joe Dog Siege"
SiteUptime: "Site monitoring services"
Slurp: "Yahoo spider"
Sogou: "Chinese search engine"
spider: "generic web spider"
Spinn3r: "Spinn3r aggregator"
Test Certificate Info: "C http library?"
trendictionbot: "trendiction search"
TweetmemeBot: "TweetMeMe Crawler"
Twikle: "Social web search bot"
TwitJobSearch: "TwitJobSearch"
Twitmunin: "Twitmunin"
Twitterbot: "Twitter URL expander"
Twurly: "Twurly"
Typhoeus: "Typhoeus"
UnwindFetch: "Gnip URL expander"
UnwindFetchor: "Gnip crawler"
VB Project: "Visual Basic"
vkShare: "VKontake Sharer"
Webscout: "Webscout"
Wget: "wget unix CLI http client"
WordPress: "WordPress spider"
Xenu Link Sleuth: "Xenu Link Sleuth"
YandexBot: "Yandex spider"
YandexDirect: "Yandex Direct spider"
YandexMetrika: "Yandex Metrika spider"
YOURLS: "YOURLS"
zelist.ro: "feed parser"
ZIBB: "ZIBB spider"
ZyBorg: "Zyborg? Hmmm...."