-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathDB_bot.go
120 lines (102 loc) · 4.6 KB
/
DB_bot.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
// DO NOT EDIT THIS FILE. IT IS AUTO-GENERATED BY "gen/gen-db.go". //
/*
* Copyright (c) 2018 wellwell.work, LLC by Zoe
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package surferua
func init() {
botDBSize = 16
botDB = []string{
"ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)",
"Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",
"Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",
"Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)",
"DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)",
"Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Exabot-Thumbnails)",
"facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)",
"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
"Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php)",
"msnbot/2.0b (+http://search.msn.com/msnbot.htm)",
"Mozilla/5.0 (compatible; ScoutJet; +http://www.scoutjet.com/)",
"SimplePie/1.3.1 (Feed Parser; http://simplepie.org; Allow like Gecko)",
"Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)",
"Sogou Orion spider/3.0( http://www.sogou.com/docs/help/webmasters.htm#07)",
"Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)",
"Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)",
}
}
// NewBotAlexa returns full user-agent string of alexa.
func NewBotAlexa() string {
return "ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)"
}
// NewBotBaidu returns full user-agent string of baidu.
func NewBotBaidu() string {
return "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"
}
// NewBotBing returns full user-agent string of bing.
func NewBotBing() string {
return "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)"
}
// NewBotBlex returns full user-agent string of blex.
func NewBotBlex() string {
return "Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)"
}
// NewBotDuck returns full user-agent string of duck.
func NewBotDuck() string {
return "DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)"
}
// NewBotExa returns full user-agent string of exa.
func NewBotExa() string {
return "Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Exabot-Thumbnails)"
}
// NewBotFacebook returns full user-agent string of facebook.
func NewBotFacebook() string {
return "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)"
}
// NewBotGoogle returns full user-agent string of google.
func NewBotGoogle() string {
return "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
}
// NewBotMj12 returns full user-agent string of mj12.
func NewBotMj12() string {
return "Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php)"
}
// NewBotMsn returns full user-agent string of msn.
func NewBotMsn() string {
return "msnbot/2.0b (+http://search.msn.com/msnbot.htm)"
}
// NewBotScoutJet returns full user-agent string of scoutJet.
func NewBotScoutJet() string {
return "Mozilla/5.0 (compatible; ScoutJet; +http://www.scoutjet.com/)"
}
// NewBotSimplePie returns full user-agent string of simplePie.
func NewBotSimplePie() string {
return "SimplePie/1.3.1 (Feed Parser; http://simplepie.org; Allow like Gecko)"
}
// NewBotSogou returns full user-agent string of sogou.
func NewBotSogou() string {
return "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)"
}
// NewBotSogouOrin returns full user-agent string of sogouOrin.
func NewBotSogouOrin() string {
return "Sogou Orion spider/3.0( http://www.sogou.com/docs/help/webmasters.htm#07)"
}
// NewBotYahoo returns full user-agent string of yahoo.
func NewBotYahoo() string {
return "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
}
// NewBotYandex returns full user-agent string of yandex.
func NewBotYandex() string {
return "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)"
}