From e3565544842a22a3679b6ee61c75eb0295418f58 Mon Sep 17 00:00:00 2001 From: loadkpi Date: Mon, 4 Nov 2024 21:46:24 +0200 Subject: [PATCH] update update_raw_files; v1.2.121 --- bin/update_raw_files | 4 ++-- lib/crawler_detect/library/raw/Exclusions.json | 2 +- spec/fixtures/crawlers.txt | 5 +++-- spec/fixtures/devices.txt | 4 +++- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/bin/update_raw_files b/bin/update_raw_files index 1a013ad..5884b76 100755 --- a/bin/update_raw_files +++ b/bin/update_raw_files @@ -6,5 +6,5 @@ wget -O $DIR/../lib/crawler_detect/library/raw/Crawlers.json https://raw.githubu wget -O $DIR/../lib/crawler_detect/library/raw/Exclusions.json https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Exclusions.json wget -O $DIR/../lib/crawler_detect/library/raw/Headers.json https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Headers.json -wget -O $DIR/../spec/fixtures/crawlers.txt https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/tests/crawlers.txt -wget -O $DIR/../spec/fixtures/devices.txt https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/tests/devices.txt +wget -O $DIR/../spec/fixtures/crawlers.txt https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/tests/data/user_agent/crawlers.txt +wget -O $DIR/../spec/fixtures/devices.txt https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/tests/data/user_agent/devices.txt diff --git a/lib/crawler_detect/library/raw/Exclusions.json b/lib/crawler_detect/library/raw/Exclusions.json index e7e0141..bf4128b 100644 --- a/lib/crawler_detect/library/raw/Exclusions.json +++ b/lib/crawler_detect/library/raw/Exclusions.json @@ -1 +1 @@ -["Safari.[\\d\\.]*","Firefox.[\\d\\.]*"," Chrome.[\\d\\.]*","Chromium.[\\d\\.]*","MSIE.[\\d\\.]","Opera\\\/[\\d\\.]*","Mozilla.[\\d\\.]*","AppleWebKit.[\\d\\.]*","Trident.[\\d\\.]*","Windows NT.[\\d\\.]*","Android [\\d\\.]*","Macintosh.","Ubuntu","Linux","[ ]Intel","Mac OS X [\\d_]*","(like )?Gecko(.[\\d\\.]*)?","KHTML,","CriOS.[\\d\\.]*","CPU iPhone OS ([0-9_])* like Mac OS X","CPU OS ([0-9_])* like Mac OS X","iPod","compatible","x86_..","i686","x64","X11","rv:[\\d\\.]*","Version.[\\d\\.]*","WOW64","Win64","Dalvik.[\\d\\.]*"," \\.NET CLR [\\d\\.]*","Presto.[\\d\\.]*","Media Center PC","BlackBerry","Build","Opera Mini\\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\\/\\d{1,2}\\.","Opera"," \\.NET[\\d\\.]*","cubot","; M bot","; CRONO","; B bot","; IDbot","; ID bot","; POWER BOT","OCTOPUS-CORE","htc_botdugls","super\\\/\\d+\\\/Android\\\/\\d+"] \ No newline at end of file +["Safari.[\\d\\.]*","Firefox.[\\d\\.]*"," Chrome.[\\d\\.]*","Chromium.[\\d\\.]*","MSIE.[\\d\\.]","Opera\\\/[\\d\\.]*","Mozilla.[\\d\\.]*","AppleWebKit.[\\d\\.]*","Trident.[\\d\\.]*","Windows NT.[\\d\\.]*","Android [\\d\\.]*","Macintosh.","Ubuntu","Linux","[ ]Intel","Mac OS X [\\d_]*","(like )?Gecko(.[\\d\\.]*)?","KHTML,","CriOS.[\\d\\.]*","CPU iPhone OS ([0-9_])* like Mac OS X","CPU OS ([0-9_])* like Mac OS X","iPod","compatible","x86_..","i686","x64","X11","rv:[\\d\\.]*","Version.[\\d\\.]*","WOW64","Win64","Dalvik.[\\d\\.]*"," \\.NET CLR [\\d\\.]*","Presto.[\\d\\.]*","Media Center PC","BlackBerry","Build","Opera Mini\\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\\/\\d{1,2}\\.","Opera"," \\.NET[\\d\\.]*","cubot","; M bot","; CRONO","; B bot","; IDbot","; ID bot","; POWER BOT","OCTOPUS-CORE","htc_botdugls","super\\\/\\d+\\\/Android\\\/\\d+","\"Yandex\"","YandexModule2"] \ No newline at end of file diff --git a/spec/fixtures/crawlers.txt b/spec/fixtures/crawlers.txt index 92e3c06..72a77f8 100644 --- a/spec/fixtures/crawlers.txt +++ b/spec/fixtures/crawlers.txt @@ -1116,7 +1116,6 @@ ShowyouBot (http://showyou.com/crawler) Yoleo Consumer v0.2 Mozilla/5.0 (compatible; IstellaBot/1.10.2 +http://www.tiscali.it/) Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server; +http://ws.daum.net/aboutWebSearch.html) Daumoa/3.0 -Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 YaBrowser/15.12.0.6151 Safari/537.36 Yandex.Translate Mozilla/5.0 (compatible; TeeRaidBot; +https://teeraid.com/bot/) Mozilla/5.0 (compatible; Scopia Crawler 1.1; +http://www.scopia.co) CopperEgg/RevealUptime/DublinIE(aws) @@ -3674,4 +3673,6 @@ facebookcatalog/1.0 Verity/1.1 (gumgum.com/verity; verity-support@gumgum.com) Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GoogleOther) Chrome/117.0.5938.132 Safari/537.36 Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.5938.132 Mobile Safari/537.36 (compatible; GoogleOther) -Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.6261.94 Mobile Safari/537.36 (compatible; GoogleOther) \ No newline at end of file +Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.6261.94 Mobile Safari/537.36 (compatible; GoogleOther) +Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots) +Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots) \ No newline at end of file diff --git a/spec/fixtures/devices.txt b/spec/fixtures/devices.txt index 5d07f58..89ec1be 100644 --- a/spec/fixtures/devices.txt +++ b/spec/fixtures/devices.txt @@ -165632,4 +165632,6 @@ Mozilla/5.0 (iPhone; CPU iPhone OS 13_3_1 like Mac OS X) AppleWebKit/605.1.15 (K Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148/mvmkbtaqqdhshwcbsygjnuqcqmpndbrt Mozilla/5.0 (Linux; Android 5.0; SM-N9100 Build/LRX21V) > AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 > Chrome/37.0.0.0 Mobile Safari/537.36 > MicroMessenger/6.0.2.56_r958800.520 NetType/WIFI Mozilla/5.0 (Linux; U; Android 5.0; en-US; SM-N900S;iPhone: Build/LRX21V) -Mozilla/5.0 (Linux; Android 9; OCTOPUS-CORE T8 Build/PPR1.181005.003) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Mobile Safari/537.36 YaApp_Android/9.50/apad YaSearchBrowser/9.50 \ No newline at end of file +Mozilla/5.0 (Linux; Android 9; OCTOPUS-CORE T8 Build/PPR1.181005.003) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Mobile Safari/537.36 YaApp_Android/9.50/apad YaSearchBrowser/9.50 +Mozilla/5.0 (Linux; Android 9; YandexModule2-00001 Build/PI; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/113.0.5672.163 YaBrowser/24.1.2.86 (lite) TV Safari/537.36 +Mozilla/5.0 (Linux; Android 9; YandexModule2-00001 Build/PI; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/113.0.5672.163 Mobile Safari/537.36 \ No newline at end of file