diff --git a/html/main.js b/html/main.js
index be0af6f..4db6d40 100644
--- a/html/main.js
+++ b/html/main.js
@@ -26,8 +26,18 @@ const COMMON_ERROR_MESSAGE = {
'Tor Error: ': 'Tor Error'
};
-const SORT_CRITERIAS = ['http.status_code', 'error', 'timing.search.error', 'version', 'tls.grade',
- 'http.grade', 'html.grade', 'timing.search.all', 'url'];
+const SORT_CRITERIAS = [
+ 'http.status_code',
+ 'error',
+ 'timing.search.error',
+ 'timing.search_go.error',
+ 'version',
+ 'tls.grade',
+ 'http.grade',
+ 'html.grade',
+ 'timing.search.all',
+ 'url'
+];
const HTML_GRADE_MAPPING = {
'V': 3,
@@ -212,16 +222,18 @@ function getTime(timing) {
}
function isError(timing) {
- if (timing.success_percentage > 0) {
- return false;
+ if (timing.success_percentage < 100) {
+ return 100 - timing.success_percentage;
}
- return (timing.error !== undefined);
+ if (timing.error !== undefined) {
+ return 100;
+ }
+ return 0;
}
const CompareFunctionCriterias = {
'http.status_code': (a, b) => -compareTool(a, b, null, 'http', 'status_code'),
'error': (a, b) => -compareTool(a, b, null, 'error'),
- 'error_wp': (a, b) => compareTool(a, b, null, 'timing', 'search_wp', 'error'),
'network.asn_privacy': (a, b) => compareTool(a, b, null, 'network', 'asn_privacy'),
'version': (a, b) => compareVersion(a.version, b.version),
'tls.grade': (a, b) => compareTool(a, b, normalizeGrade, 'tls', 'grade'),
@@ -229,8 +241,8 @@ const CompareFunctionCriterias = {
'http.grade': (a, b) => compareTool(a, b, normalizeGrade, 'http', 'grade'),
'timing.initial.all': (a, b) => -compareTool(a, b, getTime, 'timing', 'initial', 'all'),
'timing.search.error': (a, b) => -compareTool(a, b, isError, 'timing', 'search'),
+ 'timing.search_go.error': (a, b) => -compareTool(a, b, isError, 'timing', 'search_go'),
'timing.search.all': (a, b) => -compareTool(a, b, getTime, 'timing', 'search', 'all'),
- 'timing.search_wp.server': (a, b) => -compareTool(a, b, getTime, 'timing', 'search_wp', 'server'),
'timing.search_wp.all': (a, b) => -compareTool(a, b, getTime, 'timing', 'search_wp', 'all'),
'url': (a, b) => -compareTool(a, b, null, 'url'),
};
diff --git a/searxstats/fetcher/timing.py b/searxstats/fetcher/timing.py
index 95ada31..dafe8bb 100644
--- a/searxstats/fetcher/timing.py
+++ b/searxstats/fetcher/timing.py
@@ -25,18 +25,44 @@ class CheckResult:
# There are results, error on the side (above infoboxes)
alter_danger_side: etree.XPath
+ @staticmethod
+ def is_wikiengine(engine_name):
+ return engine_name.startswith('wiki')
+
async def _check_html_result_page(self, engine_name, response):
+ result_count = 0
document = await html_fromstring(response.text)
- result_element_list = self.results(document)
- if len(result_element_list) == 0:
+ for result_element, engine_names in self._iter_meaningful_results(document):
+ engine_names = [
+ extract_text(engine_element)
+ for engine_element in self.engines(result_element)
+ ]
+ result_count += 1
+ if engine_name in engine_names:
+ continue
+ return False, f'A result is not from the {engine_name}'
+ if result_count == 0:
return False, 'No result'
- for result_element in result_element_list:
- for engine_element in self.engines(result_element):
- if extract_text(engine_element).find(engine_name) >= 0:
- continue
- return False, f'A result is not from the {engine_name}'
return True, None
+ def _get_meaningful_result_count(self, document):
+ return len(list(self._iter_meaningful_results(document)))
+
+ def _iter_meaningful_results(self, document):
+ """Iterator over meaningful results
+ = results from generic engines, not from wiki* engines
+ """
+ result_element_list = self.results(document)
+ for result_element in result_element_list:
+ engine_names = [
+ extract_text(engine_element)
+ for engine_element in self.engines(result_element)
+ ]
+ only_wiki_engine = all(map(self.is_wikiengine, engine_names))
+ if only_wiki_engine:
+ continue
+ yield result_element, engine_names
+
async def check_google_result(self, response):
return await self._check_html_result_page('google', response)
@@ -45,35 +71,32 @@ async def check_wikipedia_result(self, response):
async def check_search_result(self, response):
document = await html_fromstring(response.text)
+ message = None
result_element_list = self.results(document)
alert_danger_list = self.alert_danger_main(document)
if len(alert_danger_list) > 0:
- return True, extract_text(alert_danger_list)
- alert_danger_list = self.alter_danger_side(document)
- if len(alert_danger_list) > 0:
- return True, extract_text(alert_danger_list)
+ return False, extract_text(alert_danger_list)
+ alert_danger_side_list = self.alter_danger_side(document)
+ if len(alert_danger_side_list) > 0:
+ message = extract_text(alert_danger_side_list)
if len(result_element_list) == 0:
return False, 'No result'
+ if self._get_meaningful_result_count(document) == 0:
+ return False, 'Only result(s) from wiki* engines'
if len(result_element_list) == 1:
return False, 'Only one result'
if len(result_element_list) == 2:
return False, 'Only two results'
- return True, None
+ return True, message
CheckResultByTheme = {
'simple': CheckResult(
results=etree.XPath("//div[@id='urls']//article"),
- engines=etree.XPath("//div[contains(@class, 'engines')]/span"),
+ engines=etree.XPath(".//div[contains(@class, 'engines')]/span"),
alert_danger_main=etree.XPath("//div[@id='urls']/div[contains(@class, 'dialog-error')]"),
alter_danger_side=etree.XPath("//div[@id='sidebar']/div[contains(@class, 'dialog-error')]"),
),
- 'oscar': CheckResult(
- results=etree.XPath("//div[@id='main_results']/div[contains(@class,'result-default')]"),
- engines=etree.XPath("//span[contains(@class, 'label label-default')]"),
- alert_danger_main=etree.XPath("//div[contains(@class, 'alert-danger')]/p[2]"),
- alter_danger_side=etree.XPath("//div[contains(@class, 'alert-danger')]/text()"),
- ),
}
@@ -157,16 +180,16 @@ async def fetch_one(instance_url: str, detail) -> dict:
# /search instead of / : https://github.com/searx/searx/pull/1681
search_url = urljoin(instance_url, 'search')
- theme = 'simple' if detail['generator'] == 'searxng' else 'oscar'
+ theme = 'simple'
print(search_url, '(', theme, ')')
check_result = CheckResultByTheme[theme]
- default_params = {'theme': theme}
+ default_params = {'theme': 'simple'}
# check the default engines
print('🔎 ' + instance_url)
await request_stat_with_log(search_url, timing, 'search',
client, instance_url,
- 3, 120, 160, check_result.check_search_result,
+ 5, 120, 160, check_result.check_search_result,
params={'q': 'time', **default_params},
cookies=cookies, headers=DEFAULT_HEADERS)