twister: extend reason field in Twister reports

scripts/ci/twister_report_analyzer.py

@@ -85,22 +85,22 @@ def add_counter(self, key: str, test: str = '') -> None:
     def print_counters(self, indent: int = 0):
         for key, value in self.counters.items():
             print(f'{" " * indent}{value.quantity:4}    {key}')
-            if value.subcounters.counters:
+            if value.has_subcounters():
                 value.subcounters.print_counters(indent + 4)
 
     def sort_by_quantity(self):
         self.counters = dict(
             sorted(self.counters.items(), key=lambda item: item[1].quantity, reverse=True)
         )
         for value in self.counters.values():
-            if value.subcounters.counters:
+            if value.has_subcounters():
                 value.subcounters.sort_by_quantity()
 
     def get_next_entry(self, depth: int = 0, max_depth: int = 10):
         for key, value in self.counters.items():
             # limit number of test files to 100 to not exceed CSV cell limit
             yield depth, value.quantity, key, ', '.join(value.tests[0:100])
-            if value.subcounters.counters and depth < max_depth:
+            if value.has_subcounters() and depth < max_depth:
                 yield from value.subcounters.get_next_entry(depth + 1, max_depth)
 
     def _flatten(self):
@@ -110,7 +110,7 @@ def _flatten(self):
         do not contain any further nested subcounters.
         """
         for key, value in self.counters.items():
-            if value.subcounters.counters:
+            if value.has_subcounters():
                 yield from value.subcounters._flatten()
             else:
                 yield key, value
@@ -130,6 +130,9 @@ def append(self, test: str = ''):
         if test:
             self.tests.append(test)
 
+    def has_subcounters(self):
+        return bool(self.subcounters.counters)
+
 
 class TwisterReports:
     def __init__(self):
@@ -161,41 +164,44 @@ def parse_testsuite(self, testsuite):
         if ts_status not in ('error', 'failed'):
             return
 
-        ts_reason = testsuite.get('reason') or 'Unknown reason'
-        self.errors.add_counter(ts_reason)
         ts_platform = testsuite.get('platform') or 'Unknown platform'
         self.platforms.add_counter(ts_platform)
+        ts_reason = testsuite.get('reason') or 'Unknown reason'
         ts_log = testsuite.get('log')
         test_identifier = f'{testsuite.get("platform")}:{testsuite.get("name")}'
 
-        matched = self._parse_ts_error_log(
-            self.errors.counters[ts_reason].subcounters, ts_reason, ts_log, test_identifier
-        )
+        # CMake and Build failures are treated separately.
+        # Extract detailed information to group errors. Keep the parsing methods
+        # to allow for further customization and keep backward compatibility.
+        if ts_reason.startswith('CMake build failure'):
+            reason = 'CMake build failure'
+            self.errors.add_counter(reason)
+            error_key = ts_reason.split(reason, 1)[-1].lstrip(' -')
+            if not error_key:
+                error_key = self._parse_cmake_build_failure(ts_log)
+            self.errors.counters[reason].subcounters.add_counter(error_key, test_identifier)
+            ts_reason = reason
+        elif ts_reason.startswith('Build failure'):
+            reason = 'Build failure'
+            self.errors.add_counter(reason)
+            error_key = ts_reason.split(reason, 1)[-1].lstrip(' -')
+            if not error_key:
+                error_key = self._parse_build_failure(ts_log)
+            self.errors.counters[reason].subcounters.add_counter(error_key, test_identifier)
+            ts_reason = reason
+        else:
+            self.errors.add_counter(ts_reason)
 
         # Process testcases
         for tc in testsuite.get('testcases', []):
             tc_reason = tc.get('reason')
             tc_log = tc.get('log')
             if tc_reason and tc_log:
                 self.errors.counters[ts_reason].subcounters.add_counter(tc_reason, test_identifier)
-                matched = True
 
-        if not matched:
+        if not self.errors.counters[ts_reason].has_subcounters():
             self.errors.counters[ts_reason].tests.append(test_identifier)
 
-    def _parse_ts_error_log(
-        self, counters: Counters, reason: str, log: str, test: str = ''
-    ) -> bool:
-        if reason == 'CMake build failure':
-            if error_key := self._parse_cmake_build_failure(log):
-                counters.add_counter(error_key, test)
-                return True
-        elif reason == 'Build failure':  # noqa SIM102
-            if error_key := self._parse_build_failure(log):
-                counters.add_counter(error_key, test)
-                return True
-        return False
-
     def _parse_cmake_build_failure(self, log: str) -> str | None:
         last_warning = 'no warning found'
         lines = log.splitlines()
@@ -263,32 +269,23 @@ def parse_testsuite(self, testsuite):
         if ts_status not in ('error', 'failed'):
             return
 
-        ts_reason = testsuite.get('reason') or 'Unknown reason'
-        self.errors.add_counter(ts_reason)
         ts_log = testsuite.get('log')
         test_identifier = f'{testsuite.get("platform")}:{testsuite.get("name")}'
-        self._parse_log_with_error_paterns(
-            self.errors.counters[ts_reason].subcounters, ts_log, test_identifier
-        )
+        if key := self._parse_log_with_error_paterns(ts_log):
+            self.errors.add_counter(key, test_identifier)
         # Process testcases
         for tc in testsuite.get('testcases', []):
-            tc_reason = tc.get('reason')
             tc_log = tc.get('log')
-            if tc_reason and tc_log:
-                self.errors.counters[ts_reason].subcounters.add_counter(tc_reason)
-                self._parse_log_with_error_paterns(
-                    self.errors.counters[ts_reason].subcounters.counters[tc_reason].subcounters,
-                    tc_log,
-                    test_identifier,
-                )
-
-    def _parse_log_with_error_paterns(self, counters: Counters, log: str, test: str = ''):
+            if tc_log and (key := self._parse_log_with_error_paterns(tc_log)):
+                self.errors.add_counter(key, test_identifier)
+
+    def _parse_log_with_error_paterns(self, log: str) -> str | None:
         for line in log.splitlines():
             for error_pattern in self.error_patterns:
                 if error_pattern in line:
                     logger.debug(f'Matched: {error_pattern} in {line}')
-                    counters.add_counter(error_pattern, test)
-                    return
+                    return error_pattern
+        return None
 
 
 class EnhancedJSONEncoder(json.JSONEncoder):
@@ -363,7 +360,7 @@ def main():
     if not reports.errors.counters:
         return
 
-    if args.platforms:
+    if args.platforms and reports.platforms.counters:
         print('\nErrors per platform:')
         reports.platforms.print_counters()
 

scripts/pylib/twister/twisterlib/reports.py

@@ -372,7 +372,6 @@ def json_report(self, filename, version="NA", platform=None, filters=None):
                 suite["available_rom"] = available_rom
             if instance.status in [TwisterStatus.ERROR, TwisterStatus.FAIL]:
                 suite['status'] = instance.status
-                suite["reason"] = instance.reason
                 # FIXME
                 if os.path.exists(pytest_log):
                     suite["log"] = self.process_log(pytest_log)
@@ -382,6 +381,11 @@ def json_report(self, filename, version="NA", platform=None, filters=None):
                     suite["log"] = self.process_log(device_log)
                 else:
                     suite["log"] = self.process_log(build_log)
+
+                suite["reason"] = self.get_detailed_reason(instance.reason, suite["log"])
+                # update the reason to get more details also in other reports (e.g. junit)
+                # where build log is not available
+                instance.reason = suite["reason"]
             elif instance.status == TwisterStatus.FILTER:
                 suite["status"] = TwisterStatus.FILTER
                 suite["reason"] = instance.reason
@@ -798,3 +802,55 @@ def target_report(self, json_file, outdir, suffix):
                 self.json_report(json_platform_file + "_footprint.json",
                                  version=self.env.version, platform=platform.name,
                                  filters=self.json_filters['footprint.json'])
+
+    def get_detailed_reason(self, reason: str, log: str) -> str:
+        if reason == 'CMake build failure':
+            if error_key := self._parse_cmake_build_failure(log):
+                return f"{reason} - {error_key}"
+        elif reason == 'Build failure':  # noqa SIM102
+            if error_key := self._parse_build_failure(log):
+                return f"{reason} - {error_key}"
+        return reason
+
+    @staticmethod
+    def _parse_cmake_build_failure(log: str) -> str | None:
+        last_warning = 'no warning found'
+        lines = log.splitlines()
+        for i, line in enumerate(lines):
+            if "warning: " in line:
+                last_warning = line
+            elif "devicetree error: " in line:
+                return "devicetree error"
+            elif "fatal error: " in line:
+                return line[line.index('fatal error: ') :].strip()
+            elif "error: " in line:  # error: Aborting due to Kconfig warnings
+                if "undefined symbol" in last_warning:
+                    return last_warning[last_warning.index('undefined symbol') :].strip()
+                return last_warning
+            elif "CMake Error at" in line:
+                for next_line in lines[i + 1 :]:
+                    if next_line.strip():
+                        return line + ' ' + next_line
+                return line
+        return None
+
+    @staticmethod
+    def _parse_build_failure(log: str) -> str | None:
+        last_warning = ''
+        lines = log.splitlines()
+        for i, line in enumerate(lines):
+            if "undefined reference" in line:
+                return line[line.index('undefined reference') :].strip()
+            elif "error: ld returned" in line:
+                if last_warning:
+                    return last_warning
+                elif "overflowed by" in lines[i - 1]:
+                    return "ld.bfd: region overflowed"
+                elif "ld.bfd: warning: " in lines[i - 1]:
+                    return "ld.bfd:" + lines[i - 1].split("ld.bfd:", 1)[-1]
+                return line
+            elif "error: " in line:
+                return line[line.index('error: ') :].strip()
+            elif ": in function " in line:
+                last_warning = line[line.index('in function') :].strip()
+        return None

scripts/tests/twister_blackbox/test_report.py

@@ -114,8 +114,8 @@ class TestReport:
             os.path.join(TEST_DATA, 'tests', 'one_fail_two_error_one_pass'),
             ['qemu_x86/atom'],
             [r'one_fail_two_error_one_pass.agnostic.group1.subgroup2 on qemu_x86/atom FAILED \(.*\)',
-            r'one_fail_two_error_one_pass.agnostic.group1.subgroup3 on qemu_x86/atom ERROR \(Build failure\)',
-            r'one_fail_two_error_one_pass.agnostic.group1.subgroup4 on qemu_x86/atom ERROR \(Build failure\)'],
+            r'one_fail_two_error_one_pass.agnostic.group1.subgroup3 on qemu_x86/atom ERROR \(Build failure.*\)',
+            r'one_fail_two_error_one_pass.agnostic.group1.subgroup4 on qemu_x86/atom ERROR \(Build failure.*\)'],
         )
     ]