Implements experiments to run and time traced and untraced versions of all binaries in a project

varats-core/varats/experiment/experiment_util.py

@@ -10,6 +10,7 @@
 from collections import defaultdict
 from pathlib import Path
 from types import TracebackType
+import yaml
 
 from benchbuild import source
 from benchbuild.experiment import Experiment

varats-core/varats/report/gnu_time_report.py

@@ -74,8 +74,6 @@ def __init__(self, path: Path) -> None:
                         TimeReport._parse_involuntary_ctx_switches(line)
                     continue
 
-                # print("Not matched: ", line)
-
     @property
     def command_name(self) -> str:
         """Name of the command that was executed."""
@@ -284,7 +282,7 @@ def summary(self) -> str:
 
 class WLTimeReportAggregate(
     WorkloadSpecificReportAggregate[TimeReport],
-    shorthand=TimeReport.SHORTHAND + ReportAggregate.SHORTHAND,
+    shorthand="WL" + TimeReport.SHORTHAND + ReportAggregate.SHORTHAND,
     file_type=ReportAggregate.FILE_TYPE
 ):
     """Context Manager for parsing multiple time reports stored inside a zip

varats-core/varats/report/report.py

@@ -655,7 +655,6 @@ def is_correct_report_type(cls, file_name: str) -> bool:
         except ValueError:
             return False
 
-
 class ReportSpecification():
     """Groups together multiple report types into a specification that can be
     used, e.g., by experiments, to request multiple reports."""

varats-core/varats/report/tef_report.py

@@ -18,21 +18,21 @@ class TraceEventType(Enum):
 
     value: str  # pylint: disable=invalid-name
 
-    DURATION_EVENT_BEGIN = 'B'
-    DURATION_EVENT_END = 'E'
-    COMPLETE_EVENT = 'X'
-    INSTANT_EVENT = 'i'
-    COUNTER_EVENT = 'C'
-    ASYNC_EVENT_START = 'b'
-    ASYNC_EVENT_INSTANT = 'n'
-    ASYNC_EVENT_END = 'e'
-    FLOW_EVENT_START = 's'
-    FLOW_EVENT_STEP = 't'
-    FLOW_EVENT_END = 'f'
-    SAMPLE_EVENT = 'P'
+    DURATION_EVENT_BEGIN = "B"
+    DURATION_EVENT_END = "E"
+    COMPLETE_EVENT = "X"
+    INSTANT_EVENT = "i"
+    COUNTER_EVENT = "C"
+    ASYNC_EVENT_START = "b"
+    ASYNC_EVENT_INSTANT = "n"
+    ASYNC_EVENT_END = "e"
+    FLOW_EVENT_START = "s"
+    FLOW_EVENT_STEP = "t"
+    FLOW_EVENT_END = "f"
+    SAMPLE_EVENT = "P"
 
     @staticmethod
-    def parse_event_type(raw_event_type: str) -> 'TraceEventType':
+    def parse_event_type(raw_event_type: str) -> "TraceEventType":
         """Parses a raw string that represents a trace-format event type and
         converts it to the corresponding enum value."""
         for trace_event_type in TraceEventType:
@@ -45,7 +45,7 @@ def __str__(self) -> str:
         return str(self.value)
 
 
-class TraceEvent():
+class TraceEvent:
     """Represents a trace event that was captured during the analysis of a
     target program."""
 
@@ -56,9 +56,7 @@ def __init__(
         self.__name_id_mapper = name_id_mapper
         self.__name_id = name_id
         self.__category = str(json_trace_event["cat"])
-        self.__event_type = TraceEventType.parse_event_type(
-            json_trace_event["ph"]
-        )
+        self.__event_type = TraceEventType.parse_event_type(json_trace_event["ph"])
         self.__tracing_clock_timestamp = int(json_trace_event["ts"])
         self.__pid = int(json_trace_event["pid"])
         self.__tid = int(json_trace_event["tid"])
@@ -128,9 +126,7 @@ def trace_events(self) -> tp.List[TraceEvent]:
 
     @property
     def stack_frames(self) -> None:
-        raise NotImplementedError(
-            "Stack frame parsing is currently not implemented!"
-        )
+        raise NotImplementedError("Stack frame parsing is currently not implemented!")
 
     def _parse_json(self) -> None:
         trace_events: tp.List[TraceEvent] = list()
@@ -172,7 +168,7 @@ def _parse_json(self) -> None:
 class TEFReportAggregate(
     ReportAggregate[TEFReport],
     shorthand=TEFReport.SHORTHAND + ReportAggregate.SHORTHAND,
-    file_type=ReportAggregate.FILE_TYPE
+    file_type=ReportAggregate.FILE_TYPE,
 ):
     """Context Manager for parsing multiple TEF reports stored inside a zip
     file."""

varats/varats/data/reports/dynamic_overhead_report.py

@@ -0,0 +1,53 @@
+from varats.report.report import BaseReport
+from pathlib import Path
+from collections import defaultdict
+
+
+class DynamicOverheadReport(
+    BaseReport, shorthand="DynOverhead", file_type="txt"
+):
+
+    class RegionCounter:
+
+        def __init__(self):
+            self.__in = 0
+            self.__out = 0
+
+        def enter(self):
+            self.__in += 1
+
+        def leave(self):
+            self.__out += 1
+
+        def isvalid(self):
+            return self.__in == self.__out
+
+        def count_visited(self):
+            return self.__in
+
+    def __init__(self, path: Path):
+        super().__init__(path)
+        self.__entries = defaultdict(DynamicOverheadReport.RegionCounter)
+
+        for line in open(path, "r"):
+            try:
+                command, id = line.split()
+                if command == "Entering":
+                    self.__entries[id].enter()
+                elif command == "Leaving":
+                    self.__entries[id].leave()
+            except ValueError:
+                continue
+
+        self.__total_region_count = 0
+
+        # Generate report
+        for region in self.__entries.values():
+            if region.isvalid():
+                self.__total_region_count += region.count_visited()
+
+    def isvalid(self) -> bool:
+        return all(v.isvalid() for v in self.__entries.values())
+
+    def regions_visited(self):
+        return self.__total_region_count

varats/varats/data/reports/instrumentation_verifier_report.py

@@ -3,6 +3,7 @@
 import typing as tp
 from pathlib import Path
 from zipfile import ZipFile
+import yaml
 
 from varats.report.report import BaseReport
 
@@ -15,10 +16,15 @@ def __init__(self, report_path: Path) -> None:
         super().__init__(report_path)
 
         self.__report_data = {}
+        self.__metadata = {}
 
         with ZipFile(report_path, "r") as archive:
 
             for file in archive.namelist():
+                if file == "metadata.yml":
+                    with archive.open(file, "r") as f:
+                        self.__metadata = yaml.load(f, yaml.Loader)
+
                 if not file.endswith(".ivr"):
                     continue
 
@@ -58,7 +64,9 @@ def __init__(self, report_path: Path) -> None:
                         unclosed_enter_begin = content.index(
                             'Unclosed Region-ID(s):'
                         ) + 1
-                        wrong_leaves = content[wrong_leaves_begin:-1]
+                        failure_begin = content.index("Finalization: Failure")
+                        wrong_leaves = content[
+                            wrong_leaves_begin:failure_begin - 1]
                         unclosed_regions = content[
                             unclosed_enter_begin:wrong_leaves_begin - 1]
 
@@ -137,9 +145,13 @@ def num_unentered_leaves(self, binary: str) -> int:
 
     def states(self) -> tp.Dict[str, str]:
         return {
-            binary: data['state']  # type: ignore
+            binary:
+                data['state']  # type: ignore
             for binary, data in self.__report_data.items()
         }
 
     def state(self, binary: str) -> str:
         return self.__report_data[binary]['state']  # type: ignore
+
+    def metadata(self) -> tp.Dict[tp.Any, tp.Any]:
+        return self.__metadata

varats/varats/experiments/base/time_workloads.py

@@ -1,6 +1,7 @@
 """Implements an experiment that times the execution of all project binaries."""
 
 import typing as tp
+import textwrap
 from pathlib import Path
 
 from benchbuild import Project
@@ -37,11 +38,16 @@ class TimeProjectWorkloads(OutputFolderStep):
     project: VProject
 
     def __init__(
-        self, project: Project, num: int, binary: ProjectBinaryWrapper
+        self,
+        project: Project,
+        num: int,
+        binary: ProjectBinaryWrapper,
+        categories: tp.List[WorkloadCategory] = [WorkloadCategory.EXAMPLE]
     ):
         super().__init__(project=project)
         self.__num = num
         self.__binary = binary
+        self.__workload_categories = categories
 
     def call_with_output_folder(self, tmp_dir: Path) -> actions.StepResult:
         return self.analyze(tmp_dir)
@@ -50,9 +56,11 @@ def analyze(self, tmp_dir: Path) -> actions.StepResult:
         """Only create a report file."""
 
         with local.cwd(self.project.builddir):
+            print(f"Step {self.__num}")
             for prj_command in workload_commands(
-                self.project, self.__binary, [WorkloadCategory.EXAMPLE]
+                self.project, self.__binary, self.__workload_categories
             ):
+                print("Running workload")
                 pb_cmd = prj_command.command.as_plumbum(project=self.project)
 
                 run_report_name = tmp_dir / create_workload_specific_filename(
@@ -61,11 +69,21 @@ def analyze(self, tmp_dir: Path) -> actions.StepResult:
 
                 run_cmd = time['-v', '-o', f'{run_report_name}', pb_cmd]
 
+                print("\t", run_cmd)
+
                 with cleanup(prj_command):
-                    run_cmd()
+                    run_cmd(retcode=self.__binary.valid_exit_codes)
+
+                print("Done")
 
         return actions.StepResult.OK
 
+    def __str__(self, indent: int = 0) -> str:
+        return textwrap.indent(
+            f"* Run workloads of categories {', '.join(str(x) for x in self.__workload_categories)} "
+            f"for binary {self.__binary.name} ({self.__num})", indent * " "
+        )
+
 
 class TimeWorkloads(VersionExperiment, shorthand="TWL"):
     """Generates time report files."""
@@ -95,6 +113,7 @@ def actions_for_project(
         binary = project.binaries[0]
 
         measurement_repetitions = 2
+
         result_filepath = create_new_success_result_filepath(
             self.get_handle(),
             self.get_handle().report_spec().main_report, project, binary