One read multiple report steps

lab/experiment.py

@@ -10,6 +10,7 @@
 from lab import environments, tools
 from lab.fetcher import Fetcher
 from lab.parser import Parser
+from lab.reports import Report
 from lab.steps import Step, get_step, get_steps_text
 
 # How many tasks to group into one top-level directory.
@@ -24,11 +25,21 @@
     metavar="step",
     nargs="*",
     default=[],
-    help="Name or number of a step below. If none is given, print help.",
+    help=(
+        "Name or number of a step below. If none is given, print help. "
+        "For more than one step, separate intervals by using commas. "
+        "An interval is a single number of two numbers separated by a dash. "
+        "Examples of valid arguments:\n"
+        "1\n1-10. 1,4,5,7. 1,2-4,8-10,15-20. "
+        "You can also separate single steps by spaces."
+    ),
 )
 steps_group.add_argument(
     "--all", dest="run_all_steps", action="store_true", help="Run all steps."
 )
+steps_group.add_argument(
+    "--reports", dest="run_only_reports", action="store_true", help="Run only reports."
+)
 
 STATIC_EXPERIMENT_PROPERTIES_FILENAME = "static-experiment-properties"
 STATIC_RUN_PROPERTIES_FILENAME = "static-properties"
@@ -366,6 +377,9 @@ def __init__(self, path=None, environment=None):
         self.runs = []
         self.parsers = []
 
+        # This attribute will be set by the first report that loads data.
+        self.props = {}
+
         self.set_property("experiment_file", self._script)
 
     @property
@@ -516,7 +530,9 @@ def add_fetcher(
         src = src or self.path
         dest = dest or self.eval_dir
         name = name or f"fetch-{os.path.basename(src.rstrip('/'))}"
-        self.add_step(name, Fetcher(), src, dest, merge=merge, filter=filter, **kwargs)
+        self.add_step(
+            name, Fetcher(self), src, dest, merge=merge, filter=filter, **kwargs
+        )
 
     def add_report(self, report, name="", eval_dir="", outfile=""):
         """Add *report* to the list of experiment steps.
@@ -538,6 +554,7 @@ def add_report(self, report, name="", eval_dir="", outfile=""):
         >>> exp.add_report(AbsoluteReport(attributes=["coverage"]))
 
         """
+        report.exp = self
         name = name or os.path.basename(outfile) or report.__class__.__name__.lower()
         eval_dir = eval_dir or self.eval_dir
         outfile = outfile or f"{name}.{report.output_format}"
@@ -556,16 +573,47 @@ def add_run(self, run=None):
         self.runs.append(run)
         return run
 
+    def parse_steps(self, steps_arg, run_all_steps, run_only_reports):
+        if run_all_steps:
+            return self.steps
+        elif run_only_reports:
+            return [step for step in self.steps if isinstance(step.func, Report)]
+
+        if len(steps_arg) > 1:
+            return [get_step(self.steps, name) for name in steps_arg]
+        else:
+            parsed_steps = []
+            for interval in steps_arg[0].split(","):
+                if "-" in interval:
+                    # The interval must have exactly 2 members: init and end.
+                    split = interval.split("-")
+                    if len(split) != 2:
+                        logging.critical(
+                            "Intervals must have exactly 2 elements: init and end"
+                        )
+                        ARGPARSER.print_help()
+                        sys.exit(1)
+                    else:
+                        for i in range(int(split[0]), int(split[1]) + 1):
+                            parsed_steps.append(str(i))
+                else:
+                    parsed_steps.append(interval)
+
+            return [get_step(self.steps, name) for name in parsed_steps]
+
     def run_steps(self):
         """Parse the commandline and run selected steps."""
         ARGPARSER.epilog = get_steps_text(self.steps)
         args = ARGPARSER.parse_args()
-        assert not args.steps or not args.run_all_steps
-        if not args.steps and not args.run_all_steps:
+        assert not args.steps or (not args.run_all_steps and not args.run_only_reports)
+        if args.steps:
+            assert not args.run_all_steps and not args.run_only_reports
+        else:
+            assert not args.run_all_steps or not args.run_only_reports
+        if not args.steps and not args.run_all_steps and not args.run_only_reports:
             ARGPARSER.print_help()
             return
-        # Run all steps if --all is passed.
-        steps = [get_step(self.steps, name) for name in args.steps] or self.steps
+        steps = self.parse_steps(args.steps, args.run_all_steps, args.run_only_reports)
         # Use LocalEnvironment if the main experiment step is inactive.
         if any(environments.is_run_step(step) for step in steps):
             env = self.environment

lab/fetcher.py

@@ -43,6 +43,9 @@ class Fetcher:
 
     """
 
+    def __init__(self, exp):
+        self.exp = exp
+
     def fetch_dir(self, run_dir):
         """Combine "static-properties" and "properties" from a run dir and return it."""
         run_dir = Path(run_dir)
@@ -159,6 +162,7 @@ def __call__(self, src_dir, eval_dir=None, merge=None, filter=None, **kwargs):
 
         tools.makedirs(eval_dir)
         combined_props.write()
+        self.exp.props[eval_dir] = combined_props
         func = logging.info if unexplained_errors == 0 else logging.warning
         func(
             f"Wrote properties file. It contains {unexplained_errors} "

lab/reports/__init__.py

@@ -242,6 +242,8 @@ def __init__(self, attributes=None, format="html", filter=None, **kwargs):
         self.output_format = format
         self.toc = True
         self.run_filter = tools.RunFilter(filter, **kwargs)
+        # The exp is set by experiment.py in add_report function.
+        self.exp = None
 
     def __call__(self, eval_dir, outfile):
         """Make the report.
@@ -412,15 +414,21 @@ def _scan_data(self):
         self._all_attributes = self._get_type_map(attributes)
 
     def _load_data(self):
-        props_file = os.path.join(self.eval_dir, "properties")
-        logging.info("Reading properties file")
-        self.props = tools.Properties(filename=props_file)
-        if not self.props:
-            logging.critical(f"No properties found in {self.eval_dir}")
-        logging.info("Reading properties file finished")
+        if self.eval_dir not in self.exp.props or not self.exp.props[self.eval_dir]:
+            props_file = os.path.join(self.eval_dir, "properties")
+            logging.info("Reading properties file")
+            self.exp.props[self.eval_dir] = tools.Properties(filename=props_file)
+            if not self.exp.props[self.eval_dir]:
+                logging.critical(f"No properties found in {self.eval_dir}")
+            logging.info("Reading properties file finished")
+
+        self.props = self.exp.props[self.eval_dir]
 
     def _apply_filter(self):
-        self.run_filter.apply(self.props)
+        # Removing elements would require a deepcopy of self.exp.props, with
+        # the resulting overhead in time and memory. Instead, a new dict
+        # containing only references to the non-filtered elements is generated.
+        self.props = self.run_filter.mirror(self.props)
         if not self.props:
             logging.critical("All runs have been filtered -> Nothing to report.")
 

lab/tools.py

@@ -370,6 +370,29 @@ def apply(self, props):
                     new_run_id = "-".join(new_run["id"]) if "id" in run else old_run_id
                     props[new_run_id] = new_run
 
+    def mirror(self, props):
+        for attribute in self.filtered_attributes:
+            if not any(attribute in run for run in props.values()):
+                logging.critical(
+                    f'No run has the attribute "{attribute}" (from '
+                    f'"filter_{attribute}"). Is this a typo?'
+                )
+        if not self.filters:
+            return props
+        else:
+            filtered_props = {}
+            for filter_ in self.filters:
+                for old_run_id, run in list(props.items()):
+                    new_run = self.apply_filter_to_run(filter_, run)
+                    if new_run:
+                        # Filters may change the ID. Don't complain if ID is missing.
+                        new_run_id = (
+                            "-".join(new_run["id"]) if "id" in run else old_run_id
+                        )
+                        filtered_props[new_run_id] = new_run
+
+            return filtered_props
+
 
 def fast_updatetree(src, dst, symlinks=False, ignore=None):
     """