diff --git a/lab/experiment.py b/lab/experiment.py index c66b8d28e..1d0afe5d4 100644 --- a/lab/experiment.py +++ b/lab/experiment.py @@ -10,6 +10,7 @@ from lab import environments, tools from lab.fetcher import Fetcher from lab.parser import Parser +from lab.reports import Report from lab.steps import Step, get_step, get_steps_text # How many tasks to group into one top-level directory. @@ -24,11 +25,21 @@ metavar="step", nargs="*", default=[], - help="Name or number of a step below. If none is given, print help.", + help=( + "Name or number of a step below. If none is given, print help. " + "For more than one step, separate intervals by using commas. " + "An interval is a single number of two numbers separated by a dash. " + "Examples of valid arguments:\n" + "1\n1-10. 1,4,5,7. 1,2-4,8-10,15-20. " + "You can also separate single steps by spaces." + ), ) steps_group.add_argument( "--all", dest="run_all_steps", action="store_true", help="Run all steps." ) +steps_group.add_argument( + "--reports", dest="run_only_reports", action="store_true", help="Run only reports." +) STATIC_EXPERIMENT_PROPERTIES_FILENAME = "static-experiment-properties" STATIC_RUN_PROPERTIES_FILENAME = "static-properties" @@ -366,6 +377,9 @@ def __init__(self, path=None, environment=None): self.runs = [] self.parsers = [] + # This attribute will be set by the first report that loads data. + self.props = {} + self.set_property("experiment_file", self._script) @property @@ -516,7 +530,9 @@ def add_fetcher( src = src or self.path dest = dest or self.eval_dir name = name or f"fetch-{os.path.basename(src.rstrip('/'))}" - self.add_step(name, Fetcher(), src, dest, merge=merge, filter=filter, **kwargs) + self.add_step( + name, Fetcher(self), src, dest, merge=merge, filter=filter, **kwargs + ) def add_report(self, report, name="", eval_dir="", outfile=""): """Add *report* to the list of experiment steps. @@ -538,6 +554,7 @@ def add_report(self, report, name="", eval_dir="", outfile=""): >>> exp.add_report(AbsoluteReport(attributes=["coverage"])) """ + report.exp = self name = name or os.path.basename(outfile) or report.__class__.__name__.lower() eval_dir = eval_dir or self.eval_dir outfile = outfile or f"{name}.{report.output_format}" @@ -556,16 +573,47 @@ def add_run(self, run=None): self.runs.append(run) return run + def parse_steps(self, steps_arg, run_all_steps, run_only_reports): + if run_all_steps: + return self.steps + elif run_only_reports: + return [step for step in self.steps if isinstance(step.func, Report)] + + if len(steps_arg) > 1: + return [get_step(self.steps, name) for name in steps_arg] + else: + parsed_steps = [] + for interval in steps_arg[0].split(","): + if "-" in interval: + # The interval must have exactly 2 members: init and end. + split = interval.split("-") + if len(split) != 2: + logging.critical( + "Intervals must have exactly 2 elements: init and end" + ) + ARGPARSER.print_help() + sys.exit(1) + else: + for i in range(int(split[0]), int(split[1]) + 1): + parsed_steps.append(str(i)) + else: + parsed_steps.append(interval) + + return [get_step(self.steps, name) for name in parsed_steps] + def run_steps(self): """Parse the commandline and run selected steps.""" ARGPARSER.epilog = get_steps_text(self.steps) args = ARGPARSER.parse_args() - assert not args.steps or not args.run_all_steps - if not args.steps and not args.run_all_steps: + assert not args.steps or (not args.run_all_steps and not args.run_only_reports) + if args.steps: + assert not args.run_all_steps and not args.run_only_reports + else: + assert not args.run_all_steps or not args.run_only_reports + if not args.steps and not args.run_all_steps and not args.run_only_reports: ARGPARSER.print_help() return - # Run all steps if --all is passed. - steps = [get_step(self.steps, name) for name in args.steps] or self.steps + steps = self.parse_steps(args.steps, args.run_all_steps, args.run_only_reports) # Use LocalEnvironment if the main experiment step is inactive. if any(environments.is_run_step(step) for step in steps): env = self.environment diff --git a/lab/fetcher.py b/lab/fetcher.py index be610e8cc..9a1328352 100644 --- a/lab/fetcher.py +++ b/lab/fetcher.py @@ -43,6 +43,9 @@ class Fetcher: """ + def __init__(self, exp): + self.exp = exp + def fetch_dir(self, run_dir): """Combine "static-properties" and "properties" from a run dir and return it.""" run_dir = Path(run_dir) @@ -159,6 +162,7 @@ def __call__(self, src_dir, eval_dir=None, merge=None, filter=None, **kwargs): tools.makedirs(eval_dir) combined_props.write() + self.exp.props[eval_dir] = combined_props func = logging.info if unexplained_errors == 0 else logging.warning func( f"Wrote properties file. It contains {unexplained_errors} " diff --git a/lab/reports/__init__.py b/lab/reports/__init__.py index 04e580e68..fe87e8fff 100644 --- a/lab/reports/__init__.py +++ b/lab/reports/__init__.py @@ -242,6 +242,8 @@ def __init__(self, attributes=None, format="html", filter=None, **kwargs): self.output_format = format self.toc = True self.run_filter = tools.RunFilter(filter, **kwargs) + # The exp is set by experiment.py in add_report function. + self.exp = None def __call__(self, eval_dir, outfile): """Make the report. @@ -412,15 +414,21 @@ def _scan_data(self): self._all_attributes = self._get_type_map(attributes) def _load_data(self): - props_file = os.path.join(self.eval_dir, "properties") - logging.info("Reading properties file") - self.props = tools.Properties(filename=props_file) - if not self.props: - logging.critical(f"No properties found in {self.eval_dir}") - logging.info("Reading properties file finished") + if self.eval_dir not in self.exp.props or not self.exp.props[self.eval_dir]: + props_file = os.path.join(self.eval_dir, "properties") + logging.info("Reading properties file") + self.exp.props[self.eval_dir] = tools.Properties(filename=props_file) + if not self.exp.props[self.eval_dir]: + logging.critical(f"No properties found in {self.eval_dir}") + logging.info("Reading properties file finished") + + self.props = self.exp.props[self.eval_dir] def _apply_filter(self): - self.run_filter.apply(self.props) + # Removing elements would require a deepcopy of self.exp.props, with + # the resulting overhead in time and memory. Instead, a new dict + # containing only references to the non-filtered elements is generated. + self.props = self.run_filter.mirror(self.props) if not self.props: logging.critical("All runs have been filtered -> Nothing to report.") diff --git a/lab/tools.py b/lab/tools.py index 9cd7d2034..361c04487 100644 --- a/lab/tools.py +++ b/lab/tools.py @@ -370,6 +370,29 @@ def apply(self, props): new_run_id = "-".join(new_run["id"]) if "id" in run else old_run_id props[new_run_id] = new_run + def mirror(self, props): + for attribute in self.filtered_attributes: + if not any(attribute in run for run in props.values()): + logging.critical( + f'No run has the attribute "{attribute}" (from ' + f'"filter_{attribute}"). Is this a typo?' + ) + if not self.filters: + return props + else: + filtered_props = {} + for filter_ in self.filters: + for old_run_id, run in list(props.items()): + new_run = self.apply_filter_to_run(filter_, run) + if new_run: + # Filters may change the ID. Don't complain if ID is missing. + new_run_id = ( + "-".join(new_run["id"]) if "id" in run else old_run_id + ) + filtered_props[new_run_id] = new_run + + return filtered_props + def fast_updatetree(src, dst, symlinks=False, ignore=None): """