Result by keys (#754)

* Fix autodetection of dependencies when in a sub-repo.

* Some results improvements

- Added '--by-key' and '--by-key-compat' to 'pav results' this prints a subtable of results in a
  pretty way, and merges results across that subtable for multiple tests.
- Added optional arguments to expression functions. Wrap specs in the new Opt() class to use.
- Added high_pass_filter and low_pass_filter expression functions.

* Update base.py

* Update core.py

* Update base.py

* Style issues.

* Fixed unittest issues.

Author: Paul-Ferrell

lib/pavilion/commands/result.py

@@ -1,5 +1,6 @@
 """Print the test results for the given test/suite."""
 
+from collections import defaultdict
 import datetime
 import errno
 import io
@@ -42,6 +43,16 @@ def _setup_arguments(self, parser):
             action="store_true", default=False,
             help="Give the results in json."
         )
+        parser.add_argument(
+            "--by-key", type=str, default='',
+            help="Show the data in the given results key instead of the regular results. \n"
+                 "Such keys must contain a dictionary of dictionaries. Use the `--by-key-compat`\n"
+                 "argument to find out which keys are compatible. Results from all matched \n"
+                 "tests are combined (duplicates are ignored).\n"
+                 "Example `pav results --by-key=per_file`.")
+        parser.add_argument(
+            "--by-key-compat", action="store_true",
+            help="List keys compatible with the '--by-key' argument.")
         group = parser.add_mutually_exclusive_group()
         group.add_argument(
             "-k", "--key", type=str, default='',
@@ -94,8 +105,6 @@ def _setup_arguments(self, parser):
     def run(self, pav_cfg, args):
         """Print the test results in a variety of formats."""
 
-        fields = self.key_fields(args)
-
         test_paths = cmd_utils.arg_filtered_tests(pav_cfg, args,
                                 verbose=self.errfile).paths
         tests = cmd_utils.get_tests_by_paths(pav_cfg, test_paths, self.errfile)
@@ -113,14 +122,66 @@ def run(self, pav_cfg, args):
         serieses = ",".join(
             set([test.series for test in tests if test.series is not None]))
         results = result_utils.get_results(pav_cfg, tests)
-        flat_results = []
-        all_passed = True
-        for rslt in results:
-            flat_results.append(utils.flatten_dictionary(rslt))
-            if rslt['result'] != TestRun.PASS:
-                all_passed = False
 
-        field_info = {}
+        if args.by_key_compat:
+            compat_keys = set()
+            for rslt in results:
+                for key in rslt:
+                    if isinstance(rslt[key], dict):
+                        for subkey, val in rslt[key].items():
+                            if isinstance(val, dict):
+                                compat_keys.add(key)
+                                break
+
+            if 'var' in compat_keys:
+                compat_keys.remove("var")
+
+            output.fprint(self.outfile, "Keys compatible with '--by-key'")
+            for key in compat_keys:
+                output.fprint(self.outfile, "  ", key)
+
+            return 0
+
+        elif args.by_key:
+            reorged_results = defaultdict(dict)
+            fields = set()
+            for rslt in results:
+                subtable = rslt.get(args.by_key, None)
+                if not isinstance(subtable, dict):
+                    continue
+                for key, values in subtable.items():
+                    if not isinstance(values, dict):
+                        continue
+                    reorged_results[key].update(values)
+                    fields = fields.union(values.keys())
+
+            fields = ['--tag'] + sorted(fields)
+            flat_results = []
+            for key, values in reorged_results.items():
+                values['--tag'] = key
+                flat_results.append(values)
+
+            flat_results.sort(key=lambda val: val['--tag'])
+
+            field_info = {
+                '--tag': {'title': ''},
+            }
+
+        else:
+            fields = self.key_fields(args)
+            flat_results = []
+            all_passed = True
+            for rslt in results:
+                flat_results.append(utils.flatten_dictionary(rslt))
+                if rslt['result'] != TestRun.PASS:
+                    all_passed = False
+            field_info = {
+                'created': {'transform': output.get_relative_timestamp},
+                'started': {'transform': output.get_relative_timestamp},
+                'finished': {'transform': output.get_relative_timestamp},
+                'duration': {'transform': output.format_duration},
+                }
+
 
         if args.list_keys:
             flat_keys = result_utils.keylist(flat_results)
@@ -132,7 +193,6 @@ def run(self, pav_cfg, args):
             title_str=f"Available keys for specified tests in {serieses}."
 
             output.draw_table(outfile=self.outfile,
-                              field_info=field_info,
                               fields=fields,
                               rows=flatter_keys,
                               border=True,
@@ -161,13 +221,6 @@ def run(self, pav_cfg, args):
         else:
             flat_sorted_results = utils.sort_table(args.sort_by, flat_results)
 
-            field_info = {
-                'created': {'transform': output.get_relative_timestamp},
-                'started': {'transform': output.get_relative_timestamp},
-                'finished': {'transform': output.get_relative_timestamp},
-                'duration': {'transform': output.format_duration},
-                }
-
             title_str=f"Test Results: {serieses}."
             output.draw_table(
                 outfile=self.outfile,

lib/pavilion/expression_functions/base.py

@@ -3,6 +3,7 @@
 import inspect
 import logging
 import re
+from typing import Any
 
 from yapsy import IPlugin
 from ..errors import FunctionPluginError
@@ -36,6 +37,18 @@ def num(val):
     raise RuntimeError("Invalid value '{}' given to num.".format(val))
 
 
+class Opt:
+    """An optional arg spec, the contained spec is checked if the value is given."""
+
+    def __init__(self, sub_spec: Any):
+        """
+        :param sub_spec: The type of the argument spec to accept.
+                         Ex:  Opt(int) or Opt([str])
+        """
+
+        self.sub_spec = sub_spec
+
+
 class FunctionPlugin(IPlugin.IPlugin):
     """Plugin base class for math functions.
 
@@ -51,6 +64,7 @@ class FunctionPlugin(IPlugin.IPlugin):
         str,
         bool,
         num,
+        Opt,
         None
     )
 
@@ -130,7 +144,6 @@ def _validate_arg_spec(self, arg):
             dicts, and types from self.VALID_SPEC_TYPES.
 
             - Lists should contain one representative containing type.
-            - Dicts should have at least one key-value pair (with string keys).
             - Dict specs don't have to contain every key the dict might have,
               just those that will be used.
             - Specs may be any structure of these types, as long
@@ -139,11 +152,15 @@ def _validate_arg_spec(self, arg):
               or bool. ints and floats are left alone, bools become
               ints, and strings become an int or a float if they can.
             - 'None' may be given as the type of contained items for lists
-              or dicts, denoting that contained type doesn't matter.
+              or dicts, denoting that contained type doesn't matter. Dict
+              specs can similarly contain no items.
         :raises FunctionPluginError: On a bad arg spec.
         """
 
-        if isinstance(arg, list):
+        if isinstance(arg, Opt):
+            self._validate_arg_spec(arg.sub_spec)
+
+        elif isinstance(arg, list):
             if len(arg) != 1:
                 raise FunctionPluginError(
                     "Invalid list spec argument. List arguments must contain "
@@ -153,12 +170,6 @@ def _validate_arg_spec(self, arg):
             self._validate_arg_spec(arg[0])
 
         elif isinstance(arg, dict):
-            if len(arg) == 0:
-                raise FunctionPluginError(
-                    "Invalid dict spec argument. Dict arguments must contain "
-                    "at least one key-value pair. This had '{}'."
-                    .format(arg)
-                )
             for key, sub_arg in arg.items():
                 self._validate_arg_spec(sub_arg)
 
@@ -182,7 +193,7 @@ def __call__(self, *args):
         """Validate/convert the arguments and call the function."""
 
         if self.arg_specs is not None:
-            if len(args) != len(self.arg_specs):
+            if len(args) > len(self.arg_specs):
                 raise FunctionPluginError(
                     "Invalid number of arguments defined for function {}. Got "
                     "{}, but expected {}"
@@ -243,6 +254,8 @@ def _spec_to_desc(self, spec):
             return [self._spec_to_desc(spec[0])]
         elif isinstance(spec, dict):
             return {k: self._spec_to_desc(v) for k, v in spec.items()}
+        elif isinstance(spec, Opt):
+            return self._spec_to_desc(spec.sub_spec) + '?'
         elif spec is None:
             return 'Any'
         else:
@@ -258,6 +271,9 @@ def _validate_arg(self, arg, spec):
         :return: The validated, auto-converted argument.
         """
 
+        if isinstance(spec, Opt):
+            return self._validate_arg(arg, spec.sub_spec)
+
         if isinstance(spec, list):
             if not isinstance(arg, list):
                 raise FunctionPluginError(
@@ -290,13 +306,13 @@ def _validate_arg(self, arg, spec):
                         .format(arg, key))
 
                 try:
-                    val_args[key] = self._validate_arg(arg[key], sub_spec)
+                    self._validate_arg(arg[key], sub_spec)
                 except FunctionPluginError as err:
                     raise FunctionPluginError(
                         "Invalid dict argument '{}' for key '{}'"
                         .format(arg[key], key), err)
 
-            return val_args
+            return arg
 
         if spec is None:
             # None denotes to leave the argument alone.

lib/pavilion/expression_functions/core.py

@@ -4,9 +4,9 @@
 import math
 import random
 import re
-from typing import List
+from typing import List, Dict, Union
 
-from .base import FunctionPlugin, num
+from .base import FunctionPlugin, num, Opt
 from ..errors import FunctionPluginError, FunctionArgError
 
 
@@ -253,18 +253,9 @@ def __init__(self):
 
         super().__init__(
             name='keys',
-            arg_specs=None,
+            arg_specs=({},),
         )
 
-    signature = "keys(dict)"
-
-    def _validate_arg(self, arg, spec):
-        if not isinstance(arg, dict):
-            raise FunctionPluginError(
-                "The dicts function only accepts dicts. Got {} of type {}."
-                .format(arg, type(arg).__name__))
-        return arg
-
     @staticmethod
     def keys(arg):
         """Return a (sorted) list of keys for the given dictionary."""
@@ -370,6 +361,93 @@ def sqrt(value: num):
         return value ** 0.5
 
 
+class HighPassFilter(CoreFunctionPlugin):
+    """Given the 'value_dict', return a new dictionary that contains only
+    items that exceed 'limit'. For dicts of dicts, you must specify an item_key
+    to check limit against.
+
+    Examples:
+     Given dict 'data={a: 1, b: 2, c: 3, d: 4}',
+     `high_pass_filter(data, 3)` would return a dict with
+     the 'c' and 'd' keys removed.
+
+     Given dict 'data={foo: {a: 5}, bar: {a: 100}}, baz: {a: 20}}'
+     `high_pass_filter(data, 20, 'a')` would return a dict containing
+     only key 'foo' and its value/s."""
+
+    def __init__(self):
+        super().__init__(
+            'high_pass_filter',
+            arg_specs=({}, num, Opt(str)))
+
+    @staticmethod
+    def high_pass_filter(value_dict: Dict, limit: Union[int, float], item_key: str = None) -> Dict:
+        """Return only items > limit"""
+
+        new_dict = {}
+        for key, values in value_dict.items():
+            if isinstance(values, dict):
+                if item_key is None:
+                    raise FunctionArgError("value_dict contained a dict, but no key was specified.")
+
+                value = values.get(item_key)
+            else:
+                if item_key is not None:
+                    raise FunctionArgError(
+                        "value_dict contained a non-dictionary, but a key was specified.")
+
+                value = values
+
+            if isinstance(value, (int, float, str)):
+                value = num(value)
+            else:
+                continue
+
+            if value > limit:
+                new_dict[key] = values
+
+        return new_dict
+
+
+class LowPassFilter(CoreFunctionPlugin):
+    """Given the 'value_dict', return a new dictionary that contains only
+    items that are less than 'limit'. For dicts of dicts, you must specify
+    a sub-key to check 'limit' against. See 'high_pass_filter' for examples."""
+
+    def __init__(self):
+        super().__init__(
+            'low_pass_filter',
+            arg_specs=({}, num, Opt(str)))
+
+    @staticmethod
+    def low_pass_filter(value_dict: Dict, limit: Union[int, float], item_key: str = None) -> Dict:
+        """Return only items > limit"""
+
+        new_dict = {}
+        for key, values in value_dict.items():
+            if isinstance(values, dict):
+                if item_key is None:
+                    raise FunctionArgError("value_dict contained a dict, but no key was specified.")
+
+                value = values.get(item_key)
+            else:
+                if item_key is not None:
+                    raise FunctionArgError(
+                        "value_dict contained a non-dictionary, but a key was specified.")
+
+                value = values
+
+            if isinstance(value, (int, float, str)):
+                value = num(value)
+            else:
+                continue
+
+            if value < limit:
+                new_dict[key] = values
+
+        return new_dict
+
+
 class Range(CoreFunctionPlugin):
     """Return a list of numbers from a..b, not inclusive of b."""
 
@@ -390,6 +468,7 @@ def range(start, end):
 
         return vals
 
+
 class Outliers(CoreFunctionPlugin):
     """Calculate outliers given a list of values and a separate list
     of their associated names. The lists should be the same length, and

test/data/configs-rerun/tests/result_tests.yaml

@@ -31,3 +31,7 @@ permuted:
         result: 'world < 50 * {{var1}}'
         other: 'world * {{var1}} + {{var2}}'
 
+
+complex:
+  result_evaluate:
+    result: '0 == 0'