added better exception handling & filtering (#26)

* added better exception handling & filtering

* fixed linting issues

* cleaned up logging

* updated versions for new release

Makefile

@@ -14,7 +14,7 @@ lint:
 	yapf $(packages) --diff --parallel --recursive --style google
 	mypy $(packages) --disallow-untyped-defs --ignore-missing-imports --warn-unused-ignores || true # TODO(jack) Figure out why mypy is failing on 'has no attribute' error
 	bandit -r $(packages)
-	pylint $(packages) --disable=missing-docstring,bad-continuation,duplicate-code,W0511 --exit-zero
+	pylint $(packages) --disable=missing-docstring,bad-continuation,duplicate-code,W0511,R0912 --exit-zero
 
 venv:
 	virtualenv -p python3.7 venv

panther_analysis_tool/main.py

@@ -311,10 +311,19 @@ def test_analysis(args: argparse.Namespace) -> Tuple[int, list]:
         list(load_analysis_specs(args.path)) +
         list(load_analysis_specs(HELPERS_LOCATION)))
 
+    if len(analysis) == 0:
+        return 1, ["Nothing to test in {}".format(args.path)]
+
     # Apply the filters as needed
     global_analysis = filter_analysis(global_analysis, args.filter)
     analysis = filter_analysis(analysis, args.filter)
 
+    if len(analysis) == 0:
+        return 1, [
+            "No analyses in {} matched filters {}".format(
+                args.path, args.filter)
+        ]
+
     # First import the globals
     for analysis_spec_filename, dir_name, analysis_spec in global_analysis:
         module, load_err = load_module(
@@ -375,9 +384,16 @@ def filter_analysis(analysis: List[Any], filters: Dict[str, List]) -> List[Any]:
 
     filtered_analysis = []
     for file_name, dir_name, analysis_spec in analysis:
-        if all(
-                analysis_spec.get(key, "") in values
-                for key, values in filters.items()):
+        match = True
+        for key, values in filters.items():
+            spec_value = analysis_spec.get(key, "")
+            spec_value = spec_value if isinstance(spec_value,
+                                                  list) else [spec_value]
+            if not set(spec_value).intersection(values):
+                match = False
+                break
+
+        if match:
             filtered_analysis.append((file_name, dir_name, analysis_spec))
 
     return filtered_analysis
@@ -410,6 +426,10 @@ def classify_analysis(
                 SchemaUnexpectedTypeError) as err:
             invalid_specs.append((analysis_spec_filename, err))
             continue
+        except Exception as err:  # pylint: disable=broad-except
+            # Catch arbitrary exceptions thrown by bad specification files
+            invalid_specs.append((analysis_spec_filename, err))
+            continue
 
     return (global_analysis, analysis, invalid_specs)
 
@@ -430,17 +450,21 @@ def run_tests(analysis: Dict[str, Any], analysis_funcs: Dict[str, Any],
                 unit_test.get('ResourceType') or unit_test['LogType'])
             result = analysis_funcs['run'](test_case)
         except KeyError as err:
-            print("KeyError: {0}".format(err))
+            logging.warning('KeyError: {%s}', err)
+            continue
+        except Exception as err:  # pylint: disable=broad-except
+            # Catch arbitrary exceptions raised by user code
+            logging.warning('Unexpected exception: {%s}', err)
             continue
         test_result = 'PASS'
         if result != unit_test['ExpectedResult']:
             test_result = 'FAIL'
             failed_tests[analysis.get('PolicyID') or
                          analysis['RuleID']].append(unit_test['Name'])
         print('\t[{}] {}'.format(test_result, unit_test['Name']))
-        if analysis_funcs.get('title'):
+        if analysis_funcs.get('title') and unit_test['ExpectedResult']:
             print('\t\t[Title] {}'.format(analysis_funcs['title'](test_case)))
-        if analysis_funcs.get('dedup'):
+        if analysis_funcs.get('dedup') and unit_test['ExpectedResult']:
             print('\t\t[Dedup] {}'.format(analysis_funcs['dedup'](test_case)))
 
     return failed_tests
@@ -453,7 +477,7 @@ def setup_parser() -> argparse.ArgumentParser:
         prog='panther_analysis_tool')
     parser.add_argument('--version',
                         action='version',
-                        version='panther_analysis_tool 0.3.0')
+                        version='panther_analysis_tool 0.3.1')
     subparsers = parser.add_subparsers()
 
     test_parser = subparsers.add_parser(
@@ -525,8 +549,12 @@ def parse_filter(filters: List[str]) -> Dict[str, Any]:
                             filt)
             continue
         key = split[0]
-        if key not in list(GLOBAL_SCHEMA.schema.keys()) + list(
-                POLICY_SCHEMA.schema.keys()) + list(RULE_SCHEMA.schema.keys()):
+        if not any([
+                key in (list(GLOBAL_SCHEMA.schema.keys()) +
+                        list(POLICY_SCHEMA.schema.keys()) +
+                        list(RULE_SCHEMA.schema.keys()))
+                for key in (key, Optional(key))
+        ]):
             logging.warning(
                 'Filter key %s is not a valid filter field, skipping', key)
             continue
@@ -547,6 +575,10 @@ def run() -> None:
     except AttributeError:
         parser.print_help()
         sys.exit(1)
+    except Exception as err:  # pylint: disable=broad-except
+        # Catch arbitrary exceptions without printing help message
+        logging.warning('Unexpected exception: "%s"', err)
+        sys.exit(1)
 
     if return_code == 1:
         if out:

setup.py

@@ -2,14 +2,14 @@
 setup(
     name='panther_analysis_tool',
     packages=['panther_analysis_tool'],
-    version='0.3.0',
+    version='0.3.1',
     license='apache-2.0',
     description=
     'Panther command line interface for writing, testing, and packaging policies/rules.',
     author='Panther Labs Inc',
     author_email='pypi@runpanther.io',
     url='https://github.com/panther-labs/panther_analysis_tool',
-    download_url = 'https://github.com/panther-labs/panther_analysis_tool/archive/v0.3.0.tar.gz',
+    download_url = 'https://github.com/panther-labs/panther_analysis_tool/archive/v0.3.1.tar.gz',
     keywords=['Security', 'CLI'],
     scripts=['bin/panther_analysis_tool'],
     install_requires=[