Lindseyw test dedup title (#43)

* check return is non-None for dedup/title methods

* add passing and failed tests for dedup/title non-None check

* update key name; added clarifying comment

* Update tests/fixtures/example_rule_missing_field.yml

convert to all yaml

Co-authored-by: Austin Byers <austin.byers@runpanther.io>

* fixing format/adding newline to end of tile

* fixing lint error

* update comment; fix key name

Co-authored-by: Austin Byers <austin.byers@runpanther.io>

panther_analysis_tool/main.py

@@ -419,16 +419,34 @@ def run_tests(analysis: Dict[str, Any], analysis_funcs: Dict[str, Any],
             failed_tests[analysis.get('PolicyID') or
                          analysis['RuleID']].append(unit_test['Name'])
             continue
-        test_result = 'PASS'
+
+        # using a dictionary to map between the tests and their outcomes
+        # assume the test passes (default "PASS")
+        # until failure condition is found (set to "FAIL")
+        test_result = defaultdict(lambda: 'PASS')
+        # check expected result
         if result != unit_test['ExpectedResult']:
-            test_result = 'FAIL'
+            test_result['outcome'] = 'FAIL'
+
+        # check dedup and title function return non-None
+        # Only applies to rules which match an incoming event
+        if unit_test['ExpectedResult']:
+            for func in ['dedup', 'title']:
+                if analysis_funcs.get(func):
+                    if not analysis_funcs[func](test_case):
+                        test_result[func] = 'FAIL'
+                        test_result['outcome'] = 'FAIL'
+
+        if test_result['outcome'] == 'FAIL':
             failed_tests[analysis.get('PolicyID') or
                          analysis['RuleID']].append(unit_test['Name'])
-        print('\t[{}] {}'.format(test_result, unit_test['Name']))
-        if analysis_funcs.get('title') and unit_test['ExpectedResult']:
-            print('\t\t[Title] {}'.format(analysis_funcs['title'](test_case)))
-        if analysis_funcs.get('dedup') and unit_test['ExpectedResult']:
-            print('\t\t[Dedup] {}'.format(analysis_funcs['dedup'](test_case)))
+
+        # print results
+        print('\t[{}] {}'.format(test_result['outcome'], unit_test['Name']))
+        for func in ['dedup', 'title']:
+            if analysis_funcs.get(func) and unit_test['ExpectedResult']:
+                print('\t\t[{}] [{}] {}'.format(
+                    test_result[func], func, analysis_funcs[func](test_case)))
 
     return failed_tests
 

tests/fixtures/example_rule.py

@@ -0,0 +1,22 @@
+from panther import test_helper # pylint: disable=import-error
+
+IGNORED_USERS = {}
+
+
+def rule(event):
+    if event['UserName'] in IGNORED_USERS:
+        return False
+
+    cred_report = event.get('CredentialReport', {})
+    if not cred_report:
+        return True
+
+    return (test_helper() and
+            cred_report.get('PasswordEnabled', False) and
+            cred_report.get('MfaActive', False))
+
+def dedup(event):
+    return None
+
+def title(event):
+    return '{} does not have MFA enabled'.format(event['UserName'])

tests/fixtures/example_rule_missing_field.yml

@@ -0,0 +1,31 @@
+AnalysisType: rule 
+Filename: example_rule.py
+DisplayName: MFA Rule
+Description: MFA is a security best practice that adds an extra layer of protection for your AWS account logins.
+Severity: Critical
+Threshold: 5
+RuleID: AWS.CloudTrail.MFAEnabled
+Enabled: true
+SummaryAttributes:
+  - p_log_type
+  - p_any_ip_addresses
+LogTypes:
+  - AWS.CloudTrail
+Tags:
+  - AWS Managed Rules - Security, Identity & Compliance
+  - AWS
+  - CIS
+  - SOC2
+Runbook: >
+  Find out who disabled MFA on the account.
+Reference: https://www.link-to-info.io
+Tests:
+  -
+    Name: User MFA enabled passes compliance but fails dedup check.
+    ExpectedResult: true
+    Log:
+      Arn: arn:aws:iam::123456789012:user/test
+      CreateDate: '2019-01-01T00:00:00'
+      CredentialReport:
+        MfaActive: true
+        PasswordEnabled: true

tests/fixtures/valid_analysis/rules/example_rule.yml

@@ -43,3 +43,17 @@ Tests:
         },
         "UserName": "test"
       }
+  -
+    Name: User MFA enabled passes compliance.
+    ExpectedResult: true
+    Log:
+      {
+        "Arn": "arn:aws:iam::123456789012:user/test",
+        "CreateDate": "2019-01-01T00:00:00",
+        "CredentialReport": {
+          "MfaActive": true,
+          "PasswordEnabled": true
+        },
+        "UserName": "test"
+      }
+

tests/unit/panther_analysis_tool/test_main.py

@@ -62,6 +62,13 @@ def test_with_filters(self):
         assert_equal(return_code, 0)
         assert_equal(len(invalid_specs), 0)
 
+    def test_invalid_rule_definition(self):
+        args = pat.setup_parser().parse_args('test --path tests/fixtures --filter Severity=Critical RuleID=AWS.CloudTrail.MFAEnabled'.split())
+        args.filter = pat.parse_filter(args.filter)
+        return_code, invalid_specs = pat.test_analysis(args)
+        assert_equal(return_code, 1)
+        assert_equal(len(invalid_specs), 1)
+
     def test_with_tag_filters(self):
         args = pat.setup_parser().parse_args('test --path tests/fixtures/valid_analysis --filter Tags=AWS,CIS'.split())
         args.filter = pat.parse_filter(args.filter)