Cleanup unneeded ResourceType/LogType fields in policy/rule tests (#34)

The ResourceType (in policy tests) and LogType (in rule tests) fields are not necessary since they can be inferred from the policy/rule under test. The tool has been changed to successfully parse older policy/rule tests that include these fields. However, this is a backwards-incompatible change for the analysis-api and the fields must be present in the tests when deploying them to older Panther versions. See https://github.com/panther-labs/panther/issues/610 for more context.
panther-labs · Jul 8, 2020 · ad1393c · ad1393c
1 parent c44ed8b
commit ad1393c
Show file tree

Hide file tree

Showing 13 changed files with 92 additions and 27 deletions.
diff --git a/README.md b/README.md
@@ -223,7 +223,7 @@ Reference: https://www.link-to-info.io
 Tests:
   -
     Name: Name to describe our first test.
-    ResourceType: Resource.Type.Here
+    ResourceType: Resource.Type.Here  # Not needed in Panther versions >= 1.6.0
     ExpectedResult: true/false
     Resource:
       Key: Values
@@ -288,7 +288,7 @@ Reference: https://www.link-to-info.io
 Tests:
   -
     Name: Name to describe our first test.
-    LogType: Log.Type.Here
+    LogType: Log.Type.Here  # Not needed in Panther versions >= 1.6.0
     ExpectedResult: true/false
     Resource:
       Key: Values

diff --git a/panther_analysis_tool/main.py b/panther_analysis_tool/main.py
@@ -44,9 +44,12 @@
 
 class TestCase():
 
-    def __init__(self, data: Dict[str, Any], schema: str) -> None:
+    def __init__(self, data: Dict[str, Any]) -> None:
+        """
+        Args:
+            data (Dict[str, Any]): An AWS Resource representation or Log event to test the policy or rule against respectively.
+        """
         self._data = data
-        self.schema = schema
 
     def __getitem__(self, arg: str) -> Any:
         return self._data.get(arg, None)
@@ -402,9 +405,7 @@ def run_tests(analysis: Dict[str, Any], analysis_funcs: Dict[str, Any],
 
     for unit_test in analysis['Tests']:
         try:
-            test_case = TestCase(
-                unit_test.get('Resource') or unit_test['Log'],
-                unit_test.get('ResourceType') or unit_test['LogType'])
+            test_case = TestCase(unit_test.get('Resource') or unit_test['Log'])
             result = analysis_funcs['run'](test_case)
         except KeyError as err:
             logging.warning('KeyError: {%s}', err)
@@ -434,7 +435,7 @@ def setup_parser() -> argparse.ArgumentParser:
         prog='panther_analysis_tool')
     parser.add_argument('--version',
                         action='version',
-                        version='panther_analysis_tool 0.3.2')
+                        version='panther_analysis_tool 0.3.3')
     subparsers = parser.add_subparsers()
 
     test_parser = subparsers.add_parser(

diff --git a/panther_analysis_tool/schemas.py b/panther_analysis_tool/schemas.py
@@ -65,12 +65,13 @@
         },
         Optional('Tests'): [{
             'Name': str,
-            'ResourceType': str,
+            Optional('ResourceType'):
+                str,  # Not needed anymore, optional for backwards compatibility
             'ExpectedResult': bool,
             'Resource': object,
         }],
     },
-    ignore_extra_keys=False)
+    ignore_extra_keys=False)  # Prevent user typos on optional fields
 
 RULE_SCHEMA = Schema(
     {
@@ -102,9 +103,10 @@
         },
         Optional('Tests'): [{
             'Name': str,
-            'LogType': str,
+            Optional('LogType'):
+                str,  # Not needed anymore, optional for backwards compatibility
             'ExpectedResult': bool,
             'Log': object,
         }],
     },
-    ignore_extra_keys=False)
+    ignore_extra_keys=False)  # Prevent user typos on optional fields
diff --git a/setup.py b/setup.py
@@ -2,7 +2,7 @@
 setup(
     name='panther_analysis_tool',
     packages=['panther_analysis_tool'],
-    version='0.3.2',
+    version='0.3.3',
     license='apache-2.0',
     description=
     'Panther command line interface for writing, testing, and packaging policies/rules.',

diff --git a/tests/fixtures/aws_globals.yml b/tests/fixtures/aws_globals.yml
@@ -13,5 +13,4 @@ Tests:
   -
     Name: Dummy Test
     ExpectedResult: true
-    ResourceType: AWS.Dummy.Type
     Resource: {}
diff --git a/tests/fixtures/example_malformed_policy.yml b/tests/fixtures/example_malformed_policy.yml
@@ -15,7 +15,6 @@ Reference: https://www.link-to-info.io
 Tests:
   -
     Name: Root MFA not enabled triggers a violation.
-    ResourceType: AWS.IAM.RootUser.Snapshot
     ExpectedResult: false
     Resource:
       Arn: arn:aws:iam::123456789012:user/root
@@ -26,7 +25,6 @@ Tests:
       UserName: root
   -
     Name: User MFA not enabled triggers a violation.
-    ResourceType: AWS.IAM.User.Snapshot
     ExpectedResult: false
     Resource:
       {
@@ -40,7 +38,6 @@ Tests:
       }
   -
     Name: User with no password enabled does not trigger a policy violation.
-    ResourceType: AWS.IAM.User.Snapshot
     ExpectedResult: true
     DOG: lol
     Resource:

diff --git a/tests/fixtures/example_policy.yml b/tests/fixtures/example_policy.yml
@@ -22,7 +22,6 @@ Suppressions:
 Tests:
   -
     Name: Root MFA not enabled triggers a violation.
-    ResourceType: AWS.IAM.RootUser.Snapshot
     ExpectedResult: false
     Resource:
       Arn: arn:aws:iam::123456789012:user/root
@@ -33,7 +32,6 @@ Tests:
       UserName: root
   -
     Name: User MFA not enabled triggers a violation.
-    ResourceType: AWS.IAM.User.Snapshot
     ExpectedResult: false
     Resource:
       {
@@ -47,7 +45,6 @@ Tests:
       }
   -
     Name: User MFA enabled.
-    ResourceType: AWS.IAM.User.Snapshot
     ExpectedResult: false
     Resource:
       {

diff --git a/tests/fixtures/example_policy_import.yml b/tests/fixtures/example_policy_import.yml
@@ -9,7 +9,6 @@ ResourceTypes:
 Tests:
   -
     Name: Always Returns True
-    ResourceType: AWS.Import.Test
     ExpectedResult: true
     Resource:
       Key: value
diff --git a/tests/fixtures/valid_analysis/policies/example_policy.yml b/tests/fixtures/valid_analysis/policies/example_policy.yml
@@ -24,7 +24,6 @@ Reference: https://www.link-to-info.io
 Tests:
   -
     Name: Root MFA not enabled fails compliance
-    ResourceType: AWS.IAM.RootUser.Snapshot
     ExpectedResult: false
     Resource:
       Arn: arn:aws:iam::123456789012:user/root
@@ -35,7 +34,6 @@ Tests:
       UserName: root
   -
     Name: User MFA not enabled fails compliance
-    ResourceType: AWS.IAM.User.Snapshot
     ExpectedResult: false
     Resource:
       {

diff --git a/tests/fixtures/valid_analysis/policies/example_policy_beta.yml b/tests/fixtures/valid_analysis/policies/example_policy_beta.yml
@@ -19,7 +19,6 @@ Reference: https://www.link-to-info.io
 Tests:
   -
     Name: Root MFA not enabled fails compliance
-    ResourceType: AWS.IAM.RootUser.Snapshot
     ExpectedResult: false
     Resource:
       Arn: arn:aws:iam::123456789012:user/root
@@ -30,7 +29,6 @@ Tests:
       UserName: root
   -
     Name: User MFA not enabled fails compliance
-    ResourceType: AWS.IAM.User.Snapshot
     ExpectedResult: false
     Resource:
       {

diff --git a/tests/fixtures/valid_analysis/policies/example_policy_extraneous_fields.yml b/tests/fixtures/valid_analysis/policies/example_policy_extraneous_fields.yml
@@ -0,0 +1,33 @@
+AnalysisType: policy
+Filename: example_policy.py
+DisplayName: MFA Is Enabled For User
+Description: MFA is a security best practice that adds an extra layer of protection for your AWS account logins.
+Severity: High
+PolicyID: AWS.IAM.MFAEnabled Extra Fields
+Enabled: true
+ResourceTypes:
+  - AWS.IAM.RootUser.Snapshot
+  - AWS.IAM.User.Snapshot
+Tags:
+  - AWS Managed Rules - Security, Identity & Compliance
+  - AWS
+  - CIS
+  - SOC2
+Runbook: >
+  Find out who disabled MFA on the account.
+Reference: https://www.link-to-info.io
+Suppressions:
+  - aws:resource:1
+  - aws:.*:other-resource
+Tests:
+  -
+    Name: Root MFA not enabled triggers a violation.
+    ExpectedResult: false
+    ResourceType: AWS.IAM.User.Snapshot (extraneous field)
+    Resource:
+      Arn: arn:aws:iam::123456789012:user/root
+      CreateDate: 2019-01-01T00:00:00Z
+      CredentialReport:
+        MfaActive: false
+        PasswordEnabled: true
+      UserName: root
diff --git a/tests/fixtures/valid_analysis/rules/example_rule.yml b/tests/fixtures/valid_analysis/rules/example_rule.yml
@@ -18,7 +18,6 @@ Reference: https://www.link-to-info.io
 Tests:
   -
     Name: Root MFA not enabled fails compliance
-    LogType: AWS.CloudTrail
     ExpectedResult: false
     Log:
       Arn: arn:aws:iam::123456789012:user/root
@@ -29,7 +28,6 @@ Tests:
       UserName: root
   -
     Name: User MFA not enabled fails compliance
-    LogType: AWS.CloudTrail
     ExpectedResult: false
     Log:
       {

diff --git a/tests/fixtures/valid_analysis/rules/example_rule_extraneous_fields.yml b/tests/fixtures/valid_analysis/rules/example_rule_extraneous_fields.yml
@@ -0,0 +1,43 @@
+AnalysisType: rule 
+Filename: example_rule.py
+DisplayName: MFA Rule
+Description: MFA is a security best practice that adds an extra layer of protection for your AWS account logins.
+Severity: High
+RuleID: AWS.CloudTrail.MFAEnabled Extra Fields
+Enabled: true
+LogTypes:
+  - AWS.CloudTrail
+Tags:
+  - AWS Managed Rules - Security, Identity & Compliance
+  - AWS
+  - CIS
+  - SOC2
+Runbook: >
+  Find out who disabled MFA on the account.
+Reference: https://www.link-to-info.io
+Tests:
+  -
+    Name: Root MFA not enabled fails compliance
+    LogType: AWS.CloudTrail (extraneous Field)
+    ExpectedResult: false
+    Log:
+      Arn: arn:aws:iam::123456789012:user/root
+      CreateDate: 2019-01-01T00:00:00Z
+      CredentialReport:
+        MfaActive: false
+        PasswordEnabled: true
+      UserName: root
+  -
+    Name: User MFA not enabled fails compliance
+    LogType: AWS.CloudTrail (extraneous Field)
+    ExpectedResult: false
+    Log:
+      {
+        "Arn": "arn:aws:iam::123456789012:user/test",
+        "CreateDate": "2019-01-01T00:00:00",
+        "CredentialReport": {
+          "MfaActive": false,
+          "PasswordEnabled": true
+        },
+        "UserName": "test"
+      }