Feature/additional secrets improvements

pyproject.toml

@@ -4,7 +4,7 @@
 
 [tool.poetry]
     name="pyracf"
-    version="1.0b5"
+    version="1.0b6"
     description="Python interface to RACF using IRRSMO00 RACF Callable Service."
     license = "Apache-2.0"
     authors = [

pyracf/common/security_admin.py

@@ -54,6 +54,10 @@ class SecurityAdmin:
 
     _valid_segment_traits = {}
     _extracted_key_value_pair_segment_traits_map = {}
+    _racf_key_map = {}
+    _racf_message_key_map = {}
+    _racf_key_map = {}
+    _racf_message_key_map = {}
     _case_sensitive_extracted_values = []
     __running_userid = None
     _logger = Logger()
@@ -170,13 +174,21 @@ def __raw_dump(self) -> None:
         if self.__debug:
             # Note, since the hex dump is logged to the console,
             # secrets will be redacted.
-            self._logger.log_hex_dump(raw_result_xml, self.__secret_traits)
+            self._logger.log_hex_dump(
+                raw_result_xml,
+                self.__secret_traits,
+                self._racf_key_map,
+                self._racf_message_key_map,
+            )
 
     # ============================================================================
     # Secrets Redaction
     # ============================================================================
     def __add_additional_secret_traits(self, additional_secret_traits: list) -> None:
         """Add additional fields to be redacted in logger output."""
+        unsupported_profile_types = ["permission", "groupConnection", "systemSettings"]
+        if self._profile_type in unsupported_profile_types:
+            return
         for secret in additional_secret_traits:
             if secret in self.__secret_traits:
                 continue
@@ -242,6 +254,8 @@ def _make_request(
                 self.__running_userid,
             ),
             self.__secret_traits,
+            self._racf_key_map,
+            self._racf_message_key_map,
         )
         self.__clear_state(security_request)
         if isinstance(raw_result, list):

pyracf/common/utilities/logger.py

@@ -149,30 +149,97 @@ def __redact_request_dictionary(
     def __redact_string(
         self,
         input_string: Union[str, bytes],
-        start_ind: int,
-        end_pattern: Union[str, bytes],
-    ):
-        """
-        Redacts characters in a string between a starting index and ending pattern.
+        trait_key: str,
+    ) -> Union[str, bytes]:
+        r"""
+        Redacts characters in a string between a starting index and ending tag.
         Replaces the identified characters with '********' regardless of the original length.
+
+        This function employs the following regular expressions explained below
+
+        Regex 1 ("quoted") - {trait_key.upper()}( +){{0,}}\(\'.*?(?<!\')\'\)
+        This is designed to match the pattern TRAIT('value') by matching the TRAIT name, a
+        variable (potentially zero) amount of spaces, then the ('value') portion which must
+        start and end with (' and '), but can conceivably contain any characters, but a negative
+        lookbehind is used to look for any unescaped single quotes that would indicate the matching
+        of ') is otherwise a coincidence.
+
+        Regex 2 ("nested") - {trait_key.upper()}( +){{0,}}\([^)]*\(.*\)( +){{0,}}\)
+        This is designed to match the pattern TRAIT( subtrait1(value) subtrait2(value)) by
+        matching the TRAIT name, a variable (potentially zero) amount of spaces, then the
+        ( subtrait1(value) subtrait2(value)) portion which must start and end with ( and ),
+        but must also contain a nested set of opened parentheses rather than a direct seqence of
+        them. The pattern looks for these nested open parenthesis as a sequence would have a )
+        character between them. Then the expression allows any non-newline characters and the
+        "end pattern" of ) and ) separated by a variable (potentially zero) whitespace.
+
+        If neither of these two patterns is found for a supplied trait_key, then it is assumed
+        this trait is set with the default pattern below.
+
+        Regex 3 ("default") - {trait_key.upper()}( +){{0,}}\(.*?(?<!\\)\)
+        This is designed to match the pattern TRAIT(value) by matching the TRAIT name, a variable
+        (potentially zero) amount of spaces, then the (value) portion which must start and end
+        with ( and ), but can conceivably contain any characters, but a negative lookbehind
+        is used to look for any escape \ character that would indicate the matching of the
+        ( and ) is otherwise a coincidence.
+
+        In all replacement expressions, the variable amounts of whitespace are captured so that
+        they can be preserved by this redaction operations.
+
         """
         asterisks = "********"
+        is_bytes = False
         if isinstance(input_string, bytes):
-            asterisks = asterisks.encode("cp1047")
-        pre_keyword = input_string[:start_ind]
-        post_keyword = end_pattern.join(input_string[start_ind:].split(end_pattern)[1:])
-        return pre_keyword + asterisks + end_pattern + post_keyword
+            input_string = input_string.decode("cp1047")
+            is_bytes = True
+        quoted = re.search(
+            rf"{trait_key.upper()}( +){{0,}}\(\'.*?(?<!\')\'\)", input_string
+        )
+        nested = re.search(
+            rf"{trait_key.upper()}( +){{0,}}\([^)]*\(.*\)( +){{0,}}\)", input_string
+        )
+        if quoted is not None:
+            input_string = re.sub(
+                rf"{trait_key.upper()}( +){{0,}}\(\'.*?(?<!\')\'\)",
+                rf"{trait_key.upper()}\1('{asterisks}')",
+                input_string,
+            )
+        else:
+            if nested is not None:
+                input_string = re.sub(
+                    rf"{trait_key.upper()}( +){{0,}}\([^)]*\(.*\)( +){{0,}}\)",
+                    rf"{trait_key.upper()}\1({asterisks})",
+                    input_string,
+                )
+            else:
+                input_string = re.sub(
+                    rf"{trait_key.upper()}( +){{0,}}\(.*?(?<!\\)\)",
+                    rf"{trait_key.upper()}\1({asterisks})",
+                    input_string,
+                )
+        if is_bytes:
+            return input_string.encode("cp1047")
+        return input_string
 
     def redact_request_xml(
         self,
         xml_string: Union[str, bytes],
         secret_traits: dict,
     ) -> Union[str, bytes]:
-        """
+        r"""
         Redact a list of specific secret traits in a request xml string or bytes object.
         Based the following xml pattern:
             '<xmltag attribute="any">xml value</xmltag>'
         This function also accounts for any number of arbitrary xml attributes.
+
+        This function employs the following regular expression:
+        {xml_key}(.*)>.*<\/{xml_key} - Designed to match the above pattern by starting and ending
+        with the xmltag string as shown, but the starting tage allows for any characters between
+        "xmltag" and the > character to allow for the attribute specification shown above. This
+        results in the starting of the xml as {xml_key}(.*)> and the ending as <\/{xml_key}.
+        The miscellaneous characters are "captured" as a variable and preserved by the
+        substitution operation through the use of the \1 supplied in the replacement string.
+        Between these tags, any non-newline characters are allowed using the .* expression.
         """
         is_bytes = False
         if isinstance(xml_string, bytes):
@@ -191,7 +258,11 @@ def redact_request_xml(
             # <tag operation="del" />
             if f"</{xml_key}>" not in xml_string:
                 continue
-            xml_string = self.__redact_string(xml_string, match.end(), f"</{xml_key}")
+            xml_string = re.sub(
+                rf"{xml_key}(.*)>.*<\/{xml_key}",
+                rf"{xml_key}\1>********</{xml_key}",
+                xml_string,
+            )
         if is_bytes:
             xml_string = xml_string.encode("utf-8")
         return xml_string
@@ -200,30 +271,49 @@ def redact_result_xml(
         self,
         security_result: Union[str, bytes, List[int]],
         secret_traits: dict,
+        racf_key_map: dict = {},
+        racf_message_key_map: dict = {},
     ) -> str:
         """
         Redacts a list of specific secret traits in a result xml string.
-        Based on the following RACF command pattern:
+        Based on the following RACF command patterns:
             'TRAIT (value)'
+            'TRAIT (subtrait1(value) subtrait2(value))
+            "TRAIT ('value')"
         This function also accounts for varied amounts of whitespace in the pattern.
         """
         if isinstance(security_result, list):
             return security_result
         for xml_key in secret_traits.values():
             racf_key = xml_key.split(":")[1] if ":" in xml_key else xml_key
-            end_pattern = ")"
+            if racf_key in racf_key_map:
+                print(racf_key_map[racf_key])
+                racf_key = racf_key_map[racf_key]
             if isinstance(security_result, bytes):
                 match = re.search(
-                    rf"{racf_key.upper()} +\(", security_result.decode("cp1047")
+                    rf"{racf_key.upper()}( +){{0,}}\(", security_result.decode("cp1047")
                 )
-                end_pattern = end_pattern.encode("cp1047")
             else:
-                match = re.search(rf"{racf_key.upper()} +\(", security_result)
+                match = re.search(rf"{racf_key.upper()}( +){{0,}}\(", security_result)
             if not match:
                 continue
-            security_result = self.__redact_string(
-                security_result, match.end(), end_pattern
-            )
+            security_result = self.__redact_string(security_result, racf_key)
+            if racf_key in racf_message_key_map:
+                racf_key = racf_message_key_map[racf_key]
+            if isinstance(security_result, bytes):
+                security_result = re.sub(
+                    rf"<message>([A-Z]*[0-9]*[A-Z]) [^<>]*{racf_key.upper()}[^<>]*<\/message>",
+                    rf"<message>REDACTED MESSAGE CONCERNING {racf_key.upper()}, "
+                    + r"REVIEW DOCUMENTATION OF \1 FOR MORE INFORMATION</message>",
+                    security_result.decode("cp1047"),
+                ).encode("cp1047")
+            else:
+                security_result = re.sub(
+                    rf"<message>([A-Z]*[0-9]*[A-Z]) [^<>]*{racf_key.upper()}[^<>]*<\/message>",
+                    rf"<message>REDACTED MESSAGE CONCERNING {racf_key.upper()}, "
+                    + r"REVIEW DOCUMENTATION OF \1 FOR MORE INFORMATION</message>",
+                    security_result,
+                )
         return security_result
 
     def __colorize_json(self, json_text: str) -> str:
@@ -355,7 +445,13 @@ def __indent_xml(self, minified_xml: str) -> str:
             indented_xml += f"{'  ' * indent_level}{current_line}\n"
         return indented_xml[:-2]
 
-    def log_hex_dump(self, raw_result_xml: bytes, secret_traits: dict) -> None:
+    def log_hex_dump(
+        self,
+        raw_result_xml: bytes,
+        secret_traits: dict,
+        racf_key_map: dict,
+        racf_message_key_map: dict,
+    ) -> None:
         """
         Log the raw result XML returned by IRRSMO00 as a hex dump.
         """
@@ -368,6 +464,8 @@ def log_hex_dump(self, raw_result_xml: bytes, secret_traits: dict) -> None:
         raw_result_xml = self.redact_result_xml(
             raw_result_xml,
             secret_traits,
+            racf_key_map,
+            racf_message_key_map,
         )
         for byte in raw_result_xml:
             color_function = self.__green

pyracf/data_set/data_set_admin.py

@@ -63,6 +63,14 @@ def __init__(
             "dfp": {"dfp:owner": "racf:resowner", "dfp:ckds_data_key": "racf:datakey"},
             "tme": {"tme:roles": "racf:roles"},
         }
+        self._racf_key_map = {
+            "audaltr": "audit",
+            "audcntl": "audit",
+            "audnone": "audit",
+            "audread": "audit",
+            "audupdt": "audit",
+        }
+        self._racf_message_key_map = {"uacc": "universal access"}
         self._valid_segment_traits["base"].update(
             self._common_base_traits_data_set_generic
         )

pyracf/resource/resource_admin.py

@@ -238,6 +238,14 @@ def __init__(
                 "sigrequired": "signatureRequired",
             },
         }
+        self._racf_key_map = {
+            "audaltr": "audit",
+            "audcntl": "audit",
+            "audnone": "audit",
+            "audread": "audit",
+            "audupdt": "audit",
+        }
+        self._racf_message_key_map = {"uacc": "universal access"}
         super().__init__(
             "resource",
             irrsmo00_result_buffer_size=irrsmo00_result_buffer_size,

tests/common/common_result_samples/alter_data_set_result_error_uacc_secret.json

@@ -0,0 +1,33 @@
+{
+  "securityResult": {
+    "dataSet": {
+      "name": "ESWIFT.TEST.T113622.P3020470",
+      "operation": "set",
+      "generic": "no",
+      "requestId": "DatasetRequest",
+      "commands": [
+        {
+          "safReturnCode": 8,
+          "returnCode": 16,
+          "reasonCode": 4,
+          "image": "ADDSD                ('ESWIFT.TEST.T113622.P3020470')",
+          "messages": [
+            "ICH09005I ESWIFT.TEST.T113622.P3020470 NOT FOUND IN CATALOG"
+          ]
+        },
+        {
+          "safReturnCode": 8,
+          "returnCode": 16,
+          "reasonCode": 4,
+          "image": "ALTDSD               ('ESWIFT.TEST.T113622.P3020470')  UACC        (********)",
+          "messages": [
+            "ICH22001I ESWIFT.TEST.T113622.P3020470 NOT DEFINED TO RACF"
+          ]
+        }
+      ]
+    },
+    "returnCode": 4,
+    "reasonCode": 0,
+    "runningUserid": "testuser"
+  }
+}

tests/common/common_result_samples/alter_data_set_result_success_owner_secret.json

@@ -0,0 +1,24 @@
+{
+  "securityResult": {
+    "dataSet": {
+      "name": "ESWIFT.TEST.T1136242.P3020470",
+      "operation": "set",
+      "generic": "no",
+      "requestId": "DatasetRequest",
+      "info": [
+        "Definition exists. Add command skipped due  to precheck option"
+      ],
+      "commands": [
+        {
+          "safReturnCode": 0,
+          "returnCode": 0,
+          "reasonCode": 0,
+          "image": "ALTDSD               ('ESWIFT.TEST.T1136242.P3020470')  UACC        (Read) OWNER       (********)"
+        }
+      ]
+    },
+    "returnCode": 0,
+    "reasonCode": 0,
+    "runningUserid": "testuser"
+  }
+}

tests/common/common_result_samples/alter_group_result_error_gid_secret.json

@@ -0,0 +1,28 @@
+{
+  "securityResult": {
+    "group": {
+      "name": "TESTGRP0",
+      "operation": "set",
+      "requestId": "GroupRequest",
+      "info": [
+        "Definition exists. Add command skipped due  to precheck option"
+      ],
+      "commands": [
+        {
+          "safReturnCode": 8,
+          "returnCode": 16,
+          "reasonCode": 8,
+          "image": "ALTGROUP TESTGRP0  OMVS     (GID         (********))",
+          "messages": [
+            "REDACTED MESSAGE CONCERNING GID, REVIEW DOCUMENTATION OF IKJ56702I FOR MORE INFORMATION",
+            "REDACTED MESSAGE CONCERNING GID, REVIEW DOCUMENTATION OF IKJ56701I FOR MORE INFORMATION",
+            "REDACTED MESSAGE CONCERNING GID, REVIEW DOCUMENTATION OF IKJ56701I FOR MORE INFORMATION"
+          ]
+        }
+      ]
+    },
+    "returnCode": 4,
+    "reasonCode": 0,
+    "runningUserid": "testuser"
+  }
+}