Feature/additional secrets improvements

pyproject.toml

@@ -4,7 +4,7 @@
 
 [tool.poetry]
     name="pyracf"
-    version="1.0b5"
+    version="1.0b6"
     description="Python interface to RACF using IRRSMO00 RACF Callable Service."
     license = "Apache-2.0"
     authors = [

pyracf/common/exceptions/secrets_redaction_error.py

@@ -0,0 +1,40 @@
+"""Exception to use when Additional Secrets could not be Redacted."""
+
+from typing import List
+
+
+class SecretsRedactionError(Exception):
+    """
+    Raised when a specified secret cannot be redacted because it does not map to a segement:trait.
+    """
+
+    def __init__(
+        self, profile_type: str = "", bad_secret_traits: List[str] = []
+    ) -> None:
+        profile_map = {
+            "user": "User",
+            "group": "Group",
+            "dataSet": "Data Set",
+            "resource": "General Resource",
+            "permission": "Access",
+            "groupConnection": "Group Connection",
+            "systemSettings": "Setropts",
+        }
+        self.message = (
+            f"Cannot add specified additional secrets to {profile_map[profile_type]} "
+            + "administration."
+        )
+
+        if bad_secret_traits:
+            for trait in bad_secret_traits:
+                self.message = self.message + (
+                    f"\nCould not map {trait} to a valid segment trait."
+                )
+        else:
+            self.message = self.message + (
+                f"\n{profile_map[profile_type]} administration does"
+                + " not support additional secrets redaction."
+            )
+
+    def __str__(self) -> str:
+        return self.message

pyracf/common/security_admin.py

@@ -7,6 +7,7 @@
 from typing import Any, List, Tuple, Union
 
 from .exceptions.downstream_fatal_error import DownstreamFatalError
+from .exceptions.secrets_redaction_error import SecretsRedactionError
 from .exceptions.security_request_error import SecurityRequestError
 from .exceptions.segment_error import SegmentError
 from .exceptions.segment_trait_error import SegmentTraitError
@@ -54,6 +55,8 @@ class SecurityAdmin:
 
     _valid_segment_traits = {}
     _extracted_key_value_pair_segment_traits_map = {}
+    # Use this structure to map traits to their RACF keywords and Message keywords for redaction
+    _racf_trait_and_message_key_map = {"trait_map": {}, "message_map": {}}
     _case_sensitive_extracted_values = []
     __running_userid = None
     _logger = Logger()
@@ -170,24 +173,39 @@ def __raw_dump(self) -> None:
         if self.__debug:
             # Note, since the hex dump is logged to the console,
             # secrets will be redacted.
-            self._logger.log_hex_dump(raw_result_xml, self.__secret_traits)
+            self._logger.log_hex_dump(
+                raw_result_xml,
+                self.__secret_traits,
+                self._racf_trait_and_message_key_map,
+            )
 
     # ============================================================================
     # Secrets Redaction
     # ============================================================================
     def __add_additional_secret_traits(self, additional_secret_traits: list) -> None:
         """Add additional fields to be redacted in logger output."""
+        unsupported_profile_types = ["permission", "groupConnection", "systemSettings"]
+        if self._profile_type in unsupported_profile_types:
+            raise SecretsRedactionError(profile_type=self._profile_type)
+        bad_secret_traits = []
         for secret in additional_secret_traits:
             if secret in self.__secret_traits:
                 continue
             if ":" not in secret:
+                bad_secret_traits.append(secret)
                 continue
             segment = secret.split(":")[0]
             if segment not in self._valid_segment_traits:
+                bad_secret_traits.append(secret)
                 continue
             if secret not in self._valid_segment_traits[segment]:
+                bad_secret_traits.append(secret)
                 continue
             self.__secret_traits[secret] = self._valid_segment_traits[segment][secret]
+        if bad_secret_traits:
+            raise SecretsRedactionError(
+                profile_type=self._profile_type, bad_secret_traits=bad_secret_traits
+            )
 
     # ============================================================================
     # Request Execution
@@ -242,6 +260,7 @@ def _make_request(
                 self.__running_userid,
             ),
             self.__secret_traits,
+            self._racf_trait_and_message_key_map,
         )
         self.__clear_state(security_request)
         if isinstance(raw_result, list):

pyracf/common/utilities/logger.py

@@ -149,49 +149,118 @@ def __redact_request_dictionary(
     def __redact_string(
         self,
         input_string: Union[str, bytes],
-        start_ind: int,
-        end_pattern: Union[str, bytes],
-    ):
-        """
-        Redacts characters in a string between a starting index and ending pattern.
+        trait_key: str,
+    ) -> Union[str, bytes]:
+        r"""
+        Redacts characters in a string between a starting index and ending tag.
         Replaces the identified characters with '********' regardless of the original length.
+
+        This function employs the following regular expressions explained below
+
+        Regex 1 ("quoted") - {trait_key.upper()}( +){{0,}}\(\'.*?(?<!\')\'\)
+        This is designed to match the pattern TRAIT('value') by matching the TRAIT name, a
+        variable (potentially zero) amount of spaces, then the ('value') portion which must
+        start and end with (' and '), but can conceivably contain any characters, but a negative
+        lookbehind is used to look for any unescaped single quotes that would indicate the matching
+        of ') is otherwise a coincidence.
+
+        Regex 2 ("nested") - {trait_key.upper()}( +){{0,}}\([^)]*\(.*\)( +){{0,}}\)
+        This is designed to match the pattern TRAIT( subtrait1(value) subtrait2(value)) by
+        matching the TRAIT name, a variable (potentially zero) amount of spaces, then the
+        ( subtrait1(value) subtrait2(value)) portion which must start and end with ( and ),
+        but must also contain a nested set of open and close parentheses rather than a direct
+        seqence of them. The pattern looks for these nested open parenthesis as a sequence would
+        have a ) character between them. Then the expression allows any non-newline characters and
+        the "end pattern" of ) and ) separated by a variable (potentially zero) whitespace.
+
+        If neither of these two patterns is found for a supplied trait_key, then it is assumed
+        this trait is set with the default pattern below.
+
+        Regex 3 ("default") - {trait_key.upper()}( +){{0,}}\(.*?(?<!\\)\)
+        This is designed to match the pattern TRAIT(value) by matching the TRAIT name, a variable
+        (potentially zero) amount of spaces, then the (value) portion which must start and end
+        with ( and ), but can conceivably contain any characters, but a negative lookbehind
+        is used to look for any escape character `\` that would indicate the matching of the
+        ( and ) is otherwise a coincidence.
+
+        In all replacement expressions, the variable amounts of whitespace are captured so that
+        they can be preserved by this redaction operations.
+
         """
         asterisks = "********"
+        is_bytes = False
+        quoted_regex = rf"{trait_key.upper()}( +){{0,}}\(\'.*?(?<!\')\'\)"
+        nested_regex = rf"{trait_key.upper()}( +){{0,}}\([^)]*\(.*\)( +){{0,}}\)"
+        default_regex = rf"{trait_key.upper()}( +){{0,}}\(.*?(?<!\\)\)"
+        redacted_regex = rf"{trait_key.upper()}\1({asterisks})"
+        redacted_w_quotes_regex = rf"{trait_key.upper()}\1('{asterisks}')"
         if isinstance(input_string, bytes):
-            asterisks = asterisks.encode("cp1047")
-        pre_keyword = input_string[:start_ind]
-        post_keyword = end_pattern.join(input_string[start_ind:].split(end_pattern)[1:])
-        return pre_keyword + asterisks + end_pattern + post_keyword
+            input_string = input_string.decode("cp1047")
+            is_bytes = True
+        quoted = re.search(quoted_regex, input_string)
+        nested = re.search(nested_regex, input_string)
+        if quoted is not None:
+            input_string = re.sub(
+                quoted_regex,
+                redacted_w_quotes_regex,
+                input_string,
+            )
+        else:
+            if nested is not None:
+                input_string = re.sub(
+                    nested_regex,
+                    redacted_regex,
+                    input_string,
+                )
+            else:
+                input_string = re.sub(
+                    default_regex,
+                    redacted_regex,
+                    input_string,
+                )
+        if is_bytes:
+            return input_string.encode("cp1047")
+        return input_string
 
     def redact_request_xml(
         self,
         xml_string: Union[str, bytes],
         secret_traits: dict,
     ) -> Union[str, bytes]:
-        """
+        r"""
         Redact a list of specific secret traits in a request xml string or bytes object.
         Based the following xml pattern:
             '<xmltag attribute="any">xml value</xmltag>'
         This function also accounts for any number of arbitrary xml attributes.
+
+        This function employs the following regular expression:
+        {xml_key}(.*)>.*<\/{xml_key} - Designed to match the above pattern by starting and ending
+        with the xmltag string as shown, but the starting tag allows for any characters between
+        "xmltag" and the > character to allow for the attribute specification shown above. This
+        results in the starting of the xml as {xml_key}(.*)> and the ending as <\/{xml_key}.
+        The characters between the xml tags are "captured" as a variable and preserved by the
+        substitution operation through the use of the \1 supplied in the replacement string.
+        Between these tags, any non-newline characters are allowed using the .* expression.
         """
         is_bytes = False
         if isinstance(xml_string, bytes):
             is_bytes = True
             xml_string = xml_string.decode("utf-8")
         for xml_key in secret_traits.values():
-            match = re.search(rf"\<{xml_key}+[^>]*\>", xml_string)
-            if not match:
-                continue
+            start_tag_eng_tag_regex = rf"{xml_key}(.*)>.*<\/{xml_key}"
+            redacted_regex = rf"{xml_key}\1>********</{xml_key}"
             # Delete operation has no end tag and and redaction should not be attempted.
             #
             # Redact this:
             # <tag operation="set">secret</tag>
             #
             # Don't try to redact this:
             # <tag operation="del" />
-            if f"</{xml_key}>" not in xml_string:
-                continue
-            xml_string = self.__redact_string(xml_string, match.end(), f"</{xml_key}")
+            xml_string = re.sub(
+                start_tag_eng_tag_regex,
+                redacted_regex,
+                xml_string,
+            )
         if is_bytes:
             xml_string = xml_string.encode("utf-8")
         return xml_string
@@ -200,30 +269,54 @@ def redact_result_xml(
         self,
         security_result: Union[str, bytes, List[int]],
         secret_traits: dict,
+        racf_trait_and_message_key_map: dict = {},
     ) -> str:
         """
         Redacts a list of specific secret traits in a result xml string.
-        Based on the following RACF command pattern:
+        Based on the following RACF command patterns:
             'TRAIT (value)'
+            'TRAIT (subtrait1(value) subtrait2(value))
+            "TRAIT ('value')"
         This function also accounts for varied amounts of whitespace in the pattern.
         """
         if isinstance(security_result, list):
             return security_result
         for xml_key in secret_traits.values():
             racf_key = xml_key.split(":")[1] if ":" in xml_key else xml_key
-            end_pattern = ")"
+            if racf_key in racf_trait_and_message_key_map.get("trait_map", {}):
+                racf_key = racf_trait_and_message_key_map["trait_map"][racf_key]
+            racf_command_argument_regex = rf"{racf_key.upper()}( +){{0,}}\("
             if isinstance(security_result, bytes):
                 match = re.search(
-                    rf"{racf_key.upper()} +\(", security_result.decode("cp1047")
+                    racf_command_argument_regex, security_result.decode("cp1047")
                 )
-                end_pattern = end_pattern.encode("cp1047")
             else:
-                match = re.search(rf"{racf_key.upper()} +\(", security_result)
+                match = re.search(rf"{racf_key.upper()}( +){{0,}}\(", security_result)
             if not match:
                 continue
-            security_result = self.__redact_string(
-                security_result, match.end(), end_pattern
+            security_result = self.__redact_string(security_result, racf_key)
+            if racf_key in racf_trait_and_message_key_map.get("message_map", {}):
+                racf_key = racf_trait_and_message_key_map["message_map"][racf_key]
+            racf_message_regex = (
+                r"<message>([A-Z]*[0-9]*[A-Z]) [^<>]*"
+                + rf"{racf_key.upper()}[^<>]*<\/message>"
             )
+            redacted_racf_message_regex = (
+                rf"<message>REDACTED MESSAGE CONCERNING {racf_key.upper()}, "
+                + r"REVIEW DOCUMENTATION OF \1 FOR MORE INFORMATION</message>"
+            )
+            if isinstance(security_result, bytes):
+                security_result = re.sub(
+                    racf_message_regex,
+                    redacted_racf_message_regex,
+                    security_result.decode("cp1047"),
+                ).encode("cp1047")
+            else:
+                security_result = re.sub(
+                    racf_message_regex,
+                    redacted_racf_message_regex,
+                    security_result,
+                )
         return security_result
 
     def __colorize_json(self, json_text: str) -> str:
@@ -355,7 +448,12 @@ def __indent_xml(self, minified_xml: str) -> str:
             indented_xml += f"{'  ' * indent_level}{current_line}\n"
         return indented_xml[:-2]
 
-    def log_hex_dump(self, raw_result_xml: bytes, secret_traits: dict) -> None:
+    def log_hex_dump(
+        self,
+        raw_result_xml: bytes,
+        secret_traits: dict,
+        racf_trait_and_message_key_map: dict,
+    ) -> None:
         """
         Log the raw result XML returned by IRRSMO00 as a hex dump.
         """
@@ -368,6 +466,7 @@ def log_hex_dump(self, raw_result_xml: bytes, secret_traits: dict) -> None:
         raw_result_xml = self.redact_result_xml(
             raw_result_xml,
             secret_traits,
+            racf_trait_and_message_key_map,
         )
         for byte in raw_result_xml:
             color_function = self.__green

pyracf/data_set/data_set_admin.py

@@ -63,6 +63,16 @@ def __init__(
             "dfp": {"dfp:owner": "racf:resowner", "dfp:ckds_data_key": "racf:datakey"},
             "tme": {"tme:roles": "racf:roles"},
         }
+        self._racf_trait_and_message_key_map["trait_map"] = {
+            "audaltr": "audit",
+            "audcntl": "audit",
+            "audnone": "audit",
+            "audread": "audit",
+            "audupdt": "audit",
+        }
+        self._racf_trait_and_message_key_map["message_map"] = {
+            "uacc": "universal access"
+        }
         self._valid_segment_traits["base"].update(
             self._common_base_traits_data_set_generic
         )

pyracf/resource/resource_admin.py

@@ -238,6 +238,16 @@ def __init__(
                 "sigrequired": "signatureRequired",
             },
         }
+        self._racf_trait_and_message_key_map["trait_map"] = {
+            "audaltr": "audit",
+            "audcntl": "audit",
+            "audnone": "audit",
+            "audread": "audit",
+            "audupdt": "audit",
+        }
+        self._racf_trait_and_message_key_map["message_map"] = {
+            "uacc": "universal access"
+        }
         super().__init__(
             "resource",
             irrsmo00_result_buffer_size=irrsmo00_result_buffer_size,

tests/user/user_log_samples/alter_user_additional_secret_added_error.log → tests/common/common_log_samples/alter_user_additional_secret_added_error.log

@@ -240,9 +240,9 @@
       <returncode>16</returncode>
       <reasoncode>8</reasoncode>
       <image>ALTUSER SQUIDWRD  NOSPECIAL      OMVS     (HOME        ('/u/clarinet') NOPROGRAM      UID         (********))</image>
-      <message>IKJ56702I INVALID UID, 90000000000</message>
-      <message>IKJ56701I MISSING OMVS UID+</message>
-      <message>IKJ56701I MISSING OMVS USER ID (UID), 1-10 NUMERIC DIGITS</message>
+      <message>REDACTED MESSAGE CONCERNING UID, REVIEW DOCUMENTATION OF IKJ56702I FOR MORE INFORMATION</message>
+      <message>REDACTED MESSAGE CONCERNING UID, REVIEW DOCUMENTATION OF IKJ56701I FOR MORE INFORMATION</message>
+      <message>REDACTED MESSAGE CONCERNING UID, REVIEW DOCUMENTATION OF IKJ56701I FOR MORE INFORMATION</message>
     </command>
   </user>
   <returncode>4</returncode>
@@ -271,9 +271,9 @@
           "reasonCode": 8,
           "image": "ALTUSER SQUIDWRD  NOSPECIAL      OMVS     (HOME        ('/u/clarinet') NOPROGRAM      UID         (********))",
           "messages": [
-            "IKJ56702I INVALID UID, 90000000000",
-            "IKJ56701I MISSING OMVS UID+",
-            "IKJ56701I MISSING OMVS USER ID (UID), 1-10 NUMERIC DIGITS"
+            "REDACTED MESSAGE CONCERNING UID, REVIEW DOCUMENTATION OF IKJ56702I FOR MORE INFORMATION",
+            "REDACTED MESSAGE CONCERNING UID, REVIEW DOCUMENTATION OF IKJ56701I FOR MORE INFORMATION",
+            "REDACTED MESSAGE CONCERNING UID, REVIEW DOCUMENTATION OF IKJ56701I FOR MORE INFORMATION"
           ]
         }
       ]