[Gh 884] IAM policy splitting for requestor IAM policies

backend/dataall/base/aws/iam.py

@@ -1,5 +1,6 @@
 import logging
 from botocore.exceptions import ClientError
+import re
 
 from .sts import SessionHelper
 
@@ -66,6 +67,25 @@ def get_role_policy(
             log.error(f'Failed to get policy {policy_name} of role {role_name} : {e}')
             return None
 
+    @staticmethod
+    def list_policy_names_by_policy_pattern(account_id: str, region: str, policy_filter_pattern: str):
+        try:
+            client = IAM.client(account_id, region)
+            # Setting Scope to 'Local' to fetch all the policies created in this account
+            paginator = client.get_paginator('list_policies')
+            policies = []
+            for page in paginator.paginate(Scope='Local'):
+                policies.extend(page['Policies'])
+            policy_names = [policy.get('PolicyName') for policy in policies]
+            return [policy_nm for policy_nm in policy_names if re.search(policy_filter_pattern, policy_nm)]
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'AccessDenied':
+                raise Exception(
+                    f'Data.all Environment Pivot Role does not have permissions to get policies with pattern {policy_filter_pattern} due to: {e}'
+                )
+            log.error(f'Failed to get policies for policy pattern due to: {e}')
+            return []
+
     @staticmethod
     def delete_role_policy(
         account_id: str,
@@ -101,6 +121,23 @@ def get_managed_policy_by_name(account_id: str, region: str, policy_name: str):
             log.error(f'Failed to get policy {policy_name}: {e}')
             return None
 
+    @staticmethod
+    def get_managed_policy_document_by_name(account_id: str, region: str, policy_name: str):
+        try:
+            arn = f'arn:aws:iam::{account_id}:policy/{policy_name}'
+            client = IAM.client(account_id, region)
+            policy = IAM.get_managed_policy_by_name(account_id, region, policy_name)
+            policyVersionId = policy['DefaultVersionId']
+            response = client.get_policy_version(PolicyArn=arn, VersionId=policyVersionId)
+            return response['PolicyVersion']['Document']
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'AccessDenied':
+                raise Exception(
+                    f'Data.all Environment Pivot Role does not have permissions to to get policy {policy_name}: {e}'
+                )
+            log.error(f'Failed to get policy {policy_name}: {e}')
+            return None
+
     @staticmethod
     def create_managed_policy(account_id: str, region: str, policy_name: str, policy: str):
         try:
@@ -275,3 +312,16 @@ def remove_invalid_role_ids(account_id: str, region: str, principal_list):
             if 'AROA' in p_id:
                 if p_id not in all_role_ids:
                     principal_list.remove(p_id)
+
+    @staticmethod
+    def get_attached_managed_policies_to_role(account_id: str, region: str, role_name: str):
+        try:
+            client = IAM.client(account_id, region)
+            response = client.list_attached_role_policies(RoleName=role_name)
+            return [policy.get('PolicyName') for policy in response['AttachedPolicies']]
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'AccessDenied':
+                raise Exception(
+                    f'Data.all Environment Pivot Role does not have permissions to get attached managed policies for {role_name}: {e}'
+                )
+            raise Exception(f'Failed to get attached managed policies for {role_name}: {e}')

backend/dataall/base/aws/service_quota.py

@@ -0,0 +1,58 @@
+import logging
+from botocore.exceptions import ClientError
+
+from .sts import SessionHelper
+
+log = logging.getLogger(__name__)
+
+
+class ServiceQuota:
+    def __init__(self, account_id, region):
+        session = SessionHelper.remote_session(accountid=account_id, region=region)
+        self.client = session.client('service-quotas')
+
+    def list_services(self):
+        try:
+            log.info('Fetching services list with service codes in aws account')
+            services_list = []
+            paginator = self.client.get_paginator('list_services')
+            for page in paginator.paginate():
+                services_list.extend(page.get('Services'))
+            return services_list
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'AccessDenied':
+                raise Exception(f'Data.all Environment Pivot Role does not have permissions to do list_services : {e}')
+            log.error(f'Failed list services and service codes due to: {e}')
+            return []
+
+    def list_service_quota(self, service_code):
+        try:
+            log.info('Fetching services quota code in aws account')
+            service_quota_code_list = []
+            paginator = self.client.get_paginator('list_service_quotas')
+            for page in paginator.paginate(ServiceCode=service_code):
+                service_quota_code_list.extend(page.get('Quotas'))
+            log.debug(f'Services quota list: {service_quota_code_list}')
+            return service_quota_code_list
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'AccessDenied':
+                raise Exception(
+                    f'Data.all Environment Pivot Role does not have permissions to do list_service_quota : {e}'
+                )
+            log.error(f'Failed list quota codes to: {e}')
+            return []
+
+    def get_service_quota_value(self, service_code, service_quota_code):
+        try:
+            log.info(
+                f'Getting service quota for service code: {service_code} and service quota code: {service_quota_code}'
+            )
+            response = self.client.get_service_quota(ServiceCode=service_code, QuotaCode=service_quota_code)
+            return response['Quota']['Value']
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'AccessDenied':
+                raise Exception(
+                    f'Data.all Environment Pivot Role does not have permissions to do get_service_quota: {e}'
+                )
+            log.error(f'Failed list quota codes to: {e}')
+            return None

backend/dataall/base/db/exceptions.py

@@ -147,6 +147,18 @@ def __str__(self):
         return f'{self.message}'
 
 
+class AWSServiceQuotaExceeded(Exception):
+    def __init__(self, action, message):
+        self.action = action
+        self.message = f"""
+                    An error occurred (AWSResourceQuotaExceeded) when calling {self.action} operation:
+                    {message}
+                """
+
+    def __str__(self):
+        return f'{self.message}'
+
+
 class EnvironmentResourcesFound(Exception):
     def __init__(self, action, message):
         self.action = action

backend/dataall/base/utils/iam_cdk_utils.py

@@ -0,0 +1,71 @@
+from typing import Dict, Any, List
+from aws_cdk import aws_iam as iam
+
+from dataall.base.utils.iam_policy_utils import (
+    split_policy_statements_in_chunks,
+    split_policy_with_resources_in_statements,
+    split_policy_with_mutiple_value_condition_in_statements,
+)
+
+
+def convert_from_json_to_iam_policy_statement_with_conditions(iam_policy: Dict[Any, Any]):
+    return iam.PolicyStatement(
+        sid=iam_policy.get('Sid'),
+        effect=iam.Effect.ALLOW if iam_policy.get('Effect').casefold() == 'Allow'.casefold() else iam.Effect.DENY,
+        actions=_convert_to_array(str, iam_policy.get('Action')),
+        resources=_convert_to_array(str, iam_policy.get('Resource')),
+        conditions=iam_policy.get('Condition'),
+    )
+
+
+def convert_from_json_to_iam_policy_statement(iam_policy: Dict[Any, Any]):
+    return iam.PolicyStatement(
+        sid=iam_policy.get('Sid'),
+        effect=iam.Effect.ALLOW if iam_policy.get('Effect').casefold() == 'Allow'.casefold() else iam.Effect.DENY,
+        actions=_convert_to_array(str, iam_policy.get('Action')),
+        resources=_convert_to_array(str, iam_policy.get('Resource')),
+    )
+
+
+def process_and_split_statements_in_chunks(statements: List[Dict]):
+    statement_chunks_json: List[List[Dict]] = split_policy_statements_in_chunks(statements)
+    statements_chunks: List[List[iam.PolicyStatement]] = []
+    for statement_js_chunk in statement_chunks_json:
+        statements: List[iam.PolicyStatement] = []
+        for statement in statement_js_chunk:
+            if statement.get('Condition', None):
+                statements.append(convert_from_json_to_iam_policy_statement_with_conditions(statement))
+            else:
+                statements.append(convert_from_json_to_iam_policy_statement(statement))
+        statements_chunks.append(statements)
+    return statements_chunks
+
+
+def process_and_split_policy_with_resources_in_statements(
+    base_sid: str, effect: str, actions: List[str], resources: List[str], condition_dict: Dict = None
+):
+    if condition_dict is not None:
+        print(f'Condition dictionary is: {condition_dict}')
+        json_statements = split_policy_with_mutiple_value_condition_in_statements(
+            base_sid=base_sid, effect=effect, actions=actions, resources=resources, condition_dict=condition_dict
+        )
+    else:
+        json_statements = split_policy_with_resources_in_statements(
+            base_sid=base_sid, effect=effect, actions=actions, resources=resources
+        )
+    iam_statements: [iam.PolicyStatement] = []
+    for json_statement in json_statements:
+        if json_statement.get('Condition', None):
+            iam_policy_statement = convert_from_json_to_iam_policy_statement_with_conditions(json_statement)
+        else:
+            iam_policy_statement = convert_from_json_to_iam_policy_statement(json_statement)
+        iam_statements.append(iam_policy_statement)
+    return iam_statements
+
+
+# If item is of item type i.e. single instance if present, then wrap in an array.
+# This is helpful at places where array is required even if one element is present
+def _convert_to_array(item_type, item):
+    if isinstance(item, item_type):
+        return [item]
+    return item

backend/dataall/base/utils/iam_policy_utils.py

@@ -1,6 +1,5 @@
-from typing import List, Callable
+from typing import List, Callable, Dict
 import logging
-from aws_cdk import aws_iam as iam
 
 logger = logging.getLogger(__name__)
 
@@ -9,7 +8,7 @@
 MAXIMUM_NUMBER_MANAGED_POLICIES = 20  # Soft limit 10, hard limit 20
 
 
-def split_policy_statements_in_chunks(statements: List):
+def split_policy_statements_in_chunks(statements: List[Dict]):
     """
     Splitter used for IAM policies with an undefined number of statements
     - Ensures that the size of the IAM policy remains below the POLICY LIMIT
@@ -18,7 +17,7 @@ def split_policy_statements_in_chunks(statements: List):
     """
     chunks = []
     index = 0
-    statements_list_of_strings = [str(s.to_json()) for s in statements]
+    statements_list_of_strings = [str(s) for s in statements]
     total_length = len(', '.join(statements_list_of_strings))
     logger.info(f'Number of statements = {len(statements)}')
     logger.info(f'Total length of statements = {total_length}')
@@ -31,13 +30,12 @@ def split_policy_statements_in_chunks(statements: List):
         chunk = []
         chunk_size = 0
         while (
-            index < len(statements)
-            and chunk_size + len(str(statements[index].to_json())) < POLICY_LIMIT - POLICY_HEADERS_BUFFER
+            index < len(statements) and chunk_size + len(str(statements[index])) < POLICY_LIMIT - POLICY_HEADERS_BUFFER
         ):
             #  Appends a statement to the chunk until we reach its maximum size.
             #  It compares, current size of the statements < allowed size for the statements section of a policy
             chunk.append(statements[index])
-            chunk_size += len(str(statements[index].to_json()))
+            chunk_size += len(str(statements[index]))
             index += 1
         chunks.append(chunk)
     logger.info(f'Total number of managed policies = {len(chunks)}')
@@ -46,15 +44,13 @@ def split_policy_statements_in_chunks(statements: List):
     return chunks
 
 
-def split_policy_with_resources_in_statements(
-    base_sid: str, effect: iam.Effect, actions: List[str], resources: List[str]
-):
+def split_policy_with_resources_in_statements(base_sid: str, effect: str, actions: List[str], resources: List[str]):
     """
     The variable part of the policy is in the resources parameter of the PolicyStatement
     """
 
     def _build_statement(split, subset):
-        return iam.PolicyStatement(sid=base_sid + str(split), effect=effect, actions=actions, resources=subset)
+        return {'Sid': base_sid + str(split), 'Effect': effect, 'Action': actions, 'Resource': subset}
 
     total_length, base_length = _policy_analyzer(resources, _build_statement)
     extra_chars = len('" ," ')
@@ -72,21 +68,21 @@ def _build_statement(split, subset):
 
 
 def split_policy_with_mutiple_value_condition_in_statements(
-    base_sid: str, effect: iam.Effect, actions: List[str], resources: List[str], condition_dict: dict
+    base_sid: str, effect: str, actions: List[str], resources: List[str], condition_dict: dict
 ):
     """
     The variable part of the policy is in the conditions parameter of the PolicyStatement
     conditions_dict passes the different components of the condition mapping
     """
 
     def _build_statement(split, subset):
-        return iam.PolicyStatement(
-            sid=base_sid + str(split),
-            effect=effect,
-            actions=actions,
-            resources=resources,
-            conditions={condition_dict.get('key'): {condition_dict.get('resource'): subset}},
-        )
+        return {
+            'Sid': base_sid + str(split),
+            'Effect': effect,
+            'Action': actions,
+            'Resource': resources,
+            'Condition': {condition_dict.get('key'): {condition_dict.get('resource'): subset}},
+        }
 
     total_length, base_length = _policy_analyzer(condition_dict.get('values'), _build_statement)
     extra_chars = len(
@@ -109,13 +105,13 @@ def _build_statement(split, subset):
     return resulting_statements
 
 
-def _policy_analyzer(resources: List[str], statement_builder: Callable[[int, List[str]], iam.PolicyStatement]):
+def _policy_analyzer(resources: List[str], statement_builder: Callable[[int, List[str]], Dict]):
     """
     Calculates the policy size with the resources (total_length) and without resources (base_length)
     """
     statement_without_resources = statement_builder(1, ['*'])
     resources_str = '" ," '.join(r for r in resources)
-    base_length = len(str(statement_without_resources.to_json()))
+    base_length = len(str(statement_without_resources))
     total_length = base_length + len(resources_str)
     logger.info(f'Policy base length = {base_length}')
     logger.info(f'Resources as string length = {len(resources_str)}')
@@ -128,7 +124,7 @@ def _policy_splitter(
     base_length: int,
     resources: List[str],
     extra_chars: int,
-    statement_builder: Callable[[int, List[str]], iam.PolicyStatement],
+    statement_builder: Callable[[int, List[str]], Dict],
 ):
     """
     Splitter used for IAM policy statements with an undefined number of resources one of the parameters of the policy.

backend/dataall/base/utils/naming_convention.py

@@ -56,6 +56,20 @@ def build_compliant_name(self) -> str:
         suffix = f'-{self.target_uri}' if len(self.target_uri) else ''
         return f"{slugify(self.resource_prefix + '-' + self.target_label[:(max_length - len(self.resource_prefix + self.target_uri))] + suffix, regex_pattern=fr'{regex}', separator=separator, lowercase=True)}"
 
+    def build_compliant_name_with_index(self, index: int = None) -> str:
+        """
+        Builds a compliant AWS resource name with an index at the end of the policy name
+        IMP - If no index is provided, then this method provides a base policy name without index. Base policy name is calculated by considering the length of string required for index
+        This is done so that the base policy name doesn't change when an index is added to the string.
+        """
+        regex = NamingConventionPattern[self.service].value['regex']
+        separator = NamingConventionPattern[self.service].value['separator']
+        max_length = NamingConventionPattern[self.service].value['max_length']
+        index_string_length = 2  # This is added to adjust the target label string even if the index is set to None. This helps in getting the base policy name when index is None
+        index_string = f'-{index}' if index is not None else ''
+        suffix = f'-{self.target_uri}' if len(self.target_uri) else ''
+        return f"{slugify(self.resource_prefix + '-' + self.target_label[:(max_length - len(self.resource_prefix + self.target_uri) - index_string_length)] + suffix + index_string, regex_pattern=fr'{regex}', separator=separator, lowercase=True)}"
+
     def validate_name(self):
         regex = NamingConventionPattern[self.service].value['regex']
         max_length = NamingConventionPattern[self.service].value['max_length']

backend/dataall/core/environment/api/resolvers.py

@@ -178,13 +178,6 @@ def get_parent_organization(context: Context, source, **kwargs):
     return org
 
 
-# used from ConsumptionRole type as field resolver
-def resolve_consumption_role_policies(context: Context, source, **kwargs):
-    return EnvironmentService.resolve_consumption_role_policies(
-        uri=source.environmentUri, IAMRoleName=source.IAMRoleName
-    )
-
-
 # used from getConsumptionRolePolicies query -- query resolver
 def get_consumption_role_policies(context: Context, source, environmentUri, IAMRoleName):
     return EnvironmentService.resolve_consumption_role_policies(uri=environmentUri, IAMRoleName=IAMRoleName)

backend/dataall/core/environment/api/types.py

@@ -3,7 +3,6 @@
 from dataall.core.environment.api.resolvers import (
     get_environment_stack,
     get_parent_organization,
-    resolve_consumption_role_policies,
     resolve_environment_networks,
     resolve_parameters,
     resolve_user_role,
@@ -180,9 +179,6 @@
         gql.Field(name='created', type=gql.String),
         gql.Field(name='updated', type=gql.String),
         gql.Field(name='deleted', type=gql.String),
-        gql.Field(
-            name='managedPolicies', type=gql.ArrayType(RoleManagedPolicy), resolver=resolve_consumption_role_policies
-        ),
     ],
 )