[Gh 884] IAM policy splitting for requestor IAM policies

backend/dataall/base/aws/iam.py

@@ -66,6 +66,25 @@ def get_role_policy(
             log.error(f'Failed to get policy {policy_name} of role {role_name} : {e}')
             return None
 
+    @staticmethod
+    def list_policy_names_by_policy_pattern(account_id: str, region: str, policy_name: str):
+        try:
+            client = IAM.client(account_id, region)
+            # Setting Scope to 'Local' to fetch all the policies created in this account
+            paginator = client.get_paginator('list_policies')
+            policies = []
+            for page in paginator.paginate(Scope='Local'):
+                policies.extend(page['Policies'])
+            policy_names = [policy.get('PolicyName') for policy in policies]
+            return [policy_nm for policy_nm in policy_names if policy_name in policy_nm]
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'AccessDenied':
+                raise Exception(
+                    f'Data.all Environment Pivot Role does not have permissions to get policies with pattern {policy_name} due to: {e}'
+                )
+            log.error(f'Failed to get policies for policy pattern due to: {e}')
+            return []
+
     @staticmethod
     def delete_role_policy(
         account_id: str,
@@ -101,6 +120,23 @@ def get_managed_policy_by_name(account_id: str, region: str, policy_name: str):
             log.error(f'Failed to get policy {policy_name}: {e}')
             return None
 
+    @staticmethod
+    def get_managed_policy_document_by_name(account_id: str, region: str, policy_name: str):
+        try:
+            arn = f'arn:aws:iam::{account_id}:policy/{policy_name}'
+            client = IAM.client(account_id, region)
+            policy = IAM.get_managed_policy_by_name(account_id, region, policy_name)
+            policyVersionId = policy['DefaultVersionId']
+            response = client.get_policy_version(PolicyArn=arn, VersionId=policyVersionId)
+            return response['PolicyVersion']['Document']
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'AccessDenied':
+                raise Exception(
+                    f'Data.all Environment Pivot Role does not have permissions to to get policy {policy_name}: {e}'
+                )
+            log.error(f'Failed to get policy {policy_name}: {e}')
+            return None
+
     @staticmethod
     def create_managed_policy(account_id: str, region: str, policy_name: str, policy: str):
         try:
@@ -275,3 +311,16 @@ def remove_invalid_role_ids(account_id: str, region: str, principal_list):
             if 'AROA' in p_id:
                 if p_id not in all_role_ids:
                     principal_list.remove(p_id)
+
+    @staticmethod
+    def get_attached_managed_policies_to_role(account_id: str, region: str, role_name: str):
+        try:
+            client = IAM.client(account_id, region)
+            response = client.list_attached_role_policies(RoleName=role_name)
+            return [policy.get('PolicyName') for policy in response['AttachedPolicies']]
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'AccessDenied':
+                raise Exception(
+                    f'Data.all Environment Pivot Role does not have permissions to get attached managed policies for {role_name}: {e}'
+                )
+            raise Exception(f'Failed to get attached managed policies for {role_name}: {e}')

backend/dataall/base/aws/service_quota.py

@@ -0,0 +1,56 @@
+import logging
+from botocore.exceptions import ClientError
+
+from .sts import SessionHelper
+
+log = logging.getLogger(__name__)
+
+
+class ServiceQuota:
+    def __init__(self, account_id, region):
+        session = SessionHelper.remote_session(accountid=account_id, region=region)
+        self.client = session.client('service-quotas')
+
+    def list_services(self):
+        try:
+            log.info('Fetching services list with service codes in aws account')
+            services_list = []
+            paginator = self.client.get_paginator('list_services')
+            for page in paginator.paginate():
+                services_list.extend(page.get('Services'))
+            return services_list
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'AccessDenied':
+                raise Exception(
+                    f'Data.all Environment Pivot Role does not have permissions to list services for getting service code : {e}'
+                )
+            log.error(f'Failed list services and service codes due to: {e}')
+            return []
+
+    def list_service_quota(self, service_code):
+        try:
+            log.info('Fetching services quota code in aws account')
+            service_quota_code_list = []
+            paginator = self.client.get_paginator('list_service_quotas')
+            for page in paginator.paginate(ServiceCode=service_code):
+                service_quota_code_list.extend(page.get('Quotas'))
+            log.debug(f'Services quota list: {service_quota_code_list}')
+            return service_quota_code_list
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'AccessDenied':
+                raise Exception(f'Data.all Environment Pivot Role does not have permissions to list quota codes: {e}')
+            log.error(f'Failed list quota codes to: {e}')
+            return []
+
+    def get_service_quota_value(self, service_code, service_quota_code):
+        try:
+            log.info(
+                f'Getting service quota for service code: {service_code} and service quota code: {service_quota_code}'
+            )
+            response = self.client.get_service_quota(ServiceCode=service_code, QuotaCode=service_quota_code)
+            return response['Quota']['Value']
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'AccessDenied':
+                raise Exception(f'Data.all Environment Pivot Role does not have permissions to list quota codes: {e}')
+            log.error(f'Failed list quota codes to: {e}')
+            return None

backend/dataall/base/utils/iam_cdk_utils.py

@@ -0,0 +1,21 @@
+from typing import Dict, Any
+from aws_cdk import aws_iam as iam
+
+
+def convert_from_json_to_iam_policy_statement_with_conditions(iam_policy: Dict[Any, Any]):
+    return iam.PolicyStatement(
+        sid=iam_policy.get('Sid'),
+        effect=iam_policy.get('Effect'),
+        actions=iam_policy.get('Action'),
+        resources=iam_policy.get('Resource'),
+        conditions=iam_policy.get('Condition'),
+    )
+
+
+def convert_from_json_to_iam_policy_statement(iam_policy: Dict[Any, Any]):
+    return iam.PolicyStatement(
+        sid=iam_policy.get('Sid'),
+        effect=iam_policy.get('Effect'),
+        actions=iam_policy.get('Action'),
+        resources=iam_policy.get('Resource'),
+    )

backend/dataall/base/utils/iam_policy_utils.py

@@ -1,6 +1,5 @@
-from typing import List, Callable
+from typing import List, Callable, Any, Dict
 import logging
-from aws_cdk import aws_iam as iam
 
 logger = logging.getLogger(__name__)
 
@@ -9,7 +8,7 @@
 MAXIMUM_NUMBER_MANAGED_POLICIES = 20  # Soft limit 10, hard limit 20
 
 
-def split_policy_statements_in_chunks(statements: List):
+def split_policy_statements_in_chunks(statements: List[Any]):
     """
     Splitter used for IAM policies with an undefined number of statements
     - Ensures that the size of the IAM policy remains below the POLICY LIMIT
@@ -18,7 +17,7 @@ def split_policy_statements_in_chunks(statements: List):
     """
     chunks = []
     index = 0
-    statements_list_of_strings = [str(s.to_json()) for s in statements]
+    statements_list_of_strings = [str(s) for s in statements]
     total_length = len(', '.join(statements_list_of_strings))
     logger.info(f'Number of statements = {len(statements)}')
     logger.info(f'Total length of statements = {total_length}')
@@ -31,13 +30,12 @@ def split_policy_statements_in_chunks(statements: List):
         chunk = []
         chunk_size = 0
         while (
-            index < len(statements)
-            and chunk_size + len(str(statements[index].to_json())) < POLICY_LIMIT - POLICY_HEADERS_BUFFER
+            index < len(statements) and chunk_size + len(str(statements[index])) < POLICY_LIMIT - POLICY_HEADERS_BUFFER
         ):
             #  Appends a statement to the chunk until we reach its maximum size.
             #  It compares, current size of the statements < allowed size for the statements section of a policy
             chunk.append(statements[index])
-            chunk_size += len(str(statements[index].to_json()))
+            chunk_size += len(str(statements[index]))
             index += 1
         chunks.append(chunk)
     logger.info(f'Total number of managed policies = {len(chunks)}')
@@ -46,15 +44,13 @@ def split_policy_statements_in_chunks(statements: List):
     return chunks
 
 
-def split_policy_with_resources_in_statements(
-    base_sid: str, effect: iam.Effect, actions: List[str], resources: List[str]
-):
+def split_policy_with_resources_in_statements(base_sid: str, effect: str, actions: List[str], resources: List[str]):
     """
     The variable part of the policy is in the resources parameter of the PolicyStatement
     """
 
     def _build_statement(split, subset):
-        return iam.PolicyStatement(sid=base_sid + str(split), effect=effect, actions=actions, resources=subset)
+        return {'Sid': base_sid + str(split), 'Effect': effect, 'Action': actions, 'Resource': subset}
 
     total_length, base_length = _policy_analyzer(resources, _build_statement)
     extra_chars = len('" ," ')
@@ -72,21 +68,21 @@ def _build_statement(split, subset):
 
 
 def split_policy_with_mutiple_value_condition_in_statements(
-    base_sid: str, effect: iam.Effect, actions: List[str], resources: List[str], condition_dict: dict
+    base_sid: str, effect: str, actions: List[str], resources: List[str], condition_dict: dict
 ):
     """
     The variable part of the policy is in the conditions parameter of the PolicyStatement
     conditions_dict passes the different components of the condition mapping
     """
 
     def _build_statement(split, subset):
-        return iam.PolicyStatement(
-            sid=base_sid + str(split),
-            effect=effect,
-            actions=actions,
-            resources=resources,
-            conditions={condition_dict.get('key'): {condition_dict.get('resource'): subset}},
-        )
+        return {
+            'Sid': base_sid + str(split),
+            'Effect': effect,
+            'Action': actions,
+            'Resource': resources,
+            'Condition': {condition_dict.get('key'): {condition_dict.get('resource'): subset}},
+        }
 
     total_length, base_length = _policy_analyzer(condition_dict.get('values'), _build_statement)
     extra_chars = len(
@@ -109,13 +105,13 @@ def _build_statement(split, subset):
     return resulting_statements
 
 
-def _policy_analyzer(resources: List[str], statement_builder: Callable[[int, List[str]], iam.PolicyStatement]):
+def _policy_analyzer(resources: List[str], statement_builder: Callable[[int, List[str]], Dict]):
     """
     Calculates the policy size with the resources (total_length) and without resources (base_length)
     """
     statement_without_resources = statement_builder(1, ['*'])
     resources_str = '" ," '.join(r for r in resources)
-    base_length = len(str(statement_without_resources.to_json()))
+    base_length = len(str(statement_without_resources))
     total_length = base_length + len(resources_str)
     logger.info(f'Policy base length = {base_length}')
     logger.info(f'Resources as string length = {len(resources_str)}')
@@ -128,7 +124,7 @@ def _policy_splitter(
     base_length: int,
     resources: List[str],
     extra_chars: int,
-    statement_builder: Callable[[int, List[str]], iam.PolicyStatement],
+    statement_builder: Callable[[int, List[str]], Dict],
 ):
     """
     Splitter used for IAM policy statements with an undefined number of resources one of the parameters of the policy.

backend/dataall/base/utils/naming_convention.py

@@ -56,6 +56,18 @@ def build_compliant_name(self) -> str:
         suffix = f'-{self.target_uri}' if len(self.target_uri) else ''
         return f"{slugify(self.resource_prefix + '-' + self.target_label[:(max_length - len(self.resource_prefix + self.target_uri))] + suffix, regex_pattern=fr'{regex}', separator=separator, lowercase=True)}"
 
+    def build_compliant_name_with_index(self, index: int = None) -> str:
+        """
+        Builds a compliant AWS resource name
+        """
+        regex = NamingConventionPattern[self.service].value['regex']
+        separator = NamingConventionPattern[self.service].value['separator']
+        max_length = NamingConventionPattern[self.service].value['max_length']
+        index_string = f'-{index}' if index is not None else '- '
+        suffix = f'-{self.target_uri}' if len(self.target_uri) else ''
+        suffix = suffix + index_string if index is not None else suffix
+        return f"{slugify(self.resource_prefix + '-' + self.target_label[:(max_length - len(self.resource_prefix + self.target_uri + index_string))] + suffix, regex_pattern=fr'{regex}', separator=separator, lowercase=True)}"
+
     def validate_name(self):
         regex = NamingConventionPattern[self.service].value['regex']
         max_length = NamingConventionPattern[self.service].value['max_length']

backend/dataall/core/environment/cdk/pivot_role_core_policies/__init__.py

@@ -10,6 +10,7 @@
     sqs,
     ssm,
     sts,
+    service_quota,
 )
 
-__all__ = ['cloudformation', 'iam', 'kms', 'logging', 's3', 'sns', 'sqs', 'ssm', 'sts']
+__all__ = ['cloudformation', 'iam', 'kms', 'logging', 's3', 'sns', 'sqs', 'ssm', 'sts', 'service_quota']

backend/dataall/core/environment/cdk/pivot_role_core_policies/iam.py

@@ -13,7 +13,7 @@ def get_statements(self):
         statements = [
             # IAM - needed for consumption roles and for S3 sharing
             iam.PolicyStatement(
-                sid='IAMListGet', effect=iam.Effect.ALLOW, actions=['iam:ListRoles', 'iam:Get*'], resources=['*']
+                sid='IAMListGet', effect=iam.Effect.ALLOW, actions=['iam:List*', 'iam:Get*'], resources=['*']
             ),
             iam.PolicyStatement(
                 sid='PassRole',

backend/dataall/core/environment/cdk/pivot_role_core_policies/service_quota.py

@@ -0,0 +1,22 @@
+from dataall.core.environment.cdk.pivot_role_stack import PivotRoleStatementSet
+from aws_cdk import aws_iam as iam
+
+
+class ServiceQuotaPivotRole(PivotRoleStatementSet):
+    """
+    Class including all permissions needed  by the pivot role to work with AWS Service Quota.
+    It allows pivot role to:
+    - List and Get Service Quota details
+    """
+
+    def get_statements(self):
+        statements = [
+            # Service Quota - Needed to determine the number of service quotas for managed policies which can be attached
+            iam.PolicyStatement(
+                sid='ServiceQuotaListGet',
+                effect=iam.Effect.ALLOW,
+                actions=['servicequotas:List*', 'servicequotas:Get*'],
+                resources=['*'],
+            )
+        ]
+        return statements

backend/dataall/core/environment/cdk/pivot_role_stack.py

@@ -3,6 +3,11 @@
 from typing import List
 from constructs import Construct
 from aws_cdk import Duration, aws_iam as iam, NestedStack
+
+from dataall.base.utils.iam_cdk_utils import (
+    convert_from_json_to_iam_policy_statement_with_conditions,
+    convert_from_json_to_iam_policy_statement,
+)
 from dataall.base.utils.iam_policy_utils import split_policy_statements_in_chunks
 
 logger = logging.getLogger(__name__)
@@ -36,7 +41,14 @@ def generate_policies(self) -> List[iam.ManagedPolicy]:
             statements.extend(service.get_statements(self))
             logger.info(f'statements: {str(service.get_statements(self))}')
 
-        statements_chunks = split_policy_statements_in_chunks(statements)
+        statements_json = [statement.to_json() for statement in statements]
+        statement_chunks_json = split_policy_statements_in_chunks(statements_json)
+        statements_chunks = []
+        for statement_js in statement_chunks_json:
+            if not statement_js.get('Condition', None):
+                statements_chunks.append(convert_from_json_to_iam_policy_statement_with_conditions(statement_js))
+            else:
+                statements_chunks.append(convert_from_json_to_iam_policy_statement(statement_js))
 
         for index, chunk in enumerate(statements_chunks):
             policies.append(