Metadata form 7: Access control and deletion behaviour

backend/dataall/core/permissions/services/resource_policy_service.py

@@ -217,6 +217,7 @@ def get_resource_policy_permissions(session, group_uri, resource_uri) -> List[Re
             raise exceptions.RequiredParameter(param_name='group_uri')
         if not resource_uri:
             raise exceptions.RequiredParameter(param_name='resource_uri')
+
         policy = ResourcePolicyRepository.find_resource_policy(
             session=session,
             group_uri=group_uri,

backend/dataall/modules/metadata_forms/api/queries.py

@@ -5,6 +5,7 @@
     get_metadata_form,
     get_attached_metadata_form,
     list_attached_forms,
+    get_entity_metadata_form_permissions,
 )
 
 listUserMetadataForms = gql.QueryField(
@@ -47,3 +48,11 @@
     resolver=get_attached_metadata_form,
     test_scope='MetadataForm',
 )
+
+getEntityMetadataFormPermissions = gql.QueryField(
+    name='getEntityMetadataFormPermissions',
+    args=[gql.Argument('entityUri', gql.NonNullableType(gql.String))],
+    type=gql.ArrayType(gql.String),
+    resolver=get_entity_metadata_form_permissions,
+    test_scope='MetadataForm',
+)

backend/dataall/modules/metadata_forms/api/resolvers.py

@@ -94,3 +94,7 @@ def has_tenant_permissions_for_metadata_forms(context: Context, source: Metadata
 
 def resolve_metadata_form_field(context: Context, source: AttachedMetadataFormField):
     return MetadataFormService.get_metadata_form_field_by_uri(uri=source.fieldUri)
+
+
+def get_entity_metadata_form_permissions(context: Context, source, entityUri):
+    return MetadataFormService.get_mf_permissions(entityUri=entityUri)

backend/dataall/modules/metadata_forms/db/metadata_form_repository.py

@@ -70,6 +70,9 @@ def query_user_metadata_forms(session, is_da_admin, groups, env_uris, org_uris,
         :param filter:
         """
 
+        env_uris = env_uris or []
+        org_uris = org_uris or []
+
         query = session.query(MetadataForm)
 
         if not is_da_admin:
@@ -140,6 +143,8 @@ def query_entity_metadata_forms(
 
         entity_orgs_uris = entity_orgs_uris or []
         entity_envs_uris = entity_envs_uris or []
+        user_org_uris = user_org_uris or []
+        user_env_uris = user_env_uris or []
 
         orgs = list(set(user_org_uris).intersection(set(entity_orgs_uris)))
         envs = list(set(user_env_uris).intersection(set(entity_envs_uris)))
@@ -247,3 +252,9 @@ def query_attached_metadata_forms(session, is_da_admin, groups, user_envs_uris,
         if filter and filter.get('metadataFormUri'):
             query = query.filter(AttachedMetadataForm.metadataFormUri == filter.get('metadataFormUri'))
         return query
+
+    @staticmethod
+    def query_all_attached_metadata_forms_for_entity(session, entityUri, entityType):
+        return session.query(AttachedMetadataForm).filter(
+            and_(AttachedMetadataForm.entityType == entityType, AttachedMetadataForm.entityUri == entityUri)
+        )

backend/dataall/modules/metadata_forms/services/attached_metadata_form_service.py

@@ -1,10 +1,9 @@
 from dataall.base.context import get_context
 from dataall.base.db import exceptions, paginate
-from dataall.core.environment.db.environment_repositories import EnvironmentRepository
-from dataall.core.organizations.db.organization_repositories import OrganizationRepository
-from dataall.core.permissions.services.tenant_policy_service import TenantPolicyValidationService
+from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService
 from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository
 from dataall.modules.metadata_forms.services.metadata_form_access_service import MetadataFormAccessService
+from dataall.modules.metadata_forms.services.metadata_form_permissions import ATTACH_METADATA_FORM
 
 
 class AttachedMetadataFormValidationService:
@@ -33,15 +32,22 @@ def validate_enrich_fields_params(mf_fields, data):
 
 class AttachedMetadataFormService:
     @staticmethod
+    def _get_entity_uri(data):
+        return data.get('entityUri')
+
+    @staticmethod
+    @ResourcePolicyService.has_resource_permission(
+        ATTACH_METADATA_FORM, parent_resource=_get_entity_uri, param_name='data'
+    )
     def create_attached_metadata_form(uri, data):
         AttachedMetadataFormValidationService.validate_filled_form_params(uri, data)
-        with get_context().db_engine.scoped_session() as session:
+        context = get_context()
+        with context.db_engine.scoped_session() as session:
             mf = MetadataFormRepository.get_metadata_form(session, uri)
             if not mf:
                 raise exceptions.ObjectNotFound('MetadataForm', uri)
             mf_fields = MetadataFormRepository.get_metadata_form_fields(session, uri)
             AttachedMetadataFormValidationService.validate_enrich_fields_params(mf_fields, data)
-
             amf = MetadataFormRepository.create_attached_metadata_form(session, uri, data)
             for f in data.get('fields'):
                 MetadataFormRepository.create_attached_metadata_form_field(
@@ -74,7 +80,11 @@ def list_attached_forms(filter=None):
             ).to_dict()
 
     @staticmethod
+    @ResourcePolicyService.has_resource_permission(
+        ATTACH_METADATA_FORM, parent_resource=_get_entity_uri, param_name='data'
+    )
     def delete_attached_metadata_form(uri):
         mf = AttachedMetadataFormService.get_attached_metadata_form(uri)
-        with get_context().db_engine.scoped_session() as session:
+        context = get_context()
+        with context.db_engine.scoped_session() as session:
             return session.delete(mf)

backend/dataall/modules/metadata_forms/services/metadata_form_permissions.py

@@ -1,6 +1,55 @@
+from dataall.core.permissions.services.environment_permissions import ENVIRONMENT_INVITED, ENVIRONMENT_ALL
+from dataall.core.permissions.services.organization_permissions import (
+    ORGANIZATION_ALL,
+    ORGANIZATION_INVITED_DESCRIPTIONS,
+)
 from dataall.core.permissions.services.tenant_permissions import TENANT_ALL, TENANT_ALL_WITH_DESC
+from dataall.core.permissions.services.resources_permissions import RESOURCES_ALL, RESOURCES_ALL_WITH_DESC
+from dataall.modules.s3_datasets.services.dataset_permissions import DATASET_WRITE, DATASET_ALL
 
-
+# ------------------------TENANT-----------------------------------
 MANAGE_METADATA_FORMS = 'MANAGE_METADATA_FORMS'
 TENANT_ALL.append(MANAGE_METADATA_FORMS)
 TENANT_ALL_WITH_DESC[MANAGE_METADATA_FORMS] = 'Manage metadata forms'
+
+# ------------------------RESOURCE---------------------------------
+# permissions to attach MF to the entity, ot make the entity the visibility base for MF
+# these permissions are attached to Organizations, Environments, Datasets etc.
+ATTACH_METADATA_FORM = 'ATTACH_METADATA_FORM'
+CREATE_METADATA_FORM = 'CREATE_METADATA_FORM'
+ALL_METADATA_FORMS_ENTITY_PERMISSIONS = [ATTACH_METADATA_FORM, CREATE_METADATA_FORM]
+RESOURCES_ALL.extend(ALL_METADATA_FORMS_ENTITY_PERMISSIONS)
+RESOURCES_ALL_WITH_DESC[CREATE_METADATA_FORM] = 'Create metadata form within this visibility scope'
+RESOURCES_ALL_WITH_DESC[ATTACH_METADATA_FORM] = 'Attach metadata form'
+
+ORGANIZATION_ALL.extend(ALL_METADATA_FORMS_ENTITY_PERMISSIONS)
+ORGANIZATION_INVITED_DESCRIPTIONS[CREATE_METADATA_FORM] = 'Create metadata form within this visibility scope'
+ORGANIZATION_INVITED_DESCRIPTIONS[ATTACH_METADATA_FORM] = 'Attach metadata form'
+
+ENVIRONMENT_INVITED.extend(ALL_METADATA_FORMS_ENTITY_PERMISSIONS)
+ENVIRONMENT_ALL.extend(ALL_METADATA_FORMS_ENTITY_PERMISSIONS)
+
+DATASET_WRITE.extend(ALL_METADATA_FORMS_ENTITY_PERMISSIONS)
+DATASET_ALL.extend(ALL_METADATA_FORMS_ENTITY_PERMISSIONS)
+# ------------------------METADATA FORM----------------------------
+# permissions to change and delete metadata forms
+# these permissions are attached to MFs
+UPDATE_METADATA_FORM_FIELD = 'UPDATE_METADATA_FORM_FIELD'
+DELETE_METADATA_FORM_FIELD = 'DELETE_METADATA_FORM_FIELD'
+DELETE_METADATA_FORM = 'DELETE_METADATA_FORM'
+EDIT_METADATA_FORM = 'EDIT_METADATA_FORM'
+
+METADATA_FORM_PERMISSIONS_ALL = [
+    UPDATE_METADATA_FORM_FIELD,
+    DELETE_METADATA_FORM_FIELD,
+    DELETE_METADATA_FORM,
+    EDIT_METADATA_FORM,
+]
+
+METADATA_FORM_EDIT_PERMISSIONS = [
+    EDIT_METADATA_FORM,
+    UPDATE_METADATA_FORM_FIELD,
+    DELETE_METADATA_FORM_FIELD,
+]
+
+RESOURCES_ALL.extend(METADATA_FORM_PERMISSIONS_ALL)

backend/dataall/modules/metadata_forms/services/metadata_form_service.py

@@ -2,15 +2,24 @@
 from dataall.base.db import exceptions, paginate
 from dataall.core.organizations.db.organization_repositories import OrganizationRepository
 from dataall.core.environment.db.environment_repositories import EnvironmentRepository
-from dataall.core.permissions.services.tenant_policy_service import TenantPolicyValidationService, TenantPolicyService
+from dataall.core.permissions.db.resource_policy.resource_policy_repositories import ResourcePolicyRepository
+from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService
+from dataall.core.permissions.services.tenant_policy_service import TenantPolicyService
 from dataall.modules.metadata_forms.db.enums import (
     MetadataFormVisibility,
     MetadataFormFieldType,
 )
 from dataall.modules.catalog.db.glossary_repositories import GlossaryRepository
 from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository
 from dataall.modules.metadata_forms.services.metadata_form_access_service import MetadataFormAccessService
-from dataall.modules.metadata_forms.services.metadata_form_permissions import MANAGE_METADATA_FORMS
+from dataall.modules.metadata_forms.services.metadata_form_permissions import (
+    MANAGE_METADATA_FORMS,
+    DELETE_METADATA_FORM,
+    DELETE_METADATA_FORM_FIELD,
+    UPDATE_METADATA_FORM_FIELD,
+    CREATE_METADATA_FORM,
+    ALL_METADATA_FORMS_ENTITY_PERMISSIONS,
+)
 
 
 class MetadataFormParamValidationService:
@@ -91,7 +100,20 @@ class MetadataFormService:
     @TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS)
     def create_metadata_form(data):
         MetadataFormParamValidationService.validate_create_form_params(data)
-        with get_context().db_engine.scoped_session() as session:
+        context = get_context()
+        with context.db_engine.scoped_session() as session:
+            if data.get('visibility') in [
+                MetadataFormVisibility.Organization.value,
+                MetadataFormVisibility.Environment.value,
+            ]:
+                ResourcePolicyService.check_user_resource_permission(
+                    session=session,
+                    username=context.username,
+                    groups=context.groups,
+                    resource_uri=data.get('homeEntity'),
+                    permission_name=CREATE_METADATA_FORM,
+                )
+
             form = MetadataFormRepository.create_metadata_form(session, data)
             return form
 
@@ -104,7 +126,7 @@ def get_metadata_form_by_uri(uri):
     # toDo: deletion logic
     @staticmethod
     @TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS)
-    @MetadataFormAccessService.can_perform('DELETE')
+    @MetadataFormAccessService.can_perform(DELETE_METADATA_FORM)
     def delete_metadata_form_by_uri(uri):
         if mf := MetadataFormService.get_metadata_form_by_uri(uri):
             with get_context().db_engine.scoped_session() as session:
@@ -181,15 +203,15 @@ def get_metadata_form_field_by_uri(uri):
 
     @staticmethod
     @TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS)
-    @MetadataFormAccessService.can_perform('ADD FIELD')
+    @MetadataFormAccessService.can_perform(UPDATE_METADATA_FORM_FIELD)
     def create_metadata_form_field(uri, data):
         MetadataFormParamValidationService.validate_create_field_params(data)
         with get_context().db_engine.scoped_session() as session:
             return MetadataFormRepository.create_metadata_form_field(session, uri, data)
 
     @staticmethod
     @TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS)
-    @MetadataFormAccessService.can_perform('ADD FIELDS')
+    @MetadataFormAccessService.can_perform(UPDATE_METADATA_FORM_FIELD)
     def create_metadata_form_fields(uri, data_arr):
         fields = []
         for data in data_arr:
@@ -198,15 +220,15 @@ def create_metadata_form_fields(uri, data_arr):
 
     @staticmethod
     @TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS)
-    @MetadataFormAccessService.can_perform('DELETE FIELD')
+    @MetadataFormAccessService.can_perform(DELETE_METADATA_FORM_FIELD)
     def delete_metadata_form_field(uri, fieldUri):
         mf = MetadataFormService.get_metadata_form_field_by_uri(fieldUri)
         with get_context().db_engine.scoped_session() as session:
             return session.delete(mf)
 
     @staticmethod
     @TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS)
-    @MetadataFormAccessService.can_perform('UPDATE FIELDS')
+    @MetadataFormAccessService.can_perform('UPDATE_METADATA_FORM_FIELD')
     def batch_metadata_form_field_update(uri, data):
         to_delete = []
         to_update = []
@@ -238,8 +260,23 @@ def batch_metadata_form_field_update(uri, data):
 
     @staticmethod
     @TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS)
-    @MetadataFormAccessService.can_perform('UPDATE FIELD')
+    @MetadataFormAccessService.can_perform(UPDATE_METADATA_FORM_FIELD)
     def update_metadata_form_field(uri, fieldUri, data):
         with get_context().db_engine.scoped_session() as session:
             MetadataFormParamValidationService.validate_update_field_params(uri, data)
             return MetadataFormRepository.update_metadata_form_field(session, fieldUri, data)
+
+    @staticmethod
+    def get_mf_permissions(entityUri):
+        context = get_context()
+        result_permissions = []
+        with context.db_engine.scoped_session() as session:
+            for permissions in ALL_METADATA_FORMS_ENTITY_PERMISSIONS:
+                if ResourcePolicyRepository.has_user_resource_permission(
+                    session=session,
+                    groups=context.groups,
+                    permission_name=permissions,
+                    resource_uri=entityUri,
+                ):
+                    result_permissions.append(permissions)
+            return result_permissions

backend/dataall/modules/redshift_datasets/services/redshift_dataset_service.py

@@ -184,6 +184,7 @@ def delete_redshift_dataset(uri):
             RedshiftDatasetService._delete_dataset_term_links(session, uri)
             VoteRepository.delete_votes(session, dataset.datasetUri, VOTE_REDSHIFT_DATASET_NAME)
             session.delete(dataset)
+
             session.commit()
             return True
 

backend/dataall/modules/s3_datasets/services/dataset_service.py

@@ -22,7 +22,6 @@
 from dataall.modules.catalog.db.glossary_repositories import GlossaryRepository
 from dataall.modules.s3_datasets.db.dataset_bucket_repositories import DatasetBucketRepository
 from dataall.modules.shares_base.db.share_object_repositories import ShareObjectRepository
-from dataall.modules.shares_base.services.share_object_service import ShareObjectService
 from dataall.modules.vote.db.vote_repositories import VoteRepository
 from dataall.modules.s3_datasets.aws.glue_dataset_client import DatasetCrawler
 from dataall.modules.s3_datasets.aws.s3_dataset_client import S3DatasetClient

backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py

@@ -40,7 +40,8 @@
 
 logger = logging.getLogger(__name__)
 ACCESS_POINT_CREATION_TIME = 30
-ACCESS_POINT_CREATION_RETRIES = 5
+ACCESS_POINT_CREATION_RETRIES = 10
+ACCESS_POINT_BACKOFF_COEFFICIENT = 1.1  # every time increase retry delay by 10%
 
 
 class S3AccessPointShareManager:
@@ -447,12 +448,14 @@ def manage_access_point_and_policy(self):
             access_point_arn = s3_client.create_bucket_access_point(self.bucket_name, self.access_point_name)
             # Access point creation is slow
             retries = 1
+            sleep_coeff = 1
             while (
                 not s3_client.get_bucket_access_point_arn(self.access_point_name)
                 and retries < ACCESS_POINT_CREATION_RETRIES
             ):
                 logger.info('Waiting 30s for access point creation to complete..')
-                time.sleep(ACCESS_POINT_CREATION_TIME)
+                time.sleep(ACCESS_POINT_CREATION_TIME * sleep_coeff)
+                sleep_coeff = sleep_coeff * ACCESS_POINT_BACKOFF_COEFFICIENT
                 retries += 1
         existing_policy = s3_client.get_access_point_policy(self.access_point_name)
         # requester will use this role to access resources

backend/dataall/modules/s3_datasets_shares/services/share_processors/glue_table_share_processor.py

@@ -153,7 +153,7 @@ def process_approved_shares(self) -> bool:
                     manager.grant_principals_permissions_to_source_table(table, share_item, share_item_filter)
                     if manager.cross_account:
                         retries = 0
-                        retry_share_table = False
+                        retry_share_table = True
                         while retry_share_table and retries < 1:
                             (
                                 retry_share_table,