Metadata form 7: Access control and deletion behaviour (#1540)

### Feature or Bugfix

- Feature


### Detail
- When entity is deleted all attached MF are deleted
- When Org or Env is deleted MF with visibility in this org|env is
deleted. And all connected attached MFs
- Resource policies: CREATE_METADATA_FORM (create MF with visibility
inside this entity), ATTACH_METADATA_FORM
- Migrations for entity owners to have all permissions 
- Triggers to delete dependencies for MF deletion and entity deletion

### Relates
- #1541

### Security
Please answer the questions below briefly where applicable, or write
`N/A`. Based on
[OWASP 10](https://owasp.org/Top10/en/).

- Does this PR introduce or modify any input fields or queries - this
includes
fetching data from storage outside the application (e.g. a database, an
S3 bucket)?
  - Is the input sanitized?
- What precautions are you taking before deserializing the data you
consume?
  - Is injection prevented by parametrizing queries?
  - Have you ensured no `eval` or similar functions are used?
- Does this PR introduce any functionality or component that requires
authorization?
- How have you ensured it respects the existing AuthN/AuthZ mechanisms?
  - Are you logging failed auth attempts?
- Are you using or adding any cryptographic features?
  - Do you use a standard proven implementations?
  - Are the used keys controlled by the customer? Where are they stored?
- Are you introducing any new policies/roles/users?
  - Have you used the least-privilege principle? How?


By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license.

---------

Co-authored-by: Sofia Sazonova <sazonova@amazon.co.uk>

backend/dataall/core/permissions/services/resource_policy_service.py

@@ -217,6 +217,7 @@ def get_resource_policy_permissions(session, group_uri, resource_uri) -> List[Re
             raise exceptions.RequiredParameter(param_name='group_uri')
         if not resource_uri:
             raise exceptions.RequiredParameter(param_name='resource_uri')
+
         policy = ResourcePolicyRepository.find_resource_policy(
             session=session,
             group_uri=group_uri,

backend/dataall/modules/metadata_forms/api/queries.py

@@ -5,6 +5,7 @@
     get_metadata_form,
     get_attached_metadata_form,
     list_attached_forms,
+    get_entity_metadata_form_permissions,
 )
 
 listUserMetadataForms = gql.QueryField(
@@ -47,3 +48,11 @@
     resolver=get_attached_metadata_form,
     test_scope='MetadataForm',
 )
+
+getEntityMetadataFormPermissions = gql.QueryField(
+    name='getEntityMetadataFormPermissions',
+    args=[gql.Argument('entityUri', gql.NonNullableType(gql.String))],
+    type=gql.ArrayType(gql.String),
+    resolver=get_entity_metadata_form_permissions,
+    test_scope='MetadataForm',
+)

backend/dataall/modules/metadata_forms/api/resolvers.py

@@ -94,3 +94,7 @@ def has_tenant_permissions_for_metadata_forms(context: Context, source: Metadata
 
 def resolve_metadata_form_field(context: Context, source: AttachedMetadataFormField):
     return MetadataFormService.get_metadata_form_field_by_uri(uri=source.fieldUri)
+
+
+def get_entity_metadata_form_permissions(context: Context, source, entityUri):
+    return MetadataFormService.get_mf_permissions(entityUri=entityUri)

backend/dataall/modules/metadata_forms/db/metadata_form_repository.py

@@ -70,6 +70,9 @@ def query_user_metadata_forms(session, is_da_admin, groups, env_uris, org_uris,
         :param filter:
         """
 
+        env_uris = env_uris or []
+        org_uris = org_uris or []
+
         query = session.query(MetadataForm)
 
         if not is_da_admin:
@@ -141,16 +144,22 @@ def query_entity_metadata_forms(
         entity_orgs_uris = entity_orgs_uris or []
         entity_envs_uris = entity_envs_uris or []
 
-        orgs = list(set(user_org_uris).intersection(set(entity_orgs_uris)))
-        envs = list(set(user_env_uris).intersection(set(entity_envs_uris)))
-
-        query = MetadataFormRepository.query_user_metadata_forms(session, is_da_admin, groups, envs, orgs, filter)
-
-        if not orgs:
-            query = query.filter(MetadataForm.visibility != MetadataFormVisibility.Organization.value)
+        query = MetadataFormRepository.query_user_metadata_forms(
+            session, is_da_admin, groups, user_env_uris, user_org_uris, filter
+        )
 
-        if not envs:
-            query = query.filter(MetadataForm.visibility != MetadataFormVisibility.Environment.value)
+        query = query.filter(
+            and_(
+                or_(
+                    MetadataForm.visibility != MetadataFormVisibility.Organization.value,
+                    MetadataForm.homeEntity.in_(entity_orgs_uris),
+                ),
+                or_(
+                    MetadataForm.visibility != MetadataFormVisibility.Environment.value,
+                    MetadataForm.homeEntity.in_(entity_envs_uris),
+                ),
+            )
+        )
 
         query = MetadataFormRepository.exclude_attached(session, query, filter)
         return query.order_by(MetadataForm.name)
@@ -247,3 +256,9 @@ def query_attached_metadata_forms(session, is_da_admin, groups, user_envs_uris,
         if filter and filter.get('metadataFormUri'):
             query = query.filter(AttachedMetadataForm.metadataFormUri == filter.get('metadataFormUri'))
         return query
+
+    @staticmethod
+    def query_all_attached_metadata_forms_for_entity(session, entityUri, entityType):
+        return session.query(AttachedMetadataForm).filter(
+            and_(AttachedMetadataForm.entityType == entityType, AttachedMetadataForm.entityUri == entityUri)
+        )

backend/dataall/modules/metadata_forms/services/attached_metadata_form_service.py

@@ -1,10 +1,9 @@
 from dataall.base.context import get_context
 from dataall.base.db import exceptions, paginate
-from dataall.core.environment.db.environment_repositories import EnvironmentRepository
-from dataall.core.organizations.db.organization_repositories import OrganizationRepository
-from dataall.core.permissions.services.tenant_policy_service import TenantPolicyValidationService
+from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService
 from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository
 from dataall.modules.metadata_forms.services.metadata_form_access_service import MetadataFormAccessService
+from dataall.modules.metadata_forms.services.metadata_form_permissions import ATTACH_METADATA_FORM
 
 
 class AttachedMetadataFormValidationService:
@@ -33,15 +32,22 @@ def validate_enrich_fields_params(mf_fields, data):
 
 class AttachedMetadataFormService:
     @staticmethod
+    def _get_entity_uri(data):
+        return data.get('entityUri')
+
+    @staticmethod
+    @ResourcePolicyService.has_resource_permission(
+        ATTACH_METADATA_FORM, parent_resource=_get_entity_uri, param_name='data'
+    )
     def create_attached_metadata_form(uri, data):
         AttachedMetadataFormValidationService.validate_filled_form_params(uri, data)
-        with get_context().db_engine.scoped_session() as session:
+        context = get_context()
+        with context.db_engine.scoped_session() as session:
             mf = MetadataFormRepository.get_metadata_form(session, uri)
             if not mf:
                 raise exceptions.ObjectNotFound('MetadataForm', uri)
             mf_fields = MetadataFormRepository.get_metadata_form_fields(session, uri)
             AttachedMetadataFormValidationService.validate_enrich_fields_params(mf_fields, data)
-
             amf = MetadataFormRepository.create_attached_metadata_form(session, uri, data)
             for f in data.get('fields'):
                 MetadataFormRepository.create_attached_metadata_form_field(
@@ -74,7 +80,11 @@ def list_attached_forms(filter=None):
             ).to_dict()
 
     @staticmethod
+    @ResourcePolicyService.has_resource_permission(
+        ATTACH_METADATA_FORM, parent_resource=_get_entity_uri, param_name='data'
+    )
     def delete_attached_metadata_form(uri):
         mf = AttachedMetadataFormService.get_attached_metadata_form(uri)
-        with get_context().db_engine.scoped_session() as session:
+        context = get_context()
+        with context.db_engine.scoped_session() as session:
             return session.delete(mf)

backend/dataall/modules/metadata_forms/services/metadata_form_permissions.py

@@ -1,6 +1,55 @@
+from dataall.core.permissions.services.environment_permissions import ENVIRONMENT_INVITED, ENVIRONMENT_ALL
+from dataall.core.permissions.services.organization_permissions import (
+    ORGANIZATION_ALL,
+    ORGANIZATION_INVITED_DESCRIPTIONS,
+)
 from dataall.core.permissions.services.tenant_permissions import TENANT_ALL, TENANT_ALL_WITH_DESC
+from dataall.core.permissions.services.resources_permissions import RESOURCES_ALL, RESOURCES_ALL_WITH_DESC
+from dataall.modules.s3_datasets.services.dataset_permissions import DATASET_WRITE, DATASET_ALL
 
-
+# ------------------------TENANT-----------------------------------
 MANAGE_METADATA_FORMS = 'MANAGE_METADATA_FORMS'
 TENANT_ALL.append(MANAGE_METADATA_FORMS)
 TENANT_ALL_WITH_DESC[MANAGE_METADATA_FORMS] = 'Manage metadata forms'
+
+# ------------------------RESOURCE---------------------------------
+# permissions to attach MF to the entity, ot make the entity the visibility base for MF
+# these permissions are attached to Organizations, Environments, Datasets etc.
+ATTACH_METADATA_FORM = 'ATTACH_METADATA_FORM'
+CREATE_METADATA_FORM = 'CREATE_METADATA_FORM'
+ALL_METADATA_FORMS_ENTITY_PERMISSIONS = [ATTACH_METADATA_FORM, CREATE_METADATA_FORM]
+RESOURCES_ALL.extend(ALL_METADATA_FORMS_ENTITY_PERMISSIONS)
+RESOURCES_ALL_WITH_DESC[CREATE_METADATA_FORM] = 'Create metadata form within this visibility scope'
+RESOURCES_ALL_WITH_DESC[ATTACH_METADATA_FORM] = 'Attach metadata form'
+
+ORGANIZATION_ALL.extend(ALL_METADATA_FORMS_ENTITY_PERMISSIONS)
+ORGANIZATION_INVITED_DESCRIPTIONS[CREATE_METADATA_FORM] = 'Create metadata form within this visibility scope'
+ORGANIZATION_INVITED_DESCRIPTIONS[ATTACH_METADATA_FORM] = 'Attach metadata form'
+
+ENVIRONMENT_INVITED.extend(ALL_METADATA_FORMS_ENTITY_PERMISSIONS)
+ENVIRONMENT_ALL.extend(ALL_METADATA_FORMS_ENTITY_PERMISSIONS)
+
+DATASET_WRITE.extend(ALL_METADATA_FORMS_ENTITY_PERMISSIONS)
+DATASET_ALL.extend(ALL_METADATA_FORMS_ENTITY_PERMISSIONS)
+# ------------------------METADATA FORM----------------------------
+# permissions to change and delete metadata forms
+# these permissions are attached to MFs
+UPDATE_METADATA_FORM_FIELD = 'UPDATE_METADATA_FORM_FIELD'
+DELETE_METADATA_FORM_FIELD = 'DELETE_METADATA_FORM_FIELD'
+DELETE_METADATA_FORM = 'DELETE_METADATA_FORM'
+EDIT_METADATA_FORM = 'EDIT_METADATA_FORM'
+
+METADATA_FORM_PERMISSIONS_ALL = [
+    UPDATE_METADATA_FORM_FIELD,
+    DELETE_METADATA_FORM_FIELD,
+    DELETE_METADATA_FORM,
+    EDIT_METADATA_FORM,
+]
+
+METADATA_FORM_EDIT_PERMISSIONS = [
+    EDIT_METADATA_FORM,
+    UPDATE_METADATA_FORM_FIELD,
+    DELETE_METADATA_FORM_FIELD,
+]
+
+RESOURCES_ALL.extend(METADATA_FORM_PERMISSIONS_ALL)

backend/dataall/modules/metadata_forms/services/metadata_form_service.py

@@ -2,15 +2,24 @@
 from dataall.base.db import exceptions, paginate
 from dataall.core.organizations.db.organization_repositories import OrganizationRepository
 from dataall.core.environment.db.environment_repositories import EnvironmentRepository
-from dataall.core.permissions.services.tenant_policy_service import TenantPolicyValidationService, TenantPolicyService
+from dataall.core.permissions.db.resource_policy.resource_policy_repositories import ResourcePolicyRepository
+from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService
+from dataall.core.permissions.services.tenant_policy_service import TenantPolicyService
 from dataall.modules.metadata_forms.db.enums import (
     MetadataFormVisibility,
     MetadataFormFieldType,
 )
 from dataall.modules.catalog.db.glossary_repositories import GlossaryRepository
 from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository
 from dataall.modules.metadata_forms.services.metadata_form_access_service import MetadataFormAccessService
-from dataall.modules.metadata_forms.services.metadata_form_permissions import MANAGE_METADATA_FORMS
+from dataall.modules.metadata_forms.services.metadata_form_permissions import (
+    MANAGE_METADATA_FORMS,
+    DELETE_METADATA_FORM,
+    DELETE_METADATA_FORM_FIELD,
+    UPDATE_METADATA_FORM_FIELD,
+    CREATE_METADATA_FORM,
+    ALL_METADATA_FORMS_ENTITY_PERMISSIONS,
+)
 
 
 class MetadataFormParamValidationService:
@@ -91,7 +100,20 @@ class MetadataFormService:
     @TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS)
     def create_metadata_form(data):
         MetadataFormParamValidationService.validate_create_form_params(data)
-        with get_context().db_engine.scoped_session() as session:
+        context = get_context()
+        with context.db_engine.scoped_session() as session:
+            if data.get('visibility') in [
+                MetadataFormVisibility.Organization.value,
+                MetadataFormVisibility.Environment.value,
+            ]:
+                ResourcePolicyService.check_user_resource_permission(
+                    session=session,
+                    username=context.username,
+                    groups=context.groups,
+                    resource_uri=data.get('homeEntity'),
+                    permission_name=CREATE_METADATA_FORM,
+                )
+
             form = MetadataFormRepository.create_metadata_form(session, data)
             return form
 
@@ -104,7 +126,7 @@ def get_metadata_form_by_uri(uri):
     # toDo: deletion logic
     @staticmethod
     @TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS)
-    @MetadataFormAccessService.can_perform('DELETE')
+    @MetadataFormAccessService.can_perform(DELETE_METADATA_FORM)
     def delete_metadata_form_by_uri(uri):
         if mf := MetadataFormService.get_metadata_form_by_uri(uri):
             with get_context().db_engine.scoped_session() as session:
@@ -181,15 +203,15 @@ def get_metadata_form_field_by_uri(uri):
 
     @staticmethod
     @TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS)
-    @MetadataFormAccessService.can_perform('ADD FIELD')
+    @MetadataFormAccessService.can_perform(UPDATE_METADATA_FORM_FIELD)
     def create_metadata_form_field(uri, data):
         MetadataFormParamValidationService.validate_create_field_params(data)
         with get_context().db_engine.scoped_session() as session:
             return MetadataFormRepository.create_metadata_form_field(session, uri, data)
 
     @staticmethod
     @TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS)
-    @MetadataFormAccessService.can_perform('ADD FIELDS')
+    @MetadataFormAccessService.can_perform(UPDATE_METADATA_FORM_FIELD)
     def create_metadata_form_fields(uri, data_arr):
         fields = []
         for data in data_arr:
@@ -198,15 +220,15 @@ def create_metadata_form_fields(uri, data_arr):
 
     @staticmethod
     @TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS)
-    @MetadataFormAccessService.can_perform('DELETE FIELD')
+    @MetadataFormAccessService.can_perform(DELETE_METADATA_FORM_FIELD)
     def delete_metadata_form_field(uri, fieldUri):
         mf = MetadataFormService.get_metadata_form_field_by_uri(fieldUri)
         with get_context().db_engine.scoped_session() as session:
             return session.delete(mf)
 
     @staticmethod
     @TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS)
-    @MetadataFormAccessService.can_perform('UPDATE FIELDS')
+    @MetadataFormAccessService.can_perform('UPDATE_METADATA_FORM_FIELD')
     def batch_metadata_form_field_update(uri, data):
         to_delete = []
         to_update = []
@@ -238,8 +260,23 @@ def batch_metadata_form_field_update(uri, data):
 
     @staticmethod
     @TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS)
-    @MetadataFormAccessService.can_perform('UPDATE FIELD')
+    @MetadataFormAccessService.can_perform(UPDATE_METADATA_FORM_FIELD)
     def update_metadata_form_field(uri, fieldUri, data):
         with get_context().db_engine.scoped_session() as session:
             MetadataFormParamValidationService.validate_update_field_params(uri, data)
             return MetadataFormRepository.update_metadata_form_field(session, fieldUri, data)
+
+    @staticmethod
+    def get_mf_permissions(entityUri):
+        context = get_context()
+        result_permissions = []
+        with context.db_engine.scoped_session() as session:
+            for permissions in ALL_METADATA_FORMS_ENTITY_PERMISSIONS:
+                if ResourcePolicyRepository.has_user_resource_permission(
+                    session=session,
+                    groups=context.groups,
+                    permission_name=permissions,
+                    resource_uri=entityUri,
+                ):
+                    result_permissions.append(permissions)
+            return result_permissions

backend/dataall/modules/redshift_datasets/services/redshift_dataset_service.py

@@ -193,6 +193,7 @@ def delete_redshift_dataset(uri):
             RedshiftDatasetService._delete_dataset_term_links(session, uri)
             VoteRepository.delete_votes(session, dataset.datasetUri, VOTE_REDSHIFT_DATASET_NAME)
             session.delete(dataset)
+
             session.commit()
             return True
 

backend/dataall/modules/s3_datasets/services/dataset_service.py

@@ -22,7 +22,6 @@
 from dataall.modules.catalog.db.glossary_repositories import GlossaryRepository
 from dataall.modules.s3_datasets.db.dataset_bucket_repositories import DatasetBucketRepository
 from dataall.modules.shares_base.db.share_object_repositories import ShareObjectRepository
-from dataall.modules.shares_base.services.share_object_service import ShareObjectService
 from dataall.modules.vote.db.vote_repositories import VoteRepository
 from dataall.modules.s3_datasets.aws.glue_dataset_client import DatasetCrawler
 from dataall.modules.s3_datasets.aws.s3_dataset_client import S3DatasetClient