From 719d86a7df5e8e329640418320c7ce5c40744889 Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Wed, 11 Sep 2024 13:04:45 +0100 Subject: [PATCH 01/26] when entity deleted, all attached MFs are deleted --- .../environment/services/environment_service.py | 5 +++++ .../organizations/services/organization_service.py | 5 +++++ .../metadata_forms/db/metadata_form_repository.py | 14 ++++++++++++++ .../services/redshift_dataset_service.py | 5 +++++ .../s3_datasets/services/dataset_service.py | 6 +++++- 5 files changed, 34 insertions(+), 1 deletion(-) diff --git a/backend/dataall/core/environment/services/environment_service.py b/backend/dataall/core/environment/services/environment_service.py index f49302ae4..f8f7a7239 100644 --- a/backend/dataall/core/environment/services/environment_service.py +++ b/backend/dataall/core/environment/services/environment_service.py @@ -46,6 +46,8 @@ from dataall.core.permissions.services.tenant_permissions import MANAGE_ENVIRONMENTS from dataall.core.stacks.db.stack_repositories import StackRepository from dataall.core.vpc.db.vpc_repositories import VpcRepository +from dataall.modules.metadata_forms.db.enums import MetadataFormEntityTypes +from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository log = logging.getLogger(__name__) @@ -885,6 +887,9 @@ def delete_environment(uri): KeyValueTagRepository.delete_key_value_tags(session, environment.environmentUri, 'environment') EnvironmentResourceManager.delete_env(session, environment) EnvironmentParameterRepository(session).delete_params(environment.environmentUri) + MetadataFormRepository.delete_attached_entity_metadata_forms( + session, environment.environmentUri, MetadataFormEntityTypes.Environments.value + ) for group in env_groups: session.delete(group) diff --git a/backend/dataall/core/organizations/services/organization_service.py b/backend/dataall/core/organizations/services/organization_service.py index 739717e81..b5b198f07 100644 --- a/backend/dataall/core/organizations/services/organization_service.py +++ b/backend/dataall/core/organizations/services/organization_service.py @@ -21,6 +21,8 @@ ORGANIZATION_INVITED_READONLY, ORGANIZATION_INVITED_DESCRIPTIONS, ) +from dataall.modules.metadata_forms.db.enums import MetadataFormEntityTypes +from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository class OrganizationService: @@ -175,6 +177,9 @@ def archive_organization(uri): resource_uri=org.organizationUri, resource_type=models.Organization.__name__, ) + MetadataFormRepository.delete_attached_entity_metadata_forms( + session, org.organizationUri, MetadataFormEntityTypes.Organizations.value + ) return True diff --git a/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py b/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py index cd404e396..8c26a9299 100644 --- a/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py +++ b/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py @@ -247,3 +247,17 @@ def query_attached_metadata_forms(session, is_da_admin, groups, user_envs_uris, if filter and filter.get('metadataFormUri'): query = query.filter(AttachedMetadataForm.metadataFormUri == filter.get('metadataFormUri')) return query + + @staticmethod + def get_all_attached_metadata_forms_for_entity(session, entityUri, entityType): + return ( + session.query(AttachedMetadataForm) + .filter(and_(AttachedMetadataForm.entityType == entityType, AttachedMetadataForm.entityUri == entityUri)) + .all() + ) + + @staticmethod + def delete_attached_entity_metadata_forms(session, entityUri, entityType): + mfs = MetadataFormRepository.get_all_attached_metadata_forms_for_entity(session, entityUri, entityType) + for mf in mfs: + session.delete(mf) diff --git a/backend/dataall/modules/redshift_datasets/services/redshift_dataset_service.py b/backend/dataall/modules/redshift_datasets/services/redshift_dataset_service.py index 573c50bd1..8eb497452 100644 --- a/backend/dataall/modules/redshift_datasets/services/redshift_dataset_service.py +++ b/backend/dataall/modules/redshift_datasets/services/redshift_dataset_service.py @@ -6,6 +6,8 @@ from dataall.core.permissions.services.tenant_policy_service import TenantPolicyService from dataall.core.permissions.services.group_policy_service import GroupPolicyService from dataall.core.environment.services.environment_service import EnvironmentService +from dataall.modules.metadata_forms.db.enums import MetadataFormEntityTypes +from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository from dataall.modules.vote.db.vote_repositories import VoteRepository from dataall.modules.catalog.db.glossary_repositories import GlossaryRepository @@ -184,6 +186,9 @@ def delete_redshift_dataset(uri): RedshiftDatasetService._delete_dataset_term_links(session, uri) VoteRepository.delete_votes(session, dataset.datasetUri, VOTE_REDSHIFT_DATASET_NAME) session.delete(dataset) + MetadataFormRepository.delete_attached_entity_metadata_forms( + session, dataset.datasetUri, MetadataFormEntityTypes.Datasets.value + ) session.commit() return True diff --git a/backend/dataall/modules/s3_datasets/services/dataset_service.py b/backend/dataall/modules/s3_datasets/services/dataset_service.py index 14cfdc2fd..47ea8f242 100644 --- a/backend/dataall/modules/s3_datasets/services/dataset_service.py +++ b/backend/dataall/modules/s3_datasets/services/dataset_service.py @@ -11,6 +11,8 @@ from dataall.core.stacks.services.stack_service import StackService from dataall.core.tasks.service_handlers import Worker from dataall.base.aws.sts import SessionHelper +from dataall.modules.metadata_forms.db.enums import MetadataFormEntityTypes +from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository from dataall.modules.s3_datasets.aws.kms_dataset_client import KmsClient from dataall.base.context import get_context from dataall.core.permissions.services.group_policy_service import GroupPolicyService @@ -22,7 +24,6 @@ from dataall.modules.catalog.db.glossary_repositories import GlossaryRepository from dataall.modules.s3_datasets.db.dataset_bucket_repositories import DatasetBucketRepository from dataall.modules.shares_base.db.share_object_repositories import ShareObjectRepository -from dataall.modules.shares_base.services.share_object_service import ShareObjectService from dataall.modules.vote.db.vote_repositories import VoteRepository from dataall.modules.s3_datasets.aws.glue_dataset_client import DatasetCrawler from dataall.modules.s3_datasets.aws.s3_dataset_client import S3DatasetClient @@ -446,6 +447,9 @@ def delete_dataset(uri: str, delete_from_aws: bool = False): if dataset.stewards: ResourcePolicyService.delete_resource_policy(session=session, resource_uri=uri, group=dataset.stewards) DatasetRepository.delete_dataset(session, dataset) + MetadataFormRepository.delete_attached_entity_metadata_forms( + session, dataset.datasetUri, MetadataFormEntityTypes.Datasets.value + ) if delete_from_aws: StackService.delete_stack( From 9c83eda774cb197a0e0a1d258d61f75010c06159 Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Wed, 11 Sep 2024 13:13:54 +0100 Subject: [PATCH 02/26] when env/org is deleted, all MFs with visibility inside this env/org and their children are deleted. --- .../core/environment/services/environment_service.py | 5 ++++- .../organizations/services/organization_service.py | 5 ++++- .../metadata_forms/db/metadata_form_repository.py | 10 ++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/backend/dataall/core/environment/services/environment_service.py b/backend/dataall/core/environment/services/environment_service.py index f8f7a7239..6f1a6cbee 100644 --- a/backend/dataall/core/environment/services/environment_service.py +++ b/backend/dataall/core/environment/services/environment_service.py @@ -46,7 +46,7 @@ from dataall.core.permissions.services.tenant_permissions import MANAGE_ENVIRONMENTS from dataall.core.stacks.db.stack_repositories import StackRepository from dataall.core.vpc.db.vpc_repositories import VpcRepository -from dataall.modules.metadata_forms.db.enums import MetadataFormEntityTypes +from dataall.modules.metadata_forms.db.enums import MetadataFormEntityTypes, MetadataFormVisibility from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository log = logging.getLogger(__name__) @@ -890,6 +890,9 @@ def delete_environment(uri): MetadataFormRepository.delete_attached_entity_metadata_forms( session, environment.environmentUri, MetadataFormEntityTypes.Environments.value ) + MetadataFormRepository.delete_all_home_metadata_forms( + session, environment.environmentUri, MetadataFormVisibility.Environment.value + ) for group in env_groups: session.delete(group) diff --git a/backend/dataall/core/organizations/services/organization_service.py b/backend/dataall/core/organizations/services/organization_service.py index b5b198f07..178b9918c 100644 --- a/backend/dataall/core/organizations/services/organization_service.py +++ b/backend/dataall/core/organizations/services/organization_service.py @@ -21,7 +21,7 @@ ORGANIZATION_INVITED_READONLY, ORGANIZATION_INVITED_DESCRIPTIONS, ) -from dataall.modules.metadata_forms.db.enums import MetadataFormEntityTypes +from dataall.modules.metadata_forms.db.enums import MetadataFormEntityTypes, MetadataFormVisibility from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository @@ -180,6 +180,9 @@ def archive_organization(uri): MetadataFormRepository.delete_attached_entity_metadata_forms( session, org.organizationUri, MetadataFormEntityTypes.Organizations.value ) + MetadataFormRepository.delete_all_home_metadata_forms( + session, org.organizationUri, MetadataFormVisibility.Organization.value + ) return True diff --git a/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py b/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py index 8c26a9299..8a0badefb 100644 --- a/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py +++ b/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py @@ -261,3 +261,13 @@ def delete_attached_entity_metadata_forms(session, entityUri, entityType): mfs = MetadataFormRepository.get_all_attached_metadata_forms_for_entity(session, entityUri, entityType) for mf in mfs: session.delete(mf) + + @staticmethod + def delete_all_home_metadata_forms(session, homeEntityUri, visibility): + mfs = ( + session.query(MetadataForm) + .filter(and_(MetadataForm.homeEntity == homeEntityUri, MetadataForm.visibility == visibility)) + .all() + ) + for mf in mfs: + session.delete(mf) From 232cf67e152c70acf325acdafa4a900f599c1d05 Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Wed, 11 Sep 2024 15:54:07 +0100 Subject: [PATCH 03/26] check resource permissions when create/attach/delete MF --- .../attached_metadata_form_service.py | 24 +++++++++++-- .../services/metadata_form_permissions.py | 30 +++++++++++++++- .../services/metadata_form_service.py | 36 ++++++++++++++----- 3 files changed, 78 insertions(+), 12 deletions(-) diff --git a/backend/dataall/modules/metadata_forms/services/attached_metadata_form_service.py b/backend/dataall/modules/metadata_forms/services/attached_metadata_form_service.py index 41f6067b4..e9e6789e5 100644 --- a/backend/dataall/modules/metadata_forms/services/attached_metadata_form_service.py +++ b/backend/dataall/modules/metadata_forms/services/attached_metadata_form_service.py @@ -2,9 +2,12 @@ from dataall.base.db import exceptions, paginate from dataall.core.environment.db.environment_repositories import EnvironmentRepository from dataall.core.organizations.db.organization_repositories import OrganizationRepository +from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService from dataall.core.permissions.services.tenant_policy_service import TenantPolicyValidationService +from dataall.modules.metadata_forms.db.enums import MetadataFormVisibility from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository from dataall.modules.metadata_forms.services.metadata_form_access_service import MetadataFormAccessService +from dataall.modules.metadata_forms.services.metadata_form_permissions import ATTACH_METADATA_FORM class AttachedMetadataFormValidationService: @@ -35,13 +38,20 @@ class AttachedMetadataFormService: @staticmethod def create_attached_metadata_form(uri, data): AttachedMetadataFormValidationService.validate_filled_form_params(uri, data) - with get_context().db_engine.scoped_session() as session: + context = get_context() + with context.db_engine.scoped_session() as session: mf = MetadataFormRepository.get_metadata_form(session, uri) if not mf: raise exceptions.ObjectNotFound('MetadataForm', uri) mf_fields = MetadataFormRepository.get_metadata_form_fields(session, uri) AttachedMetadataFormValidationService.validate_enrich_fields_params(mf_fields, data) - + ResourcePolicyService.check_user_resource_permission( + session=session, + username=context.username, + groups=context.groups, + resource_uri=data.get('entityUri'), + permission_name=ATTACH_METADATA_FORM, + ) amf = MetadataFormRepository.create_attached_metadata_form(session, uri, data) for f in data.get('fields'): MetadataFormRepository.create_attached_metadata_form_field( @@ -76,5 +86,13 @@ def list_attached_forms(filter=None): @staticmethod def delete_attached_metadata_form(uri): mf = AttachedMetadataFormService.get_attached_metadata_form(uri) - with get_context().db_engine.scoped_session() as session: + context = get_context() + with context.db_engine.scoped_session() as session: + ResourcePolicyService.check_user_resource_permission( + session=session, + username=context.username, + groups=context.groups, + resource_uri=mf.entityUri, + permission_name=ATTACH_METADATA_FORM, # attach and delete are the same for now + ) return session.delete(mf) diff --git a/backend/dataall/modules/metadata_forms/services/metadata_form_permissions.py b/backend/dataall/modules/metadata_forms/services/metadata_form_permissions.py index f5801f6e3..2438ba6d0 100644 --- a/backend/dataall/modules/metadata_forms/services/metadata_form_permissions.py +++ b/backend/dataall/modules/metadata_forms/services/metadata_form_permissions.py @@ -1,6 +1,34 @@ from dataall.core.permissions.services.tenant_permissions import TENANT_ALL, TENANT_ALL_WITH_DESC +from dataall.core.permissions.services.resources_permissions import RESOURCES_ALL, RESOURCES_ALL_WITH_DESC - +# ------------------------TENANT----------------------------------- MANAGE_METADATA_FORMS = 'MANAGE_METADATA_FORMS' TENANT_ALL.append(MANAGE_METADATA_FORMS) TENANT_ALL_WITH_DESC[MANAGE_METADATA_FORMS] = 'Manage metadata forms' + +# ------------------------RESOURCE--------------------------------- +# permissions to attach MF to the entity, ot make the entity the visibility base for MF +# these permissions are attached to Organizations, Environments, Datasets etc. +ATTACH_METADATA_FORM = 'ATTACH_METADATA_FORM' +CREATE_METADATA_FORM = 'CREATE_METADATA_FORM' +RESOURCES_ALL.extend([CREATE_METADATA_FORM, ATTACH_METADATA_FORM]) +RESOURCES_ALL_WITH_DESC[CREATE_METADATA_FORM] = 'Create metadata form within this visibility scope' +RESOURCES_ALL_WITH_DESC[ATTACH_METADATA_FORM] = 'Attach metadata form' + +# ------------------------METADATA FORM---------------------------- +# permissions to change and delete metadata forms +# these permissions are attached to MFs +UPDATE_METADATA_FORM_FIELD = 'UPDATE_METADATA_FORM_FIELD' +DELETE_METADATA_FORM_FIELD = 'DELETE_METADATA_FORM_FIELD' +DELETE_METADATA_FORM = 'DELETE_METADATA_FORM' +EDIT_METADATA_FORM = 'EDIT_METADATA_FORM' + +METADATA_FORM_PERMISSIONS_ALL = [UPDATE_METADATA_FORM_FIELD, DELETE_METADATA_FORM_FIELD, DELETE_METADATA_FORM] + +METADATA_FORM_EDIT_PERMISSIONS = [ + EDIT_METADATA_FORM, + UPDATE_METADATA_FORM_FIELD, + DELETE_METADATA_FORM_FIELD, +] + +RESOURCES_ALL.extend(METADATA_FORM_PERMISSIONS_ALL) diff --git a/backend/dataall/modules/metadata_forms/services/metadata_form_service.py b/backend/dataall/modules/metadata_forms/services/metadata_form_service.py index ac4ddbb91..b97d46402 100644 --- a/backend/dataall/modules/metadata_forms/services/metadata_form_service.py +++ b/backend/dataall/modules/metadata_forms/services/metadata_form_service.py @@ -2,6 +2,7 @@ from dataall.base.db import exceptions, paginate from dataall.core.organizations.db.organization_repositories import OrganizationRepository from dataall.core.environment.db.environment_repositories import EnvironmentRepository +from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService from dataall.core.permissions.services.tenant_policy_service import TenantPolicyValidationService, TenantPolicyService from dataall.modules.metadata_forms.db.enums import ( MetadataFormVisibility, @@ -10,7 +11,13 @@ from dataall.modules.catalog.db.glossary_repositories import GlossaryRepository from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository from dataall.modules.metadata_forms.services.metadata_form_access_service import MetadataFormAccessService -from dataall.modules.metadata_forms.services.metadata_form_permissions import MANAGE_METADATA_FORMS +from dataall.modules.metadata_forms.services.metadata_form_permissions import ( + MANAGE_METADATA_FORMS, + DELETE_METADATA_FORM, + DELETE_METADATA_FORM_FIELD, + UPDATE_METADATA_FORM_FIELD, + CREATE_METADATA_FORM, +) class MetadataFormParamValidationService: @@ -91,7 +98,20 @@ class MetadataFormService: @TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS) def create_metadata_form(data): MetadataFormParamValidationService.validate_create_form_params(data) - with get_context().db_engine.scoped_session() as session: + context = get_context() + with context.db_engine.scoped_session() as session: + if data.get('visibility') in [ + MetadataFormVisibility.Organization.value, + MetadataFormVisibility.Environment.value, + ]: + ResourcePolicyService.check_user_resource_permission( + session=session, + username=context.username, + groups=context.groups, + resource_uri=data.get('homeEntity'), + permission_name=CREATE_METADATA_FORM, + ) + form = MetadataFormRepository.create_metadata_form(session, data) return form @@ -104,7 +124,7 @@ def get_metadata_form_by_uri(uri): # toDo: deletion logic @staticmethod @TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS) - @MetadataFormAccessService.can_perform('DELETE') + @MetadataFormAccessService.can_perform(DELETE_METADATA_FORM) def delete_metadata_form_by_uri(uri): if mf := MetadataFormService.get_metadata_form_by_uri(uri): with get_context().db_engine.scoped_session() as session: @@ -181,7 +201,7 @@ def get_metadata_form_field_by_uri(uri): @staticmethod @TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS) - @MetadataFormAccessService.can_perform('ADD FIELD') + @MetadataFormAccessService.can_perform(UPDATE_METADATA_FORM_FIELD) def create_metadata_form_field(uri, data): MetadataFormParamValidationService.validate_create_field_params(data) with get_context().db_engine.scoped_session() as session: @@ -189,7 +209,7 @@ def create_metadata_form_field(uri, data): @staticmethod @TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS) - @MetadataFormAccessService.can_perform('ADD FIELDS') + @MetadataFormAccessService.can_perform(UPDATE_METADATA_FORM_FIELD) def create_metadata_form_fields(uri, data_arr): fields = [] for data in data_arr: @@ -198,7 +218,7 @@ def create_metadata_form_fields(uri, data_arr): @staticmethod @TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS) - @MetadataFormAccessService.can_perform('DELETE FIELD') + @MetadataFormAccessService.can_perform(DELETE_METADATA_FORM_FIELD) def delete_metadata_form_field(uri, fieldUri): mf = MetadataFormService.get_metadata_form_field_by_uri(fieldUri) with get_context().db_engine.scoped_session() as session: @@ -206,7 +226,7 @@ def delete_metadata_form_field(uri, fieldUri): @staticmethod @TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS) - @MetadataFormAccessService.can_perform('UPDATE FIELDS') + @MetadataFormAccessService.can_perform('UPDATE_METADATA_FORM_FIELD') def batch_metadata_form_field_update(uri, data): to_delete = [] to_update = [] @@ -238,7 +258,7 @@ def batch_metadata_form_field_update(uri, data): @staticmethod @TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS) - @MetadataFormAccessService.can_perform('UPDATE FIELD') + @MetadataFormAccessService.can_perform(UPDATE_METADATA_FORM_FIELD) def update_metadata_form_field(uri, fieldUri, data): with get_context().db_engine.scoped_session() as session: MetadataFormParamValidationService.validate_update_field_params(uri, data) From 1ec4b916ae1223787840d51fe194b8257805c6e1 Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Wed, 11 Sep 2024 16:05:18 +0100 Subject: [PATCH 04/26] migration to backfill MF resource permissions --- ...f31999_backfill_MF_resource_permissions.py | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py diff --git a/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py b/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py new file mode 100644 index 000000000..41130e6dd --- /dev/null +++ b/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py @@ -0,0 +1,66 @@ +"""backfill_MF_resource_permissions + +Revision ID: 427db8f31999 +Revises: f87aecc36d39 +Create Date: 2024-09-11 15:55:51.444403 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy import orm + +from dataall.core.environment.db.environment_models import EnvironmentGroup, Environment +from dataall.core.organizations.db.organization_models import OrganizationGroup, Organization +from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService +from dataall.modules.datasets_base.db.dataset_models import DatasetBase +from dataall.modules.metadata_forms.services.metadata_form_permissions import ATTACH_METADATA_FORM, CREATE_METADATA_FORM + +# revision identifiers, used by Alembic. +revision = '427db8f31999' +down_revision = 'f87aecc36d39' +branch_labels = None +depends_on = None + + +def get_session(): + bind = op.get_bind() + session = orm.Session(bind=bind) + return session + + +def upgrade(): + session = get_session() + print('Adding organization resource permissions...') + orgGroups = session.query(OrganizationGroup).all() + for group in orgGroups: + ResourcePolicyService.attach_resource_policy( + session=session, + group=group.groupUri, + resource_uri=group.organizationUri, + permissions=[ATTACH_METADATA_FORM, CREATE_METADATA_FORM], + resource_type=Organization.__name__, + ) + print('Adding environment resource permissions...') + envGroups = session.query(EnvironmentGroup).all() + for group in envGroups: + ResourcePolicyService.attach_resource_policy( + session=session, + group=group.groupUri, + resource_uri=group.environmentUri, + permissions=[ATTACH_METADATA_FORM, CREATE_METADATA_FORM], + resource_type=Environment.__name__, + ) + print('Adding dataset resource permissions...') + datasets = session.query(DatasetBase).all() + for dataset in datasets: + ResourcePolicyService.attach_resource_policy( + session=session, + group=dataset.SamlGroupName, + resource_uri=dataset.datasetUri, + permissions=[ATTACH_METADATA_FORM], + resource_type=DatasetBase.__name__, + ) + + +def downgrade(): + print('no downgrade supported') From 3d5242daf5bddbfa6dbba2963bcd814e7c7a21ff Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Thu, 12 Sep 2024 10:44:34 +0100 Subject: [PATCH 05/26] migrations fix --- ...7db8f31999_backfill_MF_resource_permissions.py | 15 +++++++++++++-- .../f87aecc36d39_attached_field_type_enum.py | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py b/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py index 41130e6dd..50236b5f3 100644 --- a/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py +++ b/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py @@ -6,14 +6,17 @@ """ from alembic import op -import sqlalchemy as sa from sqlalchemy import orm from dataall.core.environment.db.environment_models import EnvironmentGroup, Environment from dataall.core.organizations.db.organization_models import OrganizationGroup, Organization +from dataall.core.permissions.api.enums import PermissionType +from dataall.core.permissions.services.permission_service import PermissionService from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService +from dataall.core.permissions.services.resources_permissions import RESOURCES_ALL_WITH_DESC from dataall.modules.datasets_base.db.dataset_models import DatasetBase -from dataall.modules.metadata_forms.services.metadata_form_permissions import ATTACH_METADATA_FORM, CREATE_METADATA_FORM +from dataall.modules.metadata_forms.services.metadata_form_permissions import ATTACH_METADATA_FORM, \ + CREATE_METADATA_FORM, METADATA_FORM_PERMISSIONS_ALL # revision identifiers, used by Alembic. revision = '427db8f31999' @@ -30,6 +33,14 @@ def get_session(): def upgrade(): session = get_session() + + for perm in [ATTACH_METADATA_FORM, CREATE_METADATA_FORM] + METADATA_FORM_PERMISSIONS_ALL: + PermissionService.save_permission( + session, + name=perm, + description=RESOURCES_ALL_WITH_DESC.get(perm, perm), + permission_type=PermissionType.RESOURCE.name, + ) print('Adding organization resource permissions...') orgGroups = session.query(OrganizationGroup).all() for group in orgGroups: diff --git a/backend/migrations/versions/f87aecc36d39_attached_field_type_enum.py b/backend/migrations/versions/f87aecc36d39_attached_field_type_enum.py index 391b52f21..68ab75285 100644 --- a/backend/migrations/versions/f87aecc36d39_attached_field_type_enum.py +++ b/backend/migrations/versions/f87aecc36d39_attached_field_type_enum.py @@ -118,8 +118,8 @@ def upgrade(): def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.execute('DROP TYPE metadataformfieldtype') op.alter_column('attached_metadata_form_field', 'type', type_=sa.VARCHAR(), existing_nullable=True) + op.execute('DROP TYPE metadataformfieldtype CASCADE') # ### end Alembic commands ### # ### commands auto generated by Alembic - please adjust! ### From 1cdaa06c51242cb465031fe29cd2b46c16854435 Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Thu, 12 Sep 2024 13:03:56 +0100 Subject: [PATCH 06/26] only owners of the orgs/envs have permissions for MF --- ...f31999_backfill_MF_resource_permissions.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py b/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py index 50236b5f3..63288cb6a 100644 --- a/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py +++ b/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py @@ -8,8 +8,8 @@ from alembic import op from sqlalchemy import orm -from dataall.core.environment.db.environment_models import EnvironmentGroup, Environment -from dataall.core.organizations.db.organization_models import OrganizationGroup, Organization +from dataall.core.environment.db.environment_models import Environment +from dataall.core.organizations.db.organization_models import Organization from dataall.core.permissions.api.enums import PermissionType from dataall.core.permissions.services.permission_service import PermissionService from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService @@ -42,22 +42,22 @@ def upgrade(): permission_type=PermissionType.RESOURCE.name, ) print('Adding organization resource permissions...') - orgGroups = session.query(OrganizationGroup).all() - for group in orgGroups: + orgs = session.query(Organization).all() + for org in orgs: ResourcePolicyService.attach_resource_policy( session=session, - group=group.groupUri, - resource_uri=group.organizationUri, + group=org.SamlGroupName, + resource_uri=org.organizationUri, permissions=[ATTACH_METADATA_FORM, CREATE_METADATA_FORM], resource_type=Organization.__name__, ) print('Adding environment resource permissions...') - envGroups = session.query(EnvironmentGroup).all() - for group in envGroups: + envs = session.query(Environment).all() + for env in envs: ResourcePolicyService.attach_resource_policy( session=session, - group=group.groupUri, - resource_uri=group.environmentUri, + group=env.SamlGroupName, + resource_uri=env.environmentUri, permissions=[ATTACH_METADATA_FORM, CREATE_METADATA_FORM], resource_type=Environment.__name__, ) From e267e5c4a12cabe0f8944fd25f2fd2f0cf88a9e1 Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Thu, 12 Sep 2024 13:11:36 +0100 Subject: [PATCH 07/26] add resource policies for MFs to ENV, ORG and DATASET policy lists --- .../services/metadata_form_permissions.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/backend/dataall/modules/metadata_forms/services/metadata_form_permissions.py b/backend/dataall/modules/metadata_forms/services/metadata_form_permissions.py index 2438ba6d0..0e57e2829 100644 --- a/backend/dataall/modules/metadata_forms/services/metadata_form_permissions.py +++ b/backend/dataall/modules/metadata_forms/services/metadata_form_permissions.py @@ -1,5 +1,9 @@ +from dataall.core.permissions.services.environment_permissions import ENVIRONMENT_INVITED, ENVIRONMENT_ALL +from dataall.core.permissions.services.organization_permissions import ORGANIZATION_ALL, \ + ORGANIZATION_INVITED_DESCRIPTIONS from dataall.core.permissions.services.tenant_permissions import TENANT_ALL, TENANT_ALL_WITH_DESC from dataall.core.permissions.services.resources_permissions import RESOURCES_ALL, RESOURCES_ALL_WITH_DESC +from dataall.modules.s3_datasets.services.dataset_permissions import DATASET_WRITE, DATASET_ALL # ------------------------TENANT----------------------------------- MANAGE_METADATA_FORMS = 'MANAGE_METADATA_FORMS' @@ -15,6 +19,15 @@ RESOURCES_ALL_WITH_DESC[CREATE_METADATA_FORM] = 'Create metadata form within this visibility scope' RESOURCES_ALL_WITH_DESC[ATTACH_METADATA_FORM] = 'Attach metadata form' +ORGANIZATION_ALL.extend([CREATE_METADATA_FORM, ATTACH_METADATA_FORM]) +ORGANIZATION_INVITED_DESCRIPTIONS[CREATE_METADATA_FORM] = 'Create metadata form within this visibility scope' +ORGANIZATION_INVITED_DESCRIPTIONS[ATTACH_METADATA_FORM] = 'Attach metadata form' + +ENVIRONMENT_INVITED.extend([CREATE_METADATA_FORM, ATTACH_METADATA_FORM]) +ENVIRONMENT_ALL.extend([CREATE_METADATA_FORM, ATTACH_METADATA_FORM]) + +DATASET_WRITE.extend([CREATE_METADATA_FORM, ATTACH_METADATA_FORM]) +DATASET_ALL.extend([CREATE_METADATA_FORM, ATTACH_METADATA_FORM]) # ------------------------METADATA FORM---------------------------- # permissions to change and delete metadata forms # these permissions are attached to MFs From 84250f8ac1651917db6b374545c42b80887043db Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Thu, 12 Sep 2024 14:01:01 +0100 Subject: [PATCH 08/26] frontend canEdit depends on resource policies --- .../services/environment_service.py | 1 - .../services/organization_service.py | 6 +----- .../services/resource_policy_service.py | 21 ++++++++++--------- .../modules/metadata_forms/api/queries.py | 9 ++++++++ .../modules/metadata_forms/api/resolvers.py | 4 ++++ .../services/metadata_form_permissions.py | 19 ++++++++++------- .../services/metadata_form_service.py | 17 +++++++++++++++ ...f31999_backfill_MF_resource_permissions.py | 8 +++++-- ...1ac7a85a2_drop_remove_group_permissions.py | 2 +- .../Environments/views/EnvironmentView.js | 1 - .../components/metadataAttachment.js | 13 ++++++++++-- .../getMetadataFormEntityPermissions.js | 12 +++++++++++ .../modules/Metadata_Forms/services/index.js | 1 + .../Organizations/views/OrganizationView.js | 1 - .../modules/S3_Datasets/views/DatasetView.js | 1 - 15 files changed, 84 insertions(+), 32 deletions(-) create mode 100644 frontend/src/modules/Metadata_Forms/services/getMetadataFormEntityPermissions.js diff --git a/backend/dataall/core/environment/services/environment_service.py b/backend/dataall/core/environment/services/environment_service.py index 6f1a6cbee..34e1c0e36 100644 --- a/backend/dataall/core/environment/services/environment_service.py +++ b/backend/dataall/core/environment/services/environment_service.py @@ -527,7 +527,6 @@ def list_group_permissions_internal(session, uri, group_uri): environment = EnvironmentService.get_environment_by_uri(session, uri) return ResourcePolicyService.get_resource_policy_permissions( - session=session, group_uri=group_uri, resource_uri=environment.environmentUri, ) diff --git a/backend/dataall/core/organizations/services/organization_service.py b/backend/dataall/core/organizations/services/organization_service.py index 178b9918c..7e4191ae5 100644 --- a/backend/dataall/core/organizations/services/organization_service.py +++ b/backend/dataall/core/organizations/services/organization_service.py @@ -317,11 +317,7 @@ def resolve_organization_by_env(uri): @staticmethod @ResourcePolicyService.has_resource_permission(GET_ORGANIZATION) def list_group_organization_permissions(uri, groupUri): - context = get_context() - with context.db_engine.scoped_session() as session: - return ResourcePolicyService.get_resource_policy_permissions( - session=session, group_uri=groupUri, resource_uri=uri - ) + return ResourcePolicyService.get_resource_policy_permissions(group_uri=groupUri, resource_uri=uri) @staticmethod def list_invited_organization_permissions_with_descriptions(): diff --git a/backend/dataall/core/permissions/services/resource_policy_service.py b/backend/dataall/core/permissions/services/resource_policy_service.py index 95de63743..a44954648 100644 --- a/backend/dataall/core/permissions/services/resource_policy_service.py +++ b/backend/dataall/core/permissions/services/resource_policy_service.py @@ -212,20 +212,21 @@ def associate_permission_to_resource_policy(session, policy, permission): session.commit() @staticmethod - def get_resource_policy_permissions(session, group_uri, resource_uri) -> List[ResourcePolicyPermission]: + def get_resource_policy_permissions(group_uri, resource_uri) -> List[ResourcePolicyPermission]: if not group_uri: raise exceptions.RequiredParameter(param_name='group_uri') if not resource_uri: raise exceptions.RequiredParameter(param_name='resource_uri') - policy = ResourcePolicyRepository.find_resource_policy( - session=session, - group_uri=group_uri, - resource_uri=resource_uri, - ) - permissions = [] - for p in policy.permissions: - permissions.append(p.permission) - return permissions + with get_context().db_engine.scoped_session() as session: + policy = ResourcePolicyRepository.find_resource_policy( + session=session, + group_uri=group_uri, + resource_uri=resource_uri, + ) + permissions = [] + for p in policy.permissions: + permissions.append(p.permission) + return permissions @staticmethod def has_resource_permission( diff --git a/backend/dataall/modules/metadata_forms/api/queries.py b/backend/dataall/modules/metadata_forms/api/queries.py index 4458e3860..e9385e091 100644 --- a/backend/dataall/modules/metadata_forms/api/queries.py +++ b/backend/dataall/modules/metadata_forms/api/queries.py @@ -5,6 +5,7 @@ get_metadata_form, get_attached_metadata_form, list_attached_forms, + get_entity_metadata_form_permissions, ) listUserMetadataForms = gql.QueryField( @@ -47,3 +48,11 @@ resolver=get_attached_metadata_form, test_scope='MetadataForm', ) + +getEntityMetadataFormPermissions = gql.QueryField( + name='getEntityMetadataFormPermissions', + args=[gql.Argument('entityUri', gql.NonNullableType(gql.String))], + type=gql.ArrayType(gql.String), + resolver=get_entity_metadata_form_permissions, + test_scope='MetadataForm', +) diff --git a/backend/dataall/modules/metadata_forms/api/resolvers.py b/backend/dataall/modules/metadata_forms/api/resolvers.py index 390a0938f..4e03e6d6c 100644 --- a/backend/dataall/modules/metadata_forms/api/resolvers.py +++ b/backend/dataall/modules/metadata_forms/api/resolvers.py @@ -94,3 +94,7 @@ def has_tenant_permissions_for_metadata_forms(context: Context, source: Metadata def resolve_metadata_form_field(context: Context, source: AttachedMetadataFormField): return MetadataFormService.get_metadata_form_field_by_uri(uri=source.fieldUri) + + +def get_entity_metadata_form_permissions(context: Context, source, entityUri): + return MetadataFormService.get_mf_permissions(entityUri=entityUri) diff --git a/backend/dataall/modules/metadata_forms/services/metadata_form_permissions.py b/backend/dataall/modules/metadata_forms/services/metadata_form_permissions.py index 0e57e2829..a5502c170 100644 --- a/backend/dataall/modules/metadata_forms/services/metadata_form_permissions.py +++ b/backend/dataall/modules/metadata_forms/services/metadata_form_permissions.py @@ -1,6 +1,8 @@ from dataall.core.permissions.services.environment_permissions import ENVIRONMENT_INVITED, ENVIRONMENT_ALL -from dataall.core.permissions.services.organization_permissions import ORGANIZATION_ALL, \ - ORGANIZATION_INVITED_DESCRIPTIONS +from dataall.core.permissions.services.organization_permissions import ( + ORGANIZATION_ALL, + ORGANIZATION_INVITED_DESCRIPTIONS, +) from dataall.core.permissions.services.tenant_permissions import TENANT_ALL, TENANT_ALL_WITH_DESC from dataall.core.permissions.services.resources_permissions import RESOURCES_ALL, RESOURCES_ALL_WITH_DESC from dataall.modules.s3_datasets.services.dataset_permissions import DATASET_WRITE, DATASET_ALL @@ -15,19 +17,20 @@ # these permissions are attached to Organizations, Environments, Datasets etc. ATTACH_METADATA_FORM = 'ATTACH_METADATA_FORM' CREATE_METADATA_FORM = 'CREATE_METADATA_FORM' -RESOURCES_ALL.extend([CREATE_METADATA_FORM, ATTACH_METADATA_FORM]) +ALL_METADATA_FORMS_ENTITY_PERMISSIONS = [ATTACH_METADATA_FORM, CREATE_METADATA_FORM] +RESOURCES_ALL.extend(ALL_METADATA_FORMS_ENTITY_PERMISSIONS) RESOURCES_ALL_WITH_DESC[CREATE_METADATA_FORM] = 'Create metadata form within this visibility scope' RESOURCES_ALL_WITH_DESC[ATTACH_METADATA_FORM] = 'Attach metadata form' -ORGANIZATION_ALL.extend([CREATE_METADATA_FORM, ATTACH_METADATA_FORM]) +ORGANIZATION_ALL.extend(ALL_METADATA_FORMS_ENTITY_PERMISSIONS) ORGANIZATION_INVITED_DESCRIPTIONS[CREATE_METADATA_FORM] = 'Create metadata form within this visibility scope' ORGANIZATION_INVITED_DESCRIPTIONS[ATTACH_METADATA_FORM] = 'Attach metadata form' -ENVIRONMENT_INVITED.extend([CREATE_METADATA_FORM, ATTACH_METADATA_FORM]) -ENVIRONMENT_ALL.extend([CREATE_METADATA_FORM, ATTACH_METADATA_FORM]) +ENVIRONMENT_INVITED.extend(ALL_METADATA_FORMS_ENTITY_PERMISSIONS) +ENVIRONMENT_ALL.extend(ALL_METADATA_FORMS_ENTITY_PERMISSIONS) -DATASET_WRITE.extend([CREATE_METADATA_FORM, ATTACH_METADATA_FORM]) -DATASET_ALL.extend([CREATE_METADATA_FORM, ATTACH_METADATA_FORM]) +DATASET_WRITE.extend(ALL_METADATA_FORMS_ENTITY_PERMISSIONS) +DATASET_ALL.extend(ALL_METADATA_FORMS_ENTITY_PERMISSIONS) # ------------------------METADATA FORM---------------------------- # permissions to change and delete metadata forms # these permissions are attached to MFs diff --git a/backend/dataall/modules/metadata_forms/services/metadata_form_service.py b/backend/dataall/modules/metadata_forms/services/metadata_form_service.py index b97d46402..50ff480f3 100644 --- a/backend/dataall/modules/metadata_forms/services/metadata_form_service.py +++ b/backend/dataall/modules/metadata_forms/services/metadata_form_service.py @@ -17,6 +17,7 @@ DELETE_METADATA_FORM_FIELD, UPDATE_METADATA_FORM_FIELD, CREATE_METADATA_FORM, + ALL_METADATA_FORMS_ENTITY_PERMISSIONS, ) @@ -263,3 +264,19 @@ def update_metadata_form_field(uri, fieldUri, data): with get_context().db_engine.scoped_session() as session: MetadataFormParamValidationService.validate_update_field_params(uri, data) return MetadataFormRepository.update_metadata_form_field(session, fieldUri, data) + + @staticmethod + def get_mf_permissions(entityUri): + context = get_context() + result_permissions = [] + with context.db_engine.scoped_session() as session: + for permissions in ALL_METADATA_FORMS_ENTITY_PERMISSIONS: + if ResourcePolicyService.check_user_resource_permission( + session=session, + username=context.username, + groups=context.groups, + resource_uri=entityUri, + permission_name=permissions, + ): + result_permissions.append(permissions) + return result_permissions diff --git a/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py b/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py index 63288cb6a..4ecdfe4b2 100644 --- a/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py +++ b/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py @@ -5,6 +5,7 @@ Create Date: 2024-09-11 15:55:51.444403 """ + from alembic import op from sqlalchemy import orm @@ -15,8 +16,11 @@ from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService from dataall.core.permissions.services.resources_permissions import RESOURCES_ALL_WITH_DESC from dataall.modules.datasets_base.db.dataset_models import DatasetBase -from dataall.modules.metadata_forms.services.metadata_form_permissions import ATTACH_METADATA_FORM, \ - CREATE_METADATA_FORM, METADATA_FORM_PERMISSIONS_ALL +from dataall.modules.metadata_forms.services.metadata_form_permissions import ( + ATTACH_METADATA_FORM, + CREATE_METADATA_FORM, + METADATA_FORM_PERMISSIONS_ALL, +) # revision identifiers, used by Alembic. revision = '427db8f31999' diff --git a/backend/migrations/versions/a991ac7a85a2_drop_remove_group_permissions.py b/backend/migrations/versions/a991ac7a85a2_drop_remove_group_permissions.py index 64ded7dd5..a6dcea574 100644 --- a/backend/migrations/versions/a991ac7a85a2_drop_remove_group_permissions.py +++ b/backend/migrations/versions/a991ac7a85a2_drop_remove_group_permissions.py @@ -45,7 +45,7 @@ def upgrade(): .all() ) for group in suspicious_permissions_principals: - permissions = ResourcePolicyService.get_resource_policy_permissions(session, group, env.environmentUri) + permissions = ResourcePolicyService.get_resource_policy_permissions(group, env.environmentUri) permissions = [permission.name for permission in permissions if permission.name != REMOVE_ENVIRONMENT_GROUP] ResourcePolicyService.update_resource_policy( session, diff --git a/frontend/src/modules/Environments/views/EnvironmentView.js b/frontend/src/modules/Environments/views/EnvironmentView.js index 790d97f57..de5d91ee1 100644 --- a/frontend/src/modules/Environments/views/EnvironmentView.js +++ b/frontend/src/modules/Environments/views/EnvironmentView.js @@ -275,7 +275,6 @@ const EnvironmentView = () => { )} {currentTab === 'teams' && } diff --git a/frontend/src/modules/Metadata_Forms/components/metadataAttachment.js b/frontend/src/modules/Metadata_Forms/components/metadataAttachment.js index e29f78675..c68a2d31c 100644 --- a/frontend/src/modules/Metadata_Forms/components/metadataAttachment.js +++ b/frontend/src/modules/Metadata_Forms/components/metadataAttachment.js @@ -15,7 +15,7 @@ import { } from '@mui/material'; import { deleteAttachedMetadataForm, - getAttachedMetadataForm, + getAttachedMetadataForm, getEntityMetadataFormPermissions, getMetadataForm, listAttachedMetadataForms, listEntityMetadataForms @@ -30,7 +30,7 @@ import DoNotDisturbAltOutlinedIcon from '@mui/icons-material/DoNotDisturbAltOutl import DeleteIcon from '@mui/icons-material/DeleteOutlined'; export const MetadataAttachment = (props) => { - const { entityType, entityUri, canEdit } = props; + const { entityType, entityUri } = props; const client = useClient(); const dispatch = useDispatch(); const [selectedForm, setSelectedForm] = useState(null); @@ -38,6 +38,7 @@ export const MetadataAttachment = (props) => { const [loadingFields, setLoadingFields] = useState(false); const [formsList, setFormsList] = useState([]); const [fields, setFields] = useState([]); + const [canEdit, setCanEdit] = useState(false); const [filter] = useState({ ...Defaults.filter, entityType: entityType, @@ -139,9 +140,17 @@ export const MetadataAttachment = (props) => { } }; + const getPermissions = async () => { + const response = await client.query(getEntityMetadataFormPermissions(entityUri)); + if (!response.errors) { + setCanEdit(response.data.getEntityMetadataFormPermissions.includes('ATTACH_METADATA_FORM')); + } + } + useEffect(() => { if (client) { fetchList().catch((e) => dispatch({ type: SET_ERROR, error: e.message })); + getPermissions().catch((e) => dispatch({ type: SET_ERROR, error: e.message })); fetchAvailableForms().catch((e) => dispatch({ type: SET_ERROR, error: e.message }) ); diff --git a/frontend/src/modules/Metadata_Forms/services/getMetadataFormEntityPermissions.js b/frontend/src/modules/Metadata_Forms/services/getMetadataFormEntityPermissions.js new file mode 100644 index 000000000..9db92bf32 --- /dev/null +++ b/frontend/src/modules/Metadata_Forms/services/getMetadataFormEntityPermissions.js @@ -0,0 +1,12 @@ +import { gql } from 'apollo-boost'; + +export const getEntityMetadataFormPermissions = (entityUri) => ({ + variables: { + entityUri + }, + query: gql` + query getEntityMetadataFormPermissions($entityUri: String!) { + getEntityMetadataFormPermissions(entityUri: $entityUri) + } + ` +}); \ No newline at end of file diff --git a/frontend/src/modules/Metadata_Forms/services/index.js b/frontend/src/modules/Metadata_Forms/services/index.js index 99e3b2312..74c383379 100644 --- a/frontend/src/modules/Metadata_Forms/services/index.js +++ b/frontend/src/modules/Metadata_Forms/services/index.js @@ -8,3 +8,4 @@ export * from './attachMetadataForm'; export * from './getAttachedMetadataForm'; export * from './listAttachedMetadataForms'; export * from './deleteAttachedMetadataForm'; +export * from './getMetadataFormEntityPermissions'; diff --git a/frontend/src/modules/Organizations/views/OrganizationView.js b/frontend/src/modules/Organizations/views/OrganizationView.js index ccc868a39..758409c46 100644 --- a/frontend/src/modules/Organizations/views/OrganizationView.js +++ b/frontend/src/modules/Organizations/views/OrganizationView.js @@ -237,7 +237,6 @@ const OrganizationView = () => { )} diff --git a/frontend/src/modules/S3_Datasets/views/DatasetView.js b/frontend/src/modules/S3_Datasets/views/DatasetView.js index 6f1800a64..91eff5588 100644 --- a/frontend/src/modules/S3_Datasets/views/DatasetView.js +++ b/frontend/src/modules/S3_Datasets/views/DatasetView.js @@ -365,7 +365,6 @@ const DatasetView = () => { )} {currentTab === 'overview' && ( From 84a80670e811768656ecfdd6911ffbc5b933cf37 Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Thu, 12 Sep 2024 14:03:25 +0100 Subject: [PATCH 09/26] linting --- .../components/metadataAttachment.js | 19 ++++++++++++++----- .../getMetadataFormEntityPermissions.js | 4 ++-- .../modules/S3_Datasets/views/DatasetView.js | 5 +---- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/frontend/src/modules/Metadata_Forms/components/metadataAttachment.js b/frontend/src/modules/Metadata_Forms/components/metadataAttachment.js index c68a2d31c..1ac206290 100644 --- a/frontend/src/modules/Metadata_Forms/components/metadataAttachment.js +++ b/frontend/src/modules/Metadata_Forms/components/metadataAttachment.js @@ -15,7 +15,8 @@ import { } from '@mui/material'; import { deleteAttachedMetadataForm, - getAttachedMetadataForm, getEntityMetadataFormPermissions, + getAttachedMetadataForm, + getEntityMetadataFormPermissions, getMetadataForm, listAttachedMetadataForms, listEntityMetadataForms @@ -141,16 +142,24 @@ export const MetadataAttachment = (props) => { }; const getPermissions = async () => { - const response = await client.query(getEntityMetadataFormPermissions(entityUri)); + const response = await client.query( + getEntityMetadataFormPermissions(entityUri) + ); if (!response.errors) { - setCanEdit(response.data.getEntityMetadataFormPermissions.includes('ATTACH_METADATA_FORM')); + setCanEdit( + response.data.getEntityMetadataFormPermissions.includes( + 'ATTACH_METADATA_FORM' + ) + ); } - } + }; useEffect(() => { if (client) { fetchList().catch((e) => dispatch({ type: SET_ERROR, error: e.message })); - getPermissions().catch((e) => dispatch({ type: SET_ERROR, error: e.message })); + getPermissions().catch((e) => + dispatch({ type: SET_ERROR, error: e.message }) + ); fetchAvailableForms().catch((e) => dispatch({ type: SET_ERROR, error: e.message }) ); diff --git a/frontend/src/modules/Metadata_Forms/services/getMetadataFormEntityPermissions.js b/frontend/src/modules/Metadata_Forms/services/getMetadataFormEntityPermissions.js index 9db92bf32..ca268874b 100644 --- a/frontend/src/modules/Metadata_Forms/services/getMetadataFormEntityPermissions.js +++ b/frontend/src/modules/Metadata_Forms/services/getMetadataFormEntityPermissions.js @@ -6,7 +6,7 @@ export const getEntityMetadataFormPermissions = (entityUri) => ({ }, query: gql` query getEntityMetadataFormPermissions($entityUri: String!) { - getEntityMetadataFormPermissions(entityUri: $entityUri) + getEntityMetadataFormPermissions(entityUri: $entityUri) } ` -}); \ No newline at end of file +}); diff --git a/frontend/src/modules/S3_Datasets/views/DatasetView.js b/frontend/src/modules/S3_Datasets/views/DatasetView.js index 91eff5588..f47b376c1 100644 --- a/frontend/src/modules/S3_Datasets/views/DatasetView.js +++ b/frontend/src/modules/S3_Datasets/views/DatasetView.js @@ -362,10 +362,7 @@ const DatasetView = () => { )} {currentTab === 'metadata' && ( - + )} {currentTab === 'overview' && ( From 96828ff37a1405df426f30bf49cc7001b5cef8fd Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Fri, 13 Sep 2024 10:26:46 +0100 Subject: [PATCH 10/26] PR comments --- .../services/attached_metadata_form_service.py | 4 ---- .../metadata_forms/services/metadata_form_permissions.py | 7 ++++++- .../metadata_forms/services/metadata_form_service.py | 8 ++++---- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/backend/dataall/modules/metadata_forms/services/attached_metadata_form_service.py b/backend/dataall/modules/metadata_forms/services/attached_metadata_form_service.py index e9e6789e5..4af2e8f84 100644 --- a/backend/dataall/modules/metadata_forms/services/attached_metadata_form_service.py +++ b/backend/dataall/modules/metadata_forms/services/attached_metadata_form_service.py @@ -1,10 +1,6 @@ from dataall.base.context import get_context from dataall.base.db import exceptions, paginate -from dataall.core.environment.db.environment_repositories import EnvironmentRepository -from dataall.core.organizations.db.organization_repositories import OrganizationRepository from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService -from dataall.core.permissions.services.tenant_policy_service import TenantPolicyValidationService -from dataall.modules.metadata_forms.db.enums import MetadataFormVisibility from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository from dataall.modules.metadata_forms.services.metadata_form_access_service import MetadataFormAccessService from dataall.modules.metadata_forms.services.metadata_form_permissions import ATTACH_METADATA_FORM diff --git a/backend/dataall/modules/metadata_forms/services/metadata_form_permissions.py b/backend/dataall/modules/metadata_forms/services/metadata_form_permissions.py index a5502c170..883977a16 100644 --- a/backend/dataall/modules/metadata_forms/services/metadata_form_permissions.py +++ b/backend/dataall/modules/metadata_forms/services/metadata_form_permissions.py @@ -39,7 +39,12 @@ DELETE_METADATA_FORM = 'DELETE_METADATA_FORM' EDIT_METADATA_FORM = 'EDIT_METADATA_FORM' -METADATA_FORM_PERMISSIONS_ALL = [UPDATE_METADATA_FORM_FIELD, DELETE_METADATA_FORM_FIELD, DELETE_METADATA_FORM] +METADATA_FORM_PERMISSIONS_ALL = [ + UPDATE_METADATA_FORM_FIELD, + DELETE_METADATA_FORM_FIELD, + DELETE_METADATA_FORM, + EDIT_METADATA_FORM, +] METADATA_FORM_EDIT_PERMISSIONS = [ EDIT_METADATA_FORM, diff --git a/backend/dataall/modules/metadata_forms/services/metadata_form_service.py b/backend/dataall/modules/metadata_forms/services/metadata_form_service.py index 50ff480f3..5ac4af784 100644 --- a/backend/dataall/modules/metadata_forms/services/metadata_form_service.py +++ b/backend/dataall/modules/metadata_forms/services/metadata_form_service.py @@ -2,8 +2,9 @@ from dataall.base.db import exceptions, paginate from dataall.core.organizations.db.organization_repositories import OrganizationRepository from dataall.core.environment.db.environment_repositories import EnvironmentRepository +from dataall.core.permissions.db.resource_policy.resource_policy_repositories import ResourcePolicyRepository from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService -from dataall.core.permissions.services.tenant_policy_service import TenantPolicyValidationService, TenantPolicyService +from dataall.core.permissions.services.tenant_policy_service import TenantPolicyService from dataall.modules.metadata_forms.db.enums import ( MetadataFormVisibility, MetadataFormFieldType, @@ -271,12 +272,11 @@ def get_mf_permissions(entityUri): result_permissions = [] with context.db_engine.scoped_session() as session: for permissions in ALL_METADATA_FORMS_ENTITY_PERMISSIONS: - if ResourcePolicyService.check_user_resource_permission( + if ResourcePolicyRepository.has_user_resource_permission( session=session, - username=context.username, groups=context.groups, - resource_uri=entityUri, permission_name=permissions, + resource_uri=entityUri, ): result_permissions.append(permissions) return result_permissions From dabc41cc4eeef59006fb26f9587f60d20e667a9a Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Fri, 13 Sep 2024 13:22:20 +0100 Subject: [PATCH 11/26] instead of connecting core and modules via logic -- DB triggers --- .../services/environment_service.py | 8 -- .../services/organization_service.py | 8 -- .../services/redshift_dataset_service.py | 6 +- .../s3_datasets/services/dataset_service.py | 5 - .../versions/075d344ae2cc_mf_triggers.py | 123 ++++++++++++++++++ 5 files changed, 124 insertions(+), 26 deletions(-) create mode 100644 backend/migrations/versions/075d344ae2cc_mf_triggers.py diff --git a/backend/dataall/core/environment/services/environment_service.py b/backend/dataall/core/environment/services/environment_service.py index 34e1c0e36..89bddaca1 100644 --- a/backend/dataall/core/environment/services/environment_service.py +++ b/backend/dataall/core/environment/services/environment_service.py @@ -46,8 +46,6 @@ from dataall.core.permissions.services.tenant_permissions import MANAGE_ENVIRONMENTS from dataall.core.stacks.db.stack_repositories import StackRepository from dataall.core.vpc.db.vpc_repositories import VpcRepository -from dataall.modules.metadata_forms.db.enums import MetadataFormEntityTypes, MetadataFormVisibility -from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository log = logging.getLogger(__name__) @@ -886,12 +884,6 @@ def delete_environment(uri): KeyValueTagRepository.delete_key_value_tags(session, environment.environmentUri, 'environment') EnvironmentResourceManager.delete_env(session, environment) EnvironmentParameterRepository(session).delete_params(environment.environmentUri) - MetadataFormRepository.delete_attached_entity_metadata_forms( - session, environment.environmentUri, MetadataFormEntityTypes.Environments.value - ) - MetadataFormRepository.delete_all_home_metadata_forms( - session, environment.environmentUri, MetadataFormVisibility.Environment.value - ) for group in env_groups: session.delete(group) diff --git a/backend/dataall/core/organizations/services/organization_service.py b/backend/dataall/core/organizations/services/organization_service.py index 7e4191ae5..a016840e5 100644 --- a/backend/dataall/core/organizations/services/organization_service.py +++ b/backend/dataall/core/organizations/services/organization_service.py @@ -21,8 +21,6 @@ ORGANIZATION_INVITED_READONLY, ORGANIZATION_INVITED_DESCRIPTIONS, ) -from dataall.modules.metadata_forms.db.enums import MetadataFormEntityTypes, MetadataFormVisibility -from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository class OrganizationService: @@ -177,12 +175,6 @@ def archive_organization(uri): resource_uri=org.organizationUri, resource_type=models.Organization.__name__, ) - MetadataFormRepository.delete_attached_entity_metadata_forms( - session, org.organizationUri, MetadataFormEntityTypes.Organizations.value - ) - MetadataFormRepository.delete_all_home_metadata_forms( - session, org.organizationUri, MetadataFormVisibility.Organization.value - ) return True diff --git a/backend/dataall/modules/redshift_datasets/services/redshift_dataset_service.py b/backend/dataall/modules/redshift_datasets/services/redshift_dataset_service.py index 8eb497452..3e79a4325 100644 --- a/backend/dataall/modules/redshift_datasets/services/redshift_dataset_service.py +++ b/backend/dataall/modules/redshift_datasets/services/redshift_dataset_service.py @@ -6,8 +6,6 @@ from dataall.core.permissions.services.tenant_policy_service import TenantPolicyService from dataall.core.permissions.services.group_policy_service import GroupPolicyService from dataall.core.environment.services.environment_service import EnvironmentService -from dataall.modules.metadata_forms.db.enums import MetadataFormEntityTypes -from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository from dataall.modules.vote.db.vote_repositories import VoteRepository from dataall.modules.catalog.db.glossary_repositories import GlossaryRepository @@ -186,9 +184,7 @@ def delete_redshift_dataset(uri): RedshiftDatasetService._delete_dataset_term_links(session, uri) VoteRepository.delete_votes(session, dataset.datasetUri, VOTE_REDSHIFT_DATASET_NAME) session.delete(dataset) - MetadataFormRepository.delete_attached_entity_metadata_forms( - session, dataset.datasetUri, MetadataFormEntityTypes.Datasets.value - ) + session.commit() return True diff --git a/backend/dataall/modules/s3_datasets/services/dataset_service.py b/backend/dataall/modules/s3_datasets/services/dataset_service.py index 47ea8f242..2e68eb951 100644 --- a/backend/dataall/modules/s3_datasets/services/dataset_service.py +++ b/backend/dataall/modules/s3_datasets/services/dataset_service.py @@ -11,8 +11,6 @@ from dataall.core.stacks.services.stack_service import StackService from dataall.core.tasks.service_handlers import Worker from dataall.base.aws.sts import SessionHelper -from dataall.modules.metadata_forms.db.enums import MetadataFormEntityTypes -from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository from dataall.modules.s3_datasets.aws.kms_dataset_client import KmsClient from dataall.base.context import get_context from dataall.core.permissions.services.group_policy_service import GroupPolicyService @@ -447,9 +445,6 @@ def delete_dataset(uri: str, delete_from_aws: bool = False): if dataset.stewards: ResourcePolicyService.delete_resource_policy(session=session, resource_uri=uri, group=dataset.stewards) DatasetRepository.delete_dataset(session, dataset) - MetadataFormRepository.delete_attached_entity_metadata_forms( - session, dataset.datasetUri, MetadataFormEntityTypes.Datasets.value - ) if delete_from_aws: StackService.delete_stack( diff --git a/backend/migrations/versions/075d344ae2cc_mf_triggers.py b/backend/migrations/versions/075d344ae2cc_mf_triggers.py new file mode 100644 index 000000000..164d5327f --- /dev/null +++ b/backend/migrations/versions/075d344ae2cc_mf_triggers.py @@ -0,0 +1,123 @@ +"""mf_triggers + +Revision ID: 075d344ae2cc +Revises: 427db8f31999 +Create Date: 2024-09-13 13:12:16.951311 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '075d344ae2cc' +down_revision = '427db8f31999' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.execute(''' + CREATE OR REPLACE FUNCTION org_delete_trigger_function() + RETURNS TRIGGER AS $$ + BEGIN + -- Delete from attached_metadata_form + DELETE FROM attached_metadata_form + WHERE "entityUri" = OLD."organizationUri" + AND "entityType" = 'Organization'; + + -- Delete from metadata_form + DELETE FROM metadata_form + WHERE "homeEntity" = OLD."organizationUri" + AND visibility = 'Organization-Wide'; + + RETURN OLD; + END; + $$ LANGUAGE plpgsql; + + -- Create the trigger for organization table + CREATE TRIGGER org_delete_trigger + BEFORE DELETE ON organization + FOR EACH ROW + EXECUTE FUNCTION org_delete_trigger_function(); + ''') + + op.execute(''' + CREATE OR REPLACE FUNCTION env_delete_trigger_function() + RETURNS TRIGGER AS $$ + BEGIN + -- Delete from attached_metadata_form + DELETE FROM attached_metadata_form + WHERE "entityUri" = OLD."environmentUri" + AND "entityType" = 'Environment'; + + -- Delete from metadata_form + DELETE FROM metadata_form + WHERE "homeEntity" = OLD."environmentUri" + AND visibility = 'Environment-Wide'; + + RETURN OLD; + END; + $$ LANGUAGE plpgsql; + + -- Create the trigger for environment table + CREATE TRIGGER env_delete_trigger + BEFORE DELETE ON environment + FOR EACH ROW + EXECUTE FUNCTION env_delete_trigger_function(); + ''') + + op.execute(''' + CREATE OR REPLACE FUNCTION dataset_delete_trigger_function() + RETURNS TRIGGER AS $$ + BEGIN + -- Delete from attached_metadata_form + DELETE FROM attached_metadata_form + WHERE "entityUri" = OLD."datasetUri" + AND "entityType" = 'Dataset'; + + RETURN OLD; + END; + $$ LANGUAGE plpgsql; + + -- Create the trigger for dataset table + CREATE TRIGGER dataset_delete_trigger + BEFORE DELETE ON dataset + FOR EACH ROW + EXECUTE FUNCTION dataset_delete_trigger_function(); + ''') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.execute( + ''' + -- Drop the org_delete_trigger + DROP TRIGGER IF EXISTS org_delete_trigger ON organization; + + -- Drop the org_delete_trigger_function + DROP FUNCTION IF EXISTS org_delete_trigger_function; + ''' + ) + + op.execute( + ''' + -- Drop the env_delete_trigger + DROP TRIGGER IF EXISTS env_delete_trigger ON environment; + + -- Drop the env_delete_trigger_function + DROP FUNCTION IF EXISTS env_delete_trigger_function; + ''' + ) + + op.execute( + ''' + -- Drop the dataset_delete_trigger + DROP TRIGGER IF EXISTS dataset_delete_trigger ON dataset; + + -- Drop the dataset_delete_trigger_function + DROP FUNCTION IF EXISTS dataset_delete_trigger_function; + ''' + ) + # ### end Alembic commands ### From 9645643e5083bf0a14cd9a0133f948b6edec1127 Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Fri, 13 Sep 2024 14:04:31 +0100 Subject: [PATCH 12/26] PR changes --- .../db/metadata_form_repository.py | 22 ++++++---------- .../versions/075d344ae2cc_mf_triggers.py | 25 ++++++++++--------- 2 files changed, 20 insertions(+), 27 deletions(-) diff --git a/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py b/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py index 8a0badefb..8b284b2e0 100644 --- a/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py +++ b/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py @@ -249,25 +249,17 @@ def query_attached_metadata_forms(session, is_da_admin, groups, user_envs_uris, return query @staticmethod - def get_all_attached_metadata_forms_for_entity(session, entityUri, entityType): - return ( - session.query(AttachedMetadataForm) - .filter(and_(AttachedMetadataForm.entityType == entityType, AttachedMetadataForm.entityUri == entityUri)) - .all() + def query_all_attached_metadata_forms_for_entity(session, entityUri, entityType): + return session.query(AttachedMetadataForm).filter( + and_(AttachedMetadataForm.entityType == entityType, AttachedMetadataForm.entityUri == entityUri) ) @staticmethod def delete_attached_entity_metadata_forms(session, entityUri, entityType): - mfs = MetadataFormRepository.get_all_attached_metadata_forms_for_entity(session, entityUri, entityType) - for mf in mfs: - session.delete(mf) + MetadataFormRepository.query_all_attached_metadata_forms_for_entity(session, entityUri, entityType).delete() @staticmethod def delete_all_home_metadata_forms(session, homeEntityUri, visibility): - mfs = ( - session.query(MetadataForm) - .filter(and_(MetadataForm.homeEntity == homeEntityUri, MetadataForm.visibility == visibility)) - .all() - ) - for mf in mfs: - session.delete(mf) + session.query(MetadataForm).filter( + and_(MetadataForm.homeEntity == homeEntityUri, MetadataForm.visibility == visibility) + ).delete() diff --git a/backend/migrations/versions/075d344ae2cc_mf_triggers.py b/backend/migrations/versions/075d344ae2cc_mf_triggers.py index 164d5327f..2bcce4cfc 100644 --- a/backend/migrations/versions/075d344ae2cc_mf_triggers.py +++ b/backend/migrations/versions/075d344ae2cc_mf_triggers.py @@ -5,6 +5,7 @@ Create Date: 2024-09-13 13:12:16.951311 """ + from alembic import op import sqlalchemy as sa @@ -17,7 +18,7 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.execute(''' + op.execute(""" CREATE OR REPLACE FUNCTION org_delete_trigger_function() RETURNS TRIGGER AS $$ BEGIN @@ -40,9 +41,9 @@ def upgrade(): BEFORE DELETE ON organization FOR EACH ROW EXECUTE FUNCTION org_delete_trigger_function(); - ''') + """) - op.execute(''' + op.execute(""" CREATE OR REPLACE FUNCTION env_delete_trigger_function() RETURNS TRIGGER AS $$ BEGIN @@ -65,9 +66,9 @@ def upgrade(): BEFORE DELETE ON environment FOR EACH ROW EXECUTE FUNCTION env_delete_trigger_function(); - ''') + """) - op.execute(''' + op.execute(""" CREATE OR REPLACE FUNCTION dataset_delete_trigger_function() RETURNS TRIGGER AS $$ BEGIN @@ -85,39 +86,39 @@ def upgrade(): BEFORE DELETE ON dataset FOR EACH ROW EXECUTE FUNCTION dataset_delete_trigger_function(); - ''') + """) # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### op.execute( - ''' + """ -- Drop the org_delete_trigger DROP TRIGGER IF EXISTS org_delete_trigger ON organization; -- Drop the org_delete_trigger_function DROP FUNCTION IF EXISTS org_delete_trigger_function; - ''' + """ ) op.execute( - ''' + """ -- Drop the env_delete_trigger DROP TRIGGER IF EXISTS env_delete_trigger ON environment; -- Drop the env_delete_trigger_function DROP FUNCTION IF EXISTS env_delete_trigger_function; - ''' + """ ) op.execute( - ''' + """ -- Drop the dataset_delete_trigger DROP TRIGGER IF EXISTS dataset_delete_trigger ON dataset; -- Drop the dataset_delete_trigger_function DROP FUNCTION IF EXISTS dataset_delete_trigger_function; - ''' + """ ) # ### end Alembic commands ### From f63d539a44716527fbaf533865a970402475e2cc Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Fri, 13 Sep 2024 14:49:12 +0100 Subject: [PATCH 13/26] revert get_resource_policy_permissions change --- .../services/environment_service.py | 1 + .../services/organization_service.py | 6 ++++- .../services/resource_policy_service.py | 22 +++++++++---------- ...1ac7a85a2_drop_remove_group_permissions.py | 2 +- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/backend/dataall/core/environment/services/environment_service.py b/backend/dataall/core/environment/services/environment_service.py index 89bddaca1..f49302ae4 100644 --- a/backend/dataall/core/environment/services/environment_service.py +++ b/backend/dataall/core/environment/services/environment_service.py @@ -525,6 +525,7 @@ def list_group_permissions_internal(session, uri, group_uri): environment = EnvironmentService.get_environment_by_uri(session, uri) return ResourcePolicyService.get_resource_policy_permissions( + session=session, group_uri=group_uri, resource_uri=environment.environmentUri, ) diff --git a/backend/dataall/core/organizations/services/organization_service.py b/backend/dataall/core/organizations/services/organization_service.py index a016840e5..739717e81 100644 --- a/backend/dataall/core/organizations/services/organization_service.py +++ b/backend/dataall/core/organizations/services/organization_service.py @@ -309,7 +309,11 @@ def resolve_organization_by_env(uri): @staticmethod @ResourcePolicyService.has_resource_permission(GET_ORGANIZATION) def list_group_organization_permissions(uri, groupUri): - return ResourcePolicyService.get_resource_policy_permissions(group_uri=groupUri, resource_uri=uri) + context = get_context() + with context.db_engine.scoped_session() as session: + return ResourcePolicyService.get_resource_policy_permissions( + session=session, group_uri=groupUri, resource_uri=uri + ) @staticmethod def list_invited_organization_permissions_with_descriptions(): diff --git a/backend/dataall/core/permissions/services/resource_policy_service.py b/backend/dataall/core/permissions/services/resource_policy_service.py index a44954648..c39c354d4 100644 --- a/backend/dataall/core/permissions/services/resource_policy_service.py +++ b/backend/dataall/core/permissions/services/resource_policy_service.py @@ -212,21 +212,21 @@ def associate_permission_to_resource_policy(session, policy, permission): session.commit() @staticmethod - def get_resource_policy_permissions(group_uri, resource_uri) -> List[ResourcePolicyPermission]: + def get_resource_policy_permissions(session, group_uri, resource_uri) -> List[ResourcePolicyPermission]: if not group_uri: raise exceptions.RequiredParameter(param_name='group_uri') if not resource_uri: raise exceptions.RequiredParameter(param_name='resource_uri') - with get_context().db_engine.scoped_session() as session: - policy = ResourcePolicyRepository.find_resource_policy( - session=session, - group_uri=group_uri, - resource_uri=resource_uri, - ) - permissions = [] - for p in policy.permissions: - permissions.append(p.permission) - return permissions + + policy = ResourcePolicyRepository.find_resource_policy( + session=session, + group_uri=group_uri, + resource_uri=resource_uri, + ) + permissions = [] + for p in policy.permissions: + permissions.append(p.permission) + return permissions @staticmethod def has_resource_permission( diff --git a/backend/migrations/versions/a991ac7a85a2_drop_remove_group_permissions.py b/backend/migrations/versions/a991ac7a85a2_drop_remove_group_permissions.py index a6dcea574..64ded7dd5 100644 --- a/backend/migrations/versions/a991ac7a85a2_drop_remove_group_permissions.py +++ b/backend/migrations/versions/a991ac7a85a2_drop_remove_group_permissions.py @@ -45,7 +45,7 @@ def upgrade(): .all() ) for group in suspicious_permissions_principals: - permissions = ResourcePolicyService.get_resource_policy_permissions(group, env.environmentUri) + permissions = ResourcePolicyService.get_resource_policy_permissions(session, group, env.environmentUri) permissions = [permission.name for permission in permissions if permission.name != REMOVE_ENVIRONMENT_GROUP] ResourcePolicyService.update_resource_policy( session, From cb91ae89cdd71c7f3ce9c7bc79d8c7d53b5b3486 Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Mon, 16 Sep 2024 16:09:42 +0100 Subject: [PATCH 14/26] share improvements and bugfixes --- .../share_managers/s3_access_point_share_manager.py | 7 +++++-- .../share_processors/glue_table_share_processor.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py index 7bd275d6a..e5d70e5ef 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py @@ -40,7 +40,8 @@ logger = logging.getLogger(__name__) ACCESS_POINT_CREATION_TIME = 30 -ACCESS_POINT_CREATION_RETRIES = 5 +ACCESS_POINT_CREATION_RETRIES = 10 +ACCESS_POINT_BACKOFF_COEFFICIENT = 1.1 # every time increase retry delay by 10% class S3AccessPointShareManager: @@ -447,12 +448,14 @@ def manage_access_point_and_policy(self): access_point_arn = s3_client.create_bucket_access_point(self.bucket_name, self.access_point_name) # Access point creation is slow retries = 1 + sleep_coeff = 1 while ( not s3_client.get_bucket_access_point_arn(self.access_point_name) and retries < ACCESS_POINT_CREATION_RETRIES ): logger.info('Waiting 30s for access point creation to complete..') - time.sleep(ACCESS_POINT_CREATION_TIME) + time.sleep(ACCESS_POINT_CREATION_TIME * sleep_coeff) + sleep_coeff = sleep_coeff * ACCESS_POINT_BACKOFF_COEFFICIENT retries += 1 existing_policy = s3_client.get_access_point_policy(self.access_point_name) # requester will use this role to access resources diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_processors/glue_table_share_processor.py b/backend/dataall/modules/s3_datasets_shares/services/share_processors/glue_table_share_processor.py index cd3c9e581..284875416 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_processors/glue_table_share_processor.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_processors/glue_table_share_processor.py @@ -153,7 +153,7 @@ def process_approved_shares(self) -> bool: manager.grant_principals_permissions_to_source_table(table, share_item, share_item_filter) if manager.cross_account: retries = 0 - retry_share_table = False + retry_share_table = True while retry_share_table and retries < 1: ( retry_share_table, From d04d0053676a89604c8fed84928cdea1e07ea9fe Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Thu, 19 Sep 2024 14:40:08 +0100 Subject: [PATCH 15/26] or replace --- backend/migrations/versions/075d344ae2cc_mf_triggers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/migrations/versions/075d344ae2cc_mf_triggers.py b/backend/migrations/versions/075d344ae2cc_mf_triggers.py index 2bcce4cfc..fc31efb43 100644 --- a/backend/migrations/versions/075d344ae2cc_mf_triggers.py +++ b/backend/migrations/versions/075d344ae2cc_mf_triggers.py @@ -37,7 +37,7 @@ def upgrade(): $$ LANGUAGE plpgsql; -- Create the trigger for organization table - CREATE TRIGGER org_delete_trigger + CREATE OR REPLACE TRIGGER org_delete_trigger BEFORE DELETE ON organization FOR EACH ROW EXECUTE FUNCTION org_delete_trigger_function(); @@ -62,7 +62,7 @@ def upgrade(): $$ LANGUAGE plpgsql; -- Create the trigger for environment table - CREATE TRIGGER env_delete_trigger + CREATE OR REPLACE TRIGGER env_delete_trigger BEFORE DELETE ON environment FOR EACH ROW EXECUTE FUNCTION env_delete_trigger_function(); @@ -82,7 +82,7 @@ def upgrade(): $$ LANGUAGE plpgsql; -- Create the trigger for dataset table - CREATE TRIGGER dataset_delete_trigger + CREATE OR REPLACE TRIGGER dataset_delete_trigger BEFORE DELETE ON dataset FOR EACH ROW EXECUTE FUNCTION dataset_delete_trigger_function(); From ea578dfee8f656fa1c1052bb436e36dd5c1b777d Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Fri, 20 Sep 2024 19:29:09 +0100 Subject: [PATCH 16/26] another trigger --- .../versions/075d344ae2cc_mf_triggers.py | 35 +++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/backend/migrations/versions/075d344ae2cc_mf_triggers.py b/backend/migrations/versions/075d344ae2cc_mf_triggers.py index fc31efb43..35b1a9b5e 100644 --- a/backend/migrations/versions/075d344ae2cc_mf_triggers.py +++ b/backend/migrations/versions/075d344ae2cc_mf_triggers.py @@ -37,7 +37,7 @@ def upgrade(): $$ LANGUAGE plpgsql; -- Create the trigger for organization table - CREATE OR REPLACE TRIGGER org_delete_trigger + CREATE TRIGGER org_delete_trigger BEFORE DELETE ON organization FOR EACH ROW EXECUTE FUNCTION org_delete_trigger_function(); @@ -62,7 +62,7 @@ def upgrade(): $$ LANGUAGE plpgsql; -- Create the trigger for environment table - CREATE OR REPLACE TRIGGER env_delete_trigger + CREATE TRIGGER env_delete_trigger BEFORE DELETE ON environment FOR EACH ROW EXECUTE FUNCTION env_delete_trigger_function(); @@ -82,11 +82,30 @@ def upgrade(): $$ LANGUAGE plpgsql; -- Create the trigger for dataset table - CREATE OR REPLACE TRIGGER dataset_delete_trigger + CREATE TRIGGER dataset_delete_trigger BEFORE DELETE ON dataset FOR EACH ROW EXECUTE FUNCTION dataset_delete_trigger_function(); """) + + op.execute(""" + CREATE OR REPLACE FUNCTION metadata_form_delete_trigger_function() + RETURNS TRIGGER AS $$ + BEGIN + -- Delete from resource_permission_policy + DELETE FROM resource_policy_permission + WHERE "sid" in (SELECT sid from resource_policy where "resourceUri"=OLD.uri and "resourceType"='MetadataForm'); + DELETE FROM resource_policy where "resourceUri"=OLD.uri; + RETURN OLD; + END; + $$ LANGUAGE plpgsql; + + -- Create the trigger for dataset table + CREATE TRIGGER metadata_form_delete_trigger + BEFORE DELETE ON metadata_form + FOR EACH ROW + EXECUTE FUNCTION metadata_form_delete_trigger_function(); + """) # ### end Alembic commands ### @@ -121,4 +140,14 @@ def downgrade(): DROP FUNCTION IF EXISTS dataset_delete_trigger_function; """ ) + + op.execute( + """ + -- Drop the dataset_delete_trigger + DROP TRIGGER IF EXISTS metadata_form_delete_trigger ON metadata_form; + + -- Drop the dataset_delete_trigger_function + DROP FUNCTION IF EXISTS metadata_form_delete_trigger_function; + """ + ) # ### end Alembic commands ### From 82404dec1fd699c3b09a4175d3a213628bb0dfd0 Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Fri, 20 Sep 2024 19:36:31 +0100 Subject: [PATCH 17/26] backfill MF permission downgrade --- ...f31999_backfill_MF_resource_permissions.py | 44 ++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py b/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py index 4ecdfe4b2..a92e16ef6 100644 --- a/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py +++ b/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py @@ -78,4 +78,46 @@ def upgrade(): def downgrade(): - print('no downgrade supported') + bind = op.get_bind() + session = orm.Session(bind=bind) + all_environments = session.query(Environment).all() + for env in all_environments: + policies = ResourcePolicyService.find_resource_policies( + session=session, + group=env.SamlGroupName, + resource_uri=env.environmentUri, + resource_type=Environment.__name__, + permissions=[ATTACH_METADATA_FORM, CREATE_METADATA_FORM], + ) + for policy in policies: + for permission in policy.permissions: + session.delete(permission) + session.commit() + + all_organizations = session.query(Organization).all() + for org in all_organizations: + policies = ResourcePolicyService.find_resource_policies( + session=session, + group=org.SamlGroupName, + resource_uri=org.organizationUri, + permissions=[ATTACH_METADATA_FORM, CREATE_METADATA_FORM], + resource_type=Organization.__name__, + ) + for policy in policies: + for permission in policy.permissions: + session.delete(permission) + session.commit() + + datasets = session.query(DatasetBase).all() + for dataset in datasets: + policies = ResourcePolicyService.find_resource_policies( + session=session, + group=dataset.SamlGroupName, + resource_uri=dataset.datasetUri, + permissions=[ATTACH_METADATA_FORM], + resource_type=DatasetBase.__name__, + ) + for policy in policies: + for permission in policy.permissions: + session.delete(permission) + session.commit() From 0642d6993011c63b0feb6421dbb01f2d7ff74216 Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Fri, 20 Sep 2024 19:37:50 +0100 Subject: [PATCH 18/26] remove unused --- .../metadata_forms/db/metadata_form_repository.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py b/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py index 8b284b2e0..41bd50772 100644 --- a/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py +++ b/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py @@ -253,13 +253,3 @@ def query_all_attached_metadata_forms_for_entity(session, entityUri, entityType) return session.query(AttachedMetadataForm).filter( and_(AttachedMetadataForm.entityType == entityType, AttachedMetadataForm.entityUri == entityUri) ) - - @staticmethod - def delete_attached_entity_metadata_forms(session, entityUri, entityType): - MetadataFormRepository.query_all_attached_metadata_forms_for_entity(session, entityUri, entityType).delete() - - @staticmethod - def delete_all_home_metadata_forms(session, homeEntityUri, visibility): - session.query(MetadataForm).filter( - and_(MetadataForm.homeEntity == homeEntityUri, MetadataForm.visibility == visibility) - ).delete() From 0fbccd95f576e0c718e60e2305b64f3566f859a3 Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Fri, 20 Sep 2024 19:38:58 +0100 Subject: [PATCH 19/26] SamlAdminGroupName --- .../versions/427db8f31999_backfill_MF_resource_permissions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py b/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py index a92e16ef6..4eeaaf414 100644 --- a/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py +++ b/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py @@ -70,7 +70,7 @@ def upgrade(): for dataset in datasets: ResourcePolicyService.attach_resource_policy( session=session, - group=dataset.SamlGroupName, + group=dataset.SamlAdminGroupName, resource_uri=dataset.datasetUri, permissions=[ATTACH_METADATA_FORM], resource_type=DatasetBase.__name__, @@ -112,7 +112,7 @@ def downgrade(): for dataset in datasets: policies = ResourcePolicyService.find_resource_policies( session=session, - group=dataset.SamlGroupName, + group=dataset.SamlAdminGroupName, resource_uri=dataset.datasetUri, permissions=[ATTACH_METADATA_FORM], resource_type=DatasetBase.__name__, From 9b1153f0818934a3cd6ffc643e7c24b38527dc6c Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Mon, 23 Sep 2024 11:12:36 +0100 Subject: [PATCH 20/26] check ATTACH_METADATA_FORM via decorator --- .../attached_metadata_form_service.py | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/backend/dataall/modules/metadata_forms/services/attached_metadata_form_service.py b/backend/dataall/modules/metadata_forms/services/attached_metadata_form_service.py index 4af2e8f84..146f7d51d 100644 --- a/backend/dataall/modules/metadata_forms/services/attached_metadata_form_service.py +++ b/backend/dataall/modules/metadata_forms/services/attached_metadata_form_service.py @@ -31,7 +31,15 @@ def validate_enrich_fields_params(mf_fields, data): class AttachedMetadataFormService: + + @staticmethod + def _get_entity_uri(data): + return data.get('entityUri') + @staticmethod + @ResourcePolicyService.has_resource_permission( + ATTACH_METADATA_FORM, parent_resource=_get_entity_uri, param_name='data' + ) def create_attached_metadata_form(uri, data): AttachedMetadataFormValidationService.validate_filled_form_params(uri, data) context = get_context() @@ -41,13 +49,6 @@ def create_attached_metadata_form(uri, data): raise exceptions.ObjectNotFound('MetadataForm', uri) mf_fields = MetadataFormRepository.get_metadata_form_fields(session, uri) AttachedMetadataFormValidationService.validate_enrich_fields_params(mf_fields, data) - ResourcePolicyService.check_user_resource_permission( - session=session, - username=context.username, - groups=context.groups, - resource_uri=data.get('entityUri'), - permission_name=ATTACH_METADATA_FORM, - ) amf = MetadataFormRepository.create_attached_metadata_form(session, uri, data) for f in data.get('fields'): MetadataFormRepository.create_attached_metadata_form_field( @@ -80,15 +81,11 @@ def list_attached_forms(filter=None): ).to_dict() @staticmethod + @ResourcePolicyService.has_resource_permission( + ATTACH_METADATA_FORM, parent_resource=_get_entity_uri, param_name='data' + ) def delete_attached_metadata_form(uri): mf = AttachedMetadataFormService.get_attached_metadata_form(uri) context = get_context() with context.db_engine.scoped_session() as session: - ResourcePolicyService.check_user_resource_permission( - session=session, - username=context.username, - groups=context.groups, - resource_uri=mf.entityUri, - permission_name=ATTACH_METADATA_FORM, # attach and delete are the same for now - ) return session.delete(mf) From 1c6ef9ce71c877ef5c4248f300e15bce80d79ef3 Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Mon, 23 Sep 2024 11:16:19 +0100 Subject: [PATCH 21/26] no more arrays set to None --- .../modules/metadata_forms/db/metadata_form_repository.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py b/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py index 41bd50772..4423cce27 100644 --- a/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py +++ b/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py @@ -70,6 +70,9 @@ def query_user_metadata_forms(session, is_da_admin, groups, env_uris, org_uris, :param filter: """ + env_uris = env_uris or [] + org_uris = org_uris or [] + query = session.query(MetadataForm) if not is_da_admin: @@ -140,6 +143,8 @@ def query_entity_metadata_forms( entity_orgs_uris = entity_orgs_uris or [] entity_envs_uris = entity_envs_uris or [] + user_org_uris = user_org_uris or [] + user_env_uris = user_env_uris or [] orgs = list(set(user_org_uris).intersection(set(entity_orgs_uris))) envs = list(set(user_env_uris).intersection(set(entity_envs_uris))) From 7a04e7d3572af0d5e4a493b63e0f094cf65754de Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Mon, 23 Sep 2024 11:16:46 +0100 Subject: [PATCH 22/26] ruff --- .../metadata_forms/services/attached_metadata_form_service.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/dataall/modules/metadata_forms/services/attached_metadata_form_service.py b/backend/dataall/modules/metadata_forms/services/attached_metadata_form_service.py index 146f7d51d..005e9a45b 100644 --- a/backend/dataall/modules/metadata_forms/services/attached_metadata_form_service.py +++ b/backend/dataall/modules/metadata_forms/services/attached_metadata_form_service.py @@ -31,7 +31,6 @@ def validate_enrich_fields_params(mf_fields, data): class AttachedMetadataFormService: - @staticmethod def _get_entity_uri(data): return data.get('entityUri') From a2f4cee63121923a3afba5fd5a0c2cf9f4cecabe Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Tue, 24 Sep 2024 14:06:34 +0100 Subject: [PATCH 23/26] problem queries --- .../db/metadata_form_repository.py | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py b/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py index 4423cce27..8dcf55f1b 100644 --- a/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py +++ b/backend/dataall/modules/metadata_forms/db/metadata_form_repository.py @@ -143,19 +143,23 @@ def query_entity_metadata_forms( entity_orgs_uris = entity_orgs_uris or [] entity_envs_uris = entity_envs_uris or [] - user_org_uris = user_org_uris or [] - user_env_uris = user_env_uris or [] - orgs = list(set(user_org_uris).intersection(set(entity_orgs_uris))) - envs = list(set(user_env_uris).intersection(set(entity_envs_uris))) - - query = MetadataFormRepository.query_user_metadata_forms(session, is_da_admin, groups, envs, orgs, filter) - - if not orgs: - query = query.filter(MetadataForm.visibility != MetadataFormVisibility.Organization.value) + query = MetadataFormRepository.query_user_metadata_forms( + session, is_da_admin, groups, user_env_uris, user_org_uris, filter + ) - if not envs: - query = query.filter(MetadataForm.visibility != MetadataFormVisibility.Environment.value) + query = query.filter( + and_( + or_( + MetadataForm.visibility != MetadataFormVisibility.Organization.value, + MetadataForm.homeEntity.in_(entity_orgs_uris), + ), + or_( + MetadataForm.visibility != MetadataFormVisibility.Environment.value, + MetadataForm.homeEntity.in_(entity_envs_uris), + ), + ) + ) query = MetadataFormRepository.exclude_attached(session, query, filter) return query.order_by(MetadataForm.name) From 5719483fed23572f13d11c8a9b8a82513fccd978 Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Tue, 24 Sep 2024 14:13:23 +0100 Subject: [PATCH 24/26] fix downgrade --- ...7db8f31999_backfill_MF_resource_permissions.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py b/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py index 4eeaaf414..bbc4699c2 100644 --- a/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py +++ b/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py @@ -91,8 +91,9 @@ def downgrade(): ) for policy in policies: for permission in policy.permissions: - session.delete(permission) - session.commit() + if permission.name in [ATTACH_METADATA_FORM, CREATE_METADATA_FORM]: + session.delete(permission) + session.commit() all_organizations = session.query(Organization).all() for org in all_organizations: @@ -105,8 +106,9 @@ def downgrade(): ) for policy in policies: for permission in policy.permissions: - session.delete(permission) - session.commit() + if permission.name in [ATTACH_METADATA_FORM, CREATE_METADATA_FORM]: + session.delete(permission) + session.commit() datasets = session.query(DatasetBase).all() for dataset in datasets: @@ -119,5 +121,6 @@ def downgrade(): ) for policy in policies: for permission in policy.permissions: - session.delete(permission) - session.commit() + if permission.name in [ATTACH_METADATA_FORM]: + session.delete(permission) + session.commit() From 7b6023d173059dd9829f2d651c1782d93cc031c4 Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Tue, 24 Sep 2024 14:26:18 +0100 Subject: [PATCH 25/26] fix downgrade for redshift --- .../versions/852cdf6cf1e0_add_redshift_datasets.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/migrations/versions/852cdf6cf1e0_add_redshift_datasets.py b/backend/migrations/versions/852cdf6cf1e0_add_redshift_datasets.py index 3e7a38c35..37686a5ee 100644 --- a/backend/migrations/versions/852cdf6cf1e0_add_redshift_datasets.py +++ b/backend/migrations/versions/852cdf6cf1e0_add_redshift_datasets.py @@ -147,5 +147,6 @@ def downgrade(): ) for policy in policies: for permission in policy.permissions: - session.delete(permission) - session.commit() + if permission.name in ENVIRONMENT_REDSHIFT_ALL: + session.delete(permission) + session.commit() From 0e23814ed22d1fd76475412fa03d8331c8d4214e Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Thu, 26 Sep 2024 13:03:45 +0100 Subject: [PATCH 26/26] permision => resource_pol_permission.permission --- ...f31999_backfill_MF_resource_permissions.py | 19 ++++++++++--------- .../852cdf6cf1e0_add_redshift_datasets.py | 6 +++--- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py b/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py index bbc4699c2..5209963e8 100644 --- a/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py +++ b/backend/migrations/versions/427db8f31999_backfill_MF_resource_permissions.py @@ -90,9 +90,9 @@ def downgrade(): permissions=[ATTACH_METADATA_FORM, CREATE_METADATA_FORM], ) for policy in policies: - for permission in policy.permissions: - if permission.name in [ATTACH_METADATA_FORM, CREATE_METADATA_FORM]: - session.delete(permission) + for resource_pol_permission in policy.permissions: + if resource_pol_permission.permission.name in [ATTACH_METADATA_FORM, CREATE_METADATA_FORM]: + session.delete(resource_pol_permission) session.commit() all_organizations = session.query(Organization).all() @@ -105,9 +105,9 @@ def downgrade(): resource_type=Organization.__name__, ) for policy in policies: - for permission in policy.permissions: - if permission.name in [ATTACH_METADATA_FORM, CREATE_METADATA_FORM]: - session.delete(permission) + for resource_pol_permission in policy.permissions: + if resource_pol_permission.permission.name in [ATTACH_METADATA_FORM, CREATE_METADATA_FORM]: + session.delete(resource_pol_permission) session.commit() datasets = session.query(DatasetBase).all() @@ -119,8 +119,9 @@ def downgrade(): permissions=[ATTACH_METADATA_FORM], resource_type=DatasetBase.__name__, ) + for policy in policies: - for permission in policy.permissions: - if permission.name in [ATTACH_METADATA_FORM]: - session.delete(permission) + for resource_pol_permission in policy.permissions: + if resource_pol_permission.permission.name in [ATTACH_METADATA_FORM, CREATE_METADATA_FORM]: + session.delete(resource_pol_permission) session.commit() diff --git a/backend/migrations/versions/852cdf6cf1e0_add_redshift_datasets.py b/backend/migrations/versions/852cdf6cf1e0_add_redshift_datasets.py index 37686a5ee..422d1ffbb 100644 --- a/backend/migrations/versions/852cdf6cf1e0_add_redshift_datasets.py +++ b/backend/migrations/versions/852cdf6cf1e0_add_redshift_datasets.py @@ -146,7 +146,7 @@ def downgrade(): permissions=ENVIRONMENT_REDSHIFT_ALL, ) for policy in policies: - for permission in policy.permissions: - if permission.name in ENVIRONMENT_REDSHIFT_ALL: - session.delete(permission) + for resource_pol_permission in policy.permissions: + if resource_pol_permission.permission.name in ENVIRONMENT_REDSHIFT_ALL: + session.delete(resource_pol_permission) session.commit()