Metadata form versioning - 1 (#1637)

### Feature or Bugfix
<!-- please choose -->
- Feature


### Detail
- new db model
- migration:
  -- create default version one
  -- backfill existing MF fields and attached MFs
- when MF is created, service automatically create version 1
- fields and attached metadata forms automatically use the latest
version (for now version === 1)

### Relates
- #1621

### Security
Please answer the questions below briefly where applicable, or write
`N/A`. Based on
[OWASP 10](https://owasp.org/Top10/en/).

- Does this PR introduce or modify any input fields or queries - this
includes
fetching data from storage outside the application (e.g. a database, an
S3 bucket)?
  - Is the input sanitized?
- What precautions are you taking before deserializing the data you
consume?
  - Is injection prevented by parametrizing queries?
  - Have you ensured no `eval` or similar functions are used?
- Does this PR introduce any functionality or component that requires
authorization?
- How have you ensured it respects the existing AuthN/AuthZ mechanisms?
  - Are you logging failed auth attempts?
- Are you using or adding any cryptographic features?
  - Do you use a standard proven implementations?
  - Are the used keys controlled by the customer? Where are they stored?
- Are you introducing any new policies/roles/users?
  - Have you used the least-privilege principle? How?


By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license.

---------

Co-authored-by: Sofia Sazonova <sazonova@amazon.co.uk>

backend/dataall/modules/metadata_forms/db/metadata_form_models.py

@@ -17,24 +17,42 @@ class MetadataForm(Base):
     homeEntity = Column(String, nullable=True)
 
 
+class MetadataFormVersion(Base):
+    __tablename__ = 'metadata_form_version'
+    metadataFormUri = Column(String, ForeignKey('metadata_form.uri'), nullable=False)
+    version = Column(Integer, nullable=False)
+
+    __table_args__ = (
+        PrimaryKeyConstraint('metadataFormUri', 'version'),
+        ForeignKeyConstraint(
+            ('metadataFormUri',), ('metadata_form.uri',), name='f_key_version_metadata', ondelete='CASCADE'
+        ),
+    )
+
+
 class MetadataFormEnforcementRule(Base):
     __tablename__ = 'metadata_form_enforcement_rule'
     uri = Column(String, primary_key=True, default=utils.uuid('rule'))
     metadataFormUri = Column(String, ForeignKey('metadata_form.uri'), nullable=False)
+    version = Column(Integer, nullable=False)
     level = Column(String, nullable=False)  # enum MetadataFormEnforcementScope
     entityTypes = Column(ARRAY(String), nullable=False)  # enum MetadataFormEntityTypes
     severity = Column(String, nullable=False)  # enum MetadataFormEnforcementSeverity
 
     __table_args__ = (
         ForeignKeyConstraint(
-            ('metadataFormUri',), ('metadata_form.uri',), name='f_key_enforcement_metadata', ondelete='CASCADE'
+            ['metadataFormUri', 'version'],
+            ['metadata_form_version.metadataFormUri', 'metadata_form_version.version'],
+            name='f_key_enforcement_version_metadata',
+            ondelete='CASCADE',
         ),
     )
 
 
 class MetadataFormField(Base):
     __tablename__ = 'metadata_form_field'
     metadataFormUri = Column(String)
+    version = Column(Integer, nullable=False)
     uri = Column(String, primary_key=True, default=utils.uuid('field'))
     displayNumber = Column(Integer, nullable=False)
     description = Column(String, nullable=True)
@@ -46,21 +64,28 @@ class MetadataFormField(Base):
 
     __table_args__ = (
         ForeignKeyConstraint(
-            ('metadataFormUri',), ('metadata_form.uri',), name='fk_mf_filed_form_uri', ondelete='CASCADE'
+            ['metadataFormUri', 'version'],
+            ['metadata_form_version.metadataFormUri', 'metadata_form_version.version'],
+            name='fk_version',
+            ondelete='CASCADE',
         ),
     )
 
 
 class AttachedMetadataForm(Base):
     __tablename__ = 'attached_metadata_form'
     metadataFormUri = Column(String, nullable=False)
+    version = Column(Integer, nullable=False)
     uri = Column(String, primary_key=True, default=utils.uuid('attached_form'))
     entityUri = Column(String, nullable=False)
     entityType = Column(String, nullable=False)
 
     __table_args__ = (
         ForeignKeyConstraint(
-            ('metadataFormUri',), ('metadata_form.uri',), name='fk_attached_mf_uri', ondelete='CASCADE'
+            ['metadataFormUri', 'version'],
+            ['metadata_form_version.metadataFormUri', 'metadata_form_version.version'],
+            name='fk_attached_mf_version_uri',
+            ondelete='CASCADE',
         ),
     )
 

backend/dataall/modules/metadata_forms/db/metadata_form_repository.py

@@ -1,5 +1,6 @@
 from sqlalchemy import or_, and_
 from sqlalchemy.orm import with_polymorphic
+from sqlalchemy import func
 
 from dataall.modules.metadata_forms.db.enums import MetadataFormVisibility, MetadataFormFieldType
 from dataall.modules.metadata_forms.db.metadata_form_models import (
@@ -11,6 +12,7 @@
     BooleanAttachedMetadataFormField,
     IntegerAttachedMetadataFormField,
     GlossaryTermAttachedMetadataFormField,
+    MetadataFormVersion,
 )
 
 import json
@@ -40,10 +42,39 @@ def create_metadata_form(session, data=None):
         session.commit()
         return mf
 
+    @staticmethod
+    def create_metadata_form_version(session, metadataFormUri, version_num):
+        version = MetadataFormVersion(metadataFormUri=metadataFormUri, version=version_num)
+        session.add(version)
+        session.commit()
+        return version
+
+    @staticmethod
+    def create_metadata_form_version_next(session, metadataFormUri):
+        version_num = MetadataFormRepository.get_metadata_form_version_number_latest(session, metadataFormUri)
+        version = MetadataFormVersion(metadataFormUri=metadataFormUri, version=version_num + 1)
+        session.add(version)
+        session.commit()
+        return version
+
+    @staticmethod
+    def get_metadata_form_version_number_latest(session, metadataFormUri):
+        return (
+            session.query(func.max(MetadataFormVersion.version))
+            .filter(MetadataFormVersion.metadataFormUri == metadataFormUri)
+            .scalar()
+        )
+
+    @staticmethod
+    def get_metadata_form_version_latest(session, metadataFormUri):
+        version_num = MetadataFormRepository.get_metadata_form_version_number_latest(session, metadataFormUri)
+        return session.query(MetadataFormVersion).get((metadataFormUri, version_num))
+
     @staticmethod
     def create_attached_metadata_form(session, uri, data=None):
+        version_num = MetadataFormRepository.get_metadata_form_version_number_latest(session, uri)
         amf: AttachedMetadataForm = AttachedMetadataForm(
-            metadataFormUri=uri, entityUri=data.get('entityUri'), entityType=data.get('entityType')
+            metadataFormUri=uri, version=version_num, entityUri=data.get('entityUri'), entityType=data.get('entityType')
         )
         session.add(amf)
         session.commit()
@@ -175,8 +206,10 @@ def get_metadata_form_fields(session, form_uri):
 
     @staticmethod
     def create_metadata_form_field(session, uri, data):
+        version_num = MetadataFormRepository.get_metadata_form_version_number_latest(session, uri)
         field: MetadataFormField = MetadataFormField(
             metadataFormUri=uri,
+            version=version_num,
             name=data.get('name'),
             description=data.get('description'),
             type=data.get('type'),

backend/dataall/modules/metadata_forms/services/metadata_form_service.py

@@ -115,7 +115,12 @@ def create_metadata_form(data):
                 )
 
             form = MetadataFormRepository.create_metadata_form(session, data)
-            return form
+            try:
+                MetadataFormRepository.create_metadata_form_version(session, form.uri, 1)
+                return form
+            except Exception as e:
+                session.delete(form)
+                raise e
 
     # toDo: add permission check
     @staticmethod

backend/migrations/versions/5a798acc6282_create_version_mf.py

@@ -0,0 +1,125 @@
+"""create_version_mf
+
+Revision ID: 5a798acc6282
+Revises: 075d344ae2cc
+Create Date: 2024-10-10 17:25:09.687099
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy import orm
+
+from dataall.modules.metadata_forms.db.metadata_form_models import (
+    MetadataForm,
+    MetadataFormVersion,
+    AttachedMetadataForm,
+    MetadataFormField,
+)
+
+# revision identifiers, used by Alembic.
+revision = '5a798acc6282'
+down_revision = '075d344ae2cc'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    bind = op.get_bind()
+    session = orm.Session(bind=bind)
+
+    op.create_table(
+        'metadata_form_version',
+        sa.Column('metadataFormUri', sa.String(), nullable=False),
+        sa.Column('version', sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ['metadataFormUri'], ['metadata_form.uri'], name='f_key_version_metadata', ondelete='CASCADE'
+        ),
+        sa.PrimaryKeyConstraint('metadataFormUri', 'version'),
+    )
+    for mf in session.query(MetadataForm).all():
+        version = MetadataFormVersion(metadataFormUri=mf.uri, version=1)
+        session.add(version)
+        session.commit()
+
+    op.add_column('attached_metadata_form', sa.Column('version', sa.Integer()))
+    for amf in session.query(AttachedMetadataForm).all():
+        amf.version = 1
+        session.commit()
+    op.alter_column('attached_metadata_form', 'version', nullable=False)
+    op.drop_constraint('fk_attached_mf_uri', 'attached_metadata_form', type_='foreignkey')
+    op.create_foreign_key(
+        'fk_attached_mf_version_uri',
+        'attached_metadata_form',
+        'metadata_form_version',
+        ['metadataFormUri', 'version'],
+        ['metadataFormUri', 'version'],
+        ondelete='CASCADE',
+    )
+
+    op.add_column('metadata_form_enforcement_rule', sa.Column('version', sa.Integer(), nullable=False))
+    op.create_foreign_key(
+        'f_key_enforcement_version_metadata',
+        'metadata_form_enforcement_rule',
+        'metadata_form_version',
+        ['metadataFormUri', 'version'],
+        ['metadataFormUri', 'version'],
+        ondelete='CASCADE',
+    )
+    op.drop_constraint('f_key_enforcement_metadata', 'metadata_form_enforcement_rule', type_='foreignkey')
+
+    op.add_column('metadata_form_field', sa.Column('version', sa.Integer()))
+    for field in session.query(MetadataFormField).all():
+        field.version = 1
+        session.commit()
+    op.alter_column('metadata_form_field', 'version', nullable=False)
+    op.drop_constraint('fk_mf_filed_form_uri', 'metadata_form_field', type_='foreignkey')
+    op.create_foreign_key(
+        'fk_version',
+        'metadata_form_field',
+        'metadata_form_version',
+        ['metadataFormUri', 'version'],
+        ['metadataFormUri', 'version'],
+        ondelete='CASCADE',
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint('fk_version', 'metadata_form_field', type_='foreignkey')
+    op.drop_column('metadata_form_field', 'version')
+    op.drop_constraint('f_key_enforcement_version_metadata', 'metadata_form_enforcement_rule', type_='foreignkey')
+    op.drop_column('metadata_form_enforcement_rule', 'version')
+    op.drop_constraint('fk_attached_mf_version_uri', 'attached_metadata_form', type_='foreignkey')
+    op.drop_column('attached_metadata_form', 'version')
+    op.drop_table('metadata_form_version')
+
+    op.create_foreign_key(
+        'fk_mf_filed_form_uri',
+        'metadata_form_field',
+        'metadata_form',
+        ['metadataFormUri'],
+        ['uri'],
+        ondelete='CASCADE',
+    )
+
+    op.create_foreign_key(
+        'f_key_enforcement_metadata',
+        'metadata_form_enforcement_rule',
+        'metadata_form',
+        ['metadataFormUri'],
+        ['uri'],
+        ondelete='CASCADE',
+    )
+
+    op.create_foreign_key(
+        'fk_attached_mf_uri',
+        'attached_metadata_form',
+        'metadata_form',
+        ['metadataFormUri'],
+        ['uri'],
+        ondelete='CASCADE',
+    )
+    # ### end Alembic commands ###