From 4f922dca789b047bfb29fdd2557d4e2fc54ded00 Mon Sep 17 00:00:00 2001
From: Laszlo Kecskes <laszlokecskes@protonmail.com>
Date: Thu, 18 Apr 2024 12:53:23 +0200
Subject: [PATCH] migration test and implementation

---
 .../165-default_empty_metadata/index.ts       | 100 ++++++
 .../specs/165-default_empty_metadata.spec.ts  | 120 ++++++-
 .../specs/fixtures.ts                         | 294 ++++++++----------
 .../165-default_empty_metadata/types.ts       |   6 +-
 4 files changed, 331 insertions(+), 189 deletions(-)

diff --git a/app/api/migrations/migrations/165-default_empty_metadata/index.ts b/app/api/migrations/migrations/165-default_empty_metadata/index.ts
index 29c5141f44..2e9442e2f6 100644
--- a/app/api/migrations/migrations/165-default_empty_metadata/index.ts
+++ b/app/api/migrations/migrations/165-default_empty_metadata/index.ts
@@ -1,4 +1,11 @@
+/* eslint-disable no-await-in-loop */
 import { Db } from 'mongodb';
+import { Entity, Template } from './types';
+
+let entitiesToUpdate: Entity[] = [];
+
+const propDoesNotExist = (obj: Record<string, unknown>, prop: string) =>
+  !obj.hasOwnProperty(prop) || obj[prop] === null || obj[prop] === undefined;
 
 export default {
   delta: 165,
@@ -7,9 +14,102 @@ export default {
 
   description: 'Adds empty array as default metadata for all entities.',
 
+  batchSize: 1000,
+
   reindex: false,
 
+  propertiesByTemplate: {} as Record<string, string[]>,
+
+  fullEmptyMetadataByTemplate: {} as Record<string, Record<string, []>>,
+
+  async readProperties(db: Db) {
+    const templates = await db.collection<Template>('templates').find().toArray();
+    this.propertiesByTemplate = {};
+    this.fullEmptyMetadataByTemplate = {};
+    templates.forEach(template => {
+      const properties = template.properties?.map(property => property.name) || [];
+      const idString = template._id?.toString() || '';
+      this.propertiesByTemplate[idString] = properties;
+      this.fullEmptyMetadataByTemplate[idString] = {};
+      properties.forEach(property => {
+        this.fullEmptyMetadataByTemplate[idString][property] = [];
+      });
+    });
+  },
+
+  async flushUpdates(db: Db) {
+    if (!entitiesToUpdate.length) return;
+    const entitiesCollection = db.collection<Entity>('entities');
+    const operations = entitiesToUpdate.map(entity => ({
+      updateOne: {
+        filter: { _id: entity._id },
+        update: { $set: entity },
+      },
+    }));
+    await entitiesCollection.bulkWrite(operations);
+    this.reindex = true;
+    entitiesToUpdate = [];
+  },
+
+  async performUpdates(db: Db) {
+    if (entitiesToUpdate.length >= this.batchSize) {
+      await this.flushUpdates(db);
+    }
+  },
+
+  repairMetadata(templateId: string) {
+    return { newMetadata: this.fullEmptyMetadataByTemplate[templateId], repaired: true };
+  },
+
+  repairProperties(templateId: string, entity: Entity) {
+    const properties = this.propertiesByTemplate[templateId];
+    const newMetadata = { ...(entity.metadata || {}) };
+    const missingProperties = properties.filter(prop => propDoesNotExist(newMetadata, prop));
+    if (missingProperties.length) {
+      missingProperties.forEach(prop => {
+        newMetadata[prop] = [];
+      });
+    }
+    return { newMetadata, repaired: missingProperties.length > 0 };
+  },
+
+  repairEntity(entity: Entity) {
+    const templateId = entity.template?.toString() || '';
+    let repaired = false;
+    let newMetadata: NonNullable<Entity['metadata']> = {};
+    if (propDoesNotExist(entity, 'metadata')) {
+      ({ newMetadata, repaired } = this.repairMetadata(templateId));
+    } else {
+      ({ newMetadata, repaired } = this.repairProperties(templateId, entity));
+    }
+    return { newEntity: { ...entity, metadata: newMetadata }, repaired };
+  },
+
+  async handleEntity(db: Db, entity: Entity | null) {
+    if (!entity) return;
+    const { newEntity, repaired } = this.repairEntity(entity);
+    if (repaired) {
+      entitiesToUpdate.push(newEntity);
+      await this.performUpdates(db);
+    }
+  },
+
+  async handleEntities(db: Db) {
+    const entitiesCollection = db.collection<Entity>('entities');
+    const entityCursor = entitiesCollection.find({});
+
+    while (await entityCursor.hasNext()) {
+      const entity = await entityCursor.next();
+      await this.handleEntity(db, entity);
+    }
+    if (entitiesToUpdate.length) {
+      await this.flushUpdates(db);
+    }
+  },
+
   async up(db: Db) {
     process.stdout.write(`${this.name}...\r\n`);
+    await this.readProperties(db);
+    await this.handleEntities(db);
   },
 };
diff --git a/app/api/migrations/migrations/165-default_empty_metadata/specs/165-default_empty_metadata.spec.ts b/app/api/migrations/migrations/165-default_empty_metadata/specs/165-default_empty_metadata.spec.ts
index c465f40fb9..8f712905fc 100644
--- a/app/api/migrations/migrations/165-default_empty_metadata/specs/165-default_empty_metadata.spec.ts
+++ b/app/api/migrations/migrations/165-default_empty_metadata/specs/165-default_empty_metadata.spec.ts
@@ -1,21 +1,24 @@
 import { Db } from 'mongodb';
 
 import testingDB from 'api/utils/testing_db';
-import migration from '../index.js';
-import { Fixture } from '../types.js';
-import { fixtures } from './fixtures.js';
+import migration from '../index';
+import { Entity, Fixture } from '../types';
+import { fixtures, correctFixtures } from './fixtures';
 
 let db: Db | null;
 
 const initTest = async (fixture: Fixture) => {
   await testingDB.setupFixturesAndContext(fixture);
   db = testingDB.mongodb!;
+  migration.reindex = false;
+  migration.batchSize = 4;
+
   await migration.up(db);
 };
 
 beforeAll(async () => {
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  jest.spyOn(process.stdout, 'write').mockImplementation((str: string | Uint8Array) => true);
+  // jest.spyOn(process.stdout, 'write').mockImplementation((str: string | Uint8Array) => true);
 });
 
 afterAll(async () => {
@@ -23,19 +26,112 @@ afterAll(async () => {
 });
 
 describe('migration test', () => {
-  beforeAll(async () => {
-    await initTest(fixtures);
-  });
-
   it('should have a delta number', () => {
     expect(migration.delta).toBe(165);
   });
 
-  it('should be tested', async () => {
-    expect(true).toBe(false);
+  describe('on a correct database', () => {
+    beforeAll(async () => {
+      await initTest(correctFixtures);
+    });
+
+    it('should do nothing', async () => {
+      const entities = await db!.collection<Entity>('entities').find().toArray();
+      expect(entities).toEqual(correctFixtures.entities);
+    });
+
+    it('should not signal a reindex', async () => {
+      expect(migration.reindex).toBe(false);
+    });
   });
 
-  it('should check if a reindex is needed', async () => {
-    expect(migration.reindex).toBe(undefined);
+  describe('on a faulty database', () => {
+    beforeAll(async () => {
+      await initTest(fixtures);
+    });
+
+    const correctEmptyMetadata = {
+      text: [],
+      select: [],
+      relationship: [],
+    };
+
+    const expectedCorrectMetadata = [correctEmptyMetadata, correctEmptyMetadata];
+
+    it('should not modify correct entities', async () => {
+      const entities = await db!
+        .collection<Entity>('entities')
+        .find({ sharedId: 'correct_entity_sharedId' })
+        .toArray();
+      expect(entities).toEqual(correctFixtures.entities);
+    });
+
+    it.each([
+      {
+        title: 'entity_without_metadata',
+        expectedMetadata: expectedCorrectMetadata,
+      },
+      {
+        title: 'entity_with_undefined_metadata',
+        expectedMetadata: expectedCorrectMetadata,
+      },
+      {
+        title: 'entity_with_null_metadata',
+        expectedMetadata: expectedCorrectMetadata,
+      },
+      {
+        title: 'entity_with_empty_metadata',
+        expectedMetadata: expectedCorrectMetadata,
+      },
+      {
+        title: 'entity_with_undefined_metadata_properties',
+        expectedMetadata: expectedCorrectMetadata,
+      },
+      {
+        title: 'entity_with_null_metadata_properties',
+        expectedMetadata: expectedCorrectMetadata,
+      },
+      {
+        title: 'entity_with_empty_metadata_properties',
+        expectedMetadata: expectedCorrectMetadata,
+      },
+      {
+        title: 'entity_with_assymetric_metadata',
+        expectedMetadata: [
+          {
+            text: [],
+            select: [],
+            relationship: [],
+          },
+          {
+            text: [],
+            select: [],
+            relationship: [],
+          },
+        ],
+      },
+      {
+        title: 'other_template',
+        expectedMetadata: [
+          {
+            text: [],
+            select: [],
+          },
+          {
+            text: [],
+            select: [],
+          },
+        ],
+      },
+    ])('should fix case: $title', async ({ title, expectedMetadata }) => {
+      const sharedId = `${title}_sharedId`;
+      const entities = await db!.collection<Entity>('entities').find({ sharedId }).toArray();
+      const metadata = entities.map(e => e.metadata);
+      expect(metadata).toEqual(expectedMetadata);
+    });
+
+    it('should signal a reindex', async () => {
+      expect(migration.reindex).toBe(true);
+    });
   });
 });
diff --git a/app/api/migrations/migrations/165-default_empty_metadata/specs/fixtures.ts b/app/api/migrations/migrations/165-default_empty_metadata/specs/fixtures.ts
index f8795efe65..f7c9699aec 100644
--- a/app/api/migrations/migrations/165-default_empty_metadata/specs/fixtures.ts
+++ b/app/api/migrations/migrations/165-default_empty_metadata/specs/fixtures.ts
@@ -1,10 +1,26 @@
 import { ObjectId } from 'mongodb';
-import { Fixture, Template } from '../types';
+import { Entity, Fixture, Settings, Template } from '../types';
 
 const thesauriId = new ObjectId();
 const relTypeId = new ObjectId();
 const sourceTemplateId = new ObjectId();
-const inheritedPropertyId = new ObjectId();
+
+const settings: Settings[] = [
+  {
+    _id: new ObjectId(),
+    languages: [
+      {
+        key: 'en',
+        default: true,
+        label: 'English',
+      },
+      {
+        key: 'es',
+        label: 'Spanish',
+      },
+    ],
+  },
+];
 
 const template: Template = {
   properties: [
@@ -29,122 +45,104 @@ const template: Template = {
       relationType: relTypeId.toString(),
       name: 'relationship',
     },
-    {
-      content: sourceTemplateId.toString(),
-      _id: new ObjectId(),
-      label: 'Inherited',
-      type: 'relationship',
-      relationType: relTypeId.toString(),
-      name: 'inherited',
-      inherit: {
-        property: inheritedPropertyId.toString(),
-        type: 'text',
-      },
-    },
   ],
   entityViewPage: '',
   name: 'test_template',
+  _id: new ObjectId(),
 };
 
-const fixtures: Fixture = {
-  settings: [
+const template2: Template = {
+  properties: [
     {
       _id: new ObjectId(),
-      languages: [
+      label: 'Text',
+      type: 'text',
+      name: 'text',
+    },
+    {
+      content: thesauriId.toString(),
+      _id: new ObjectId(),
+      label: 'Select',
+      type: 'select',
+      name: 'select',
+    },
+  ],
+  entityViewPage: '',
+  name: 'test_template2',
+  _id: new ObjectId(),
+};
+
+const templates: Template[] = [template, template2];
+
+const correctEntities: Entity[] = [
+  {
+    _id: new ObjectId(),
+    title: 'correct_entity',
+    sharedId: 'correct_entity_sharedId',
+    template: template._id,
+    language: 'en',
+    metadata: {
+      text: [
         {
-          key: 'en',
-          default: true,
-          label: 'English',
+          value: 'text',
         },
+      ],
+      select: [
         {
-          key: 'es',
-          label: 'Spanish',
+          value: 'A_id',
+          label: 'A',
+        },
+      ],
+      relationship: [
+        {
+          value: 'S_sharedId',
+          label: 'S',
+          type: 'entity',
         },
       ],
     },
-  ],
-  templates: [template],
-  entities: [
-    {
-      _id: new ObjectId(),
-      title: 'correct_entity',
-      sharedId: 'correct_entity_sharedId',
-      template: template._id,
-      language: 'en',
-      metadata: {
-        text: [
-          {
-            value: 'text',
-          },
-        ],
-        select: [
-          {
-            value: 'A_id',
-            label: 'A',
-          },
-        ],
-        relationship: [
-          {
-            value: 'S_sharedId',
-            label: 'S',
-            type: 'entity',
-          },
-        ],
-        inherited: [
-          {
-            value: 'S_sharedId',
-            label: 'S',
-            type: 'entity',
-            inheritedValue: [
-              {
-                value: 'text',
-              },
-            ],
-            inheritedType: 'text',
-          },
-        ],
-      },
-    },
-    {
-      _id: new ObjectId(),
-      title: 'correct_entity_es',
-      sharedId: 'correct_entity_sharedId',
-      template: template._id,
-      language: 'es',
-      metadata: {
-        text: [
-          {
-            value: 'text_es',
-          },
-        ],
-        select: [
-          {
-            value: 'A_id',
-            label: 'A_es',
-          },
-        ],
-        relationship: [
-          {
-            value: 'S_sharedId',
-            label: 'S_es',
-            type: 'entity',
-          },
-        ],
-        inherited: [
-          {
-            value: 'S_sharedId',
-            label: 'S_es',
-            type: 'entity',
-            inheritedValue: [
-              {
-                value: 'text_es',
-              },
-            ],
-            inheritedType: 'text',
-          },
-        ],
-      },
+  },
+  {
+    _id: new ObjectId(),
+    title: 'correct_entity_es',
+    sharedId: 'correct_entity_sharedId',
+    template: template._id,
+    language: 'es',
+    metadata: {
+      text: [
+        {
+          value: 'text_es',
+        },
+      ],
+      select: [
+        {
+          value: 'A_id',
+          label: 'A_es',
+        },
+      ],
+      relationship: [
+        {
+          value: 'S_sharedId',
+          label: 'S_es',
+          type: 'entity',
+        },
+      ],
     },
+  },
+];
+
+const correctFixtures: Fixture = {
+  settings,
+  templates,
+  entities: correctEntities,
+};
+
+const fixtures: Fixture = {
+  settings,
+  templates,
+  entities: [
+    correctEntities[0],
+    correctEntities[1],
     {
       _id: new ObjectId(),
       title: 'entity_without_metadata',
@@ -222,8 +220,6 @@ const fixtures: Fixture = {
         select: undefined,
         //@ts-ignore
         relationship: undefined,
-        //@ts-ignore
-        inherited: undefined,
       },
     },
     {
@@ -239,8 +235,6 @@ const fixtures: Fixture = {
         select: undefined,
         //@ts-ignore
         relationship: undefined,
-        //@ts-ignore
-        inherited: undefined,
       },
     },
     {
@@ -256,8 +250,6 @@ const fixtures: Fixture = {
         select: null,
         //@ts-ignore
         relationship: null,
-        //@ts-ignore
-        inherited: null,
       },
     },
     {
@@ -273,8 +265,6 @@ const fixtures: Fixture = {
         select: null,
         //@ts-ignore
         relationship: null,
-        //@ts-ignore
-        inherited: null,
       },
     },
     {
@@ -287,7 +277,6 @@ const fixtures: Fixture = {
         text: [],
         select: [],
         relationship: [],
-        inherited: [],
       },
     },
     {
@@ -300,94 +289,55 @@ const fixtures: Fixture = {
         text: [],
         select: [],
         relationship: [],
-        inherited: [],
       },
     },
     {
       _id: new ObjectId(),
-      title: 'entity_with_missing_inherited_value',
-      sharedId: 'entity_with_missing_inherited_value_sharedId',
+      title: 'entity_with_assymetric_metadata',
+      sharedId: 'entity_with_assymetric_metadata_sharedId',
       template: template._id,
       language: 'en',
       metadata: {
         text: [],
         select: [],
         relationship: [],
-        inherited: [
-          {
-            value: 'S_sharedId',
-            label: 'S',
-            type: 'entity',
-            inheritedType: 'text',
-          },
-        ],
       },
     },
     {
       _id: new ObjectId(),
-      title: 'entity_with_missing_inherited_value_es',
-      sharedId: 'entity_with_missing_inherited_value_sharedId',
+      title: 'entity_with_assymetric_metadata_es',
+      sharedId: 'entity_with_assymetric_metadata_sharedId',
       template: template._id,
       language: 'es',
       metadata: {
-        text: [],
-        select: [],
-        relationship: [],
-        inherited: [
-          {
-            value: 'S_sharedId',
-            label: 'S_es',
-            type: 'entity',
-            inheritedType: 'text',
-          },
-        ],
+        //@ts-ignore
+        select: undefined,
+        //@ts-ignore
+        relationship: null,
       },
     },
     {
-      id: new ObjectId(),
-      title: 'entity_with_assymetric_case',
-      sharedId: 'entity_with_assymetric_case_sharedId',
-      template: template._id,
+      _id: new ObjectId(),
+      title: 'other_template',
+      sharedId: 'other_template_sharedId',
+      template: template2._id,
       language: 'en',
       metadata: {
-        text: [],
-        select: [],
-        relationship: [],
-        inherited: [
-          {
-            value: 'S_sharedId',
-            label: 'S',
-            type: 'entity',
-            inheritedValue: [],
-            inheritedType: 'text',
-          },
-        ],
+        //@ts-ignore
+        select: null,
       },
     },
     {
-      id: new ObjectId(),
-      title: 'entity_with_assymetric_case_es',
-      sharedId: 'entity_with_assymetric_case_sharedId',
-      template: template._id,
+      _id: new ObjectId(),
+      title: 'other_template_es',
+      sharedId: 'other_template_sharedId',
+      template: template2._id,
       language: 'es',
       metadata: {
-        //@ts-ignore
-        select: undefined,
-        //@ts-ignore
-        relationship: null,
-        inherited: [
-          {
-            value: 'S_sharedId',
-            label: 'S_es',
-            type: 'entity',
-            //@ts-ignore
-            inheritedValue: null,
-            inheritedType: 'text',
-          },
-        ],
+        text: [],
       },
     },
   ],
 };
 
-export { fixtures };
+export { fixtures, correctFixtures };
diff --git a/app/api/migrations/migrations/165-default_empty_metadata/types.ts b/app/api/migrations/migrations/165-default_empty_metadata/types.ts
index 10620a51e6..5d05a20eed 100644
--- a/app/api/migrations/migrations/165-default_empty_metadata/types.ts
+++ b/app/api/migrations/migrations/165-default_empty_metadata/types.ts
@@ -22,10 +22,6 @@ interface PropertySchema {
   type: PropertyType;
   content?: string;
   relationType?: string;
-  inherit?: {
-    property?: string;
-    type?: PropertyType;
-  };
 }
 
 interface Template {
@@ -78,4 +74,4 @@ interface Fixture {
   entities: Entity[];
 }
 
-export type { Entity, Fixture, Template };
+export type { Entity, Fixture, Template, Settings };