migration test and implementation

app/api/migrations/migrations/165-default_empty_metadata/index.ts

@@ -1,4 +1,11 @@
+/* eslint-disable no-await-in-loop */
 import { Db } from 'mongodb';
+import { Entity, Template } from './types';
+
+let entitiesToUpdate: Entity[] = [];
+
+const propDoesNotExist = (obj: Record<string, unknown>, prop: string) =>
+  !obj.hasOwnProperty(prop) || obj[prop] === null || obj[prop] === undefined;
 
 export default {
   delta: 165,
@@ -7,9 +14,102 @@ export default {
 
   description: 'Adds empty array as default metadata for all entities.',
 
+  batchSize: 1000,
+
   reindex: false,
 
+  propertiesByTemplate: {} as Record<string, string[]>,
+
+  fullEmptyMetadataByTemplate: {} as Record<string, Record<string, []>>,
+
+  async readProperties(db: Db) {
+    const templates = await db.collection<Template>('templates').find().toArray();
+    this.propertiesByTemplate = {};
+    this.fullEmptyMetadataByTemplate = {};
+    templates.forEach(template => {
+      const properties = template.properties?.map(property => property.name) || [];
+      const idString = template._id?.toString() || '';
+      this.propertiesByTemplate[idString] = properties;
+      this.fullEmptyMetadataByTemplate[idString] = {};
+      properties.forEach(property => {
+        this.fullEmptyMetadataByTemplate[idString][property] = [];
+      });
+    });
+  },
+
+  async flushUpdates(db: Db) {
+    if (!entitiesToUpdate.length) return;
+    const entitiesCollection = db.collection<Entity>('entities');
+    const operations = entitiesToUpdate.map(entity => ({
+      updateOne: {
+        filter: { _id: entity._id },
+        update: { $set: entity },
+      },
+    }));
+    await entitiesCollection.bulkWrite(operations);
+    this.reindex = true;
+    entitiesToUpdate = [];
+  },
+
+  async performUpdates(db: Db) {
+    if (entitiesToUpdate.length >= this.batchSize) {
+      await this.flushUpdates(db);
+    }
+  },
+
+  repairMetadata(templateId: string) {
+    return { newMetadata: this.fullEmptyMetadataByTemplate[templateId], repaired: true };
+  },
+
+  repairProperties(templateId: string, entity: Entity) {
+    const properties = this.propertiesByTemplate[templateId];
+    const newMetadata = { ...(entity.metadata || {}) };
+    const missingProperties = properties.filter(prop => propDoesNotExist(newMetadata, prop));
+    if (missingProperties.length) {
+      missingProperties.forEach(prop => {
+        newMetadata[prop] = [];
+      });
+    }
+    return { newMetadata, repaired: missingProperties.length > 0 };
+  },
+
+  repairEntity(entity: Entity) {
+    const templateId = entity.template?.toString() || '';
+    let repaired = false;
+    let newMetadata: NonNullable<Entity['metadata']> = {};
+    if (propDoesNotExist(entity, 'metadata')) {
+      ({ newMetadata, repaired } = this.repairMetadata(templateId));
+    } else {
+      ({ newMetadata, repaired } = this.repairProperties(templateId, entity));
+    }
+    return { newEntity: { ...entity, metadata: newMetadata }, repaired };
+  },
+
+  async handleEntity(db: Db, entity: Entity | null) {
+    if (!entity) return;
+    const { newEntity, repaired } = this.repairEntity(entity);
+    if (repaired) {
+      entitiesToUpdate.push(newEntity);
+      await this.performUpdates(db);
+    }
+  },
+
+  async handleEntities(db: Db) {
+    const entitiesCollection = db.collection<Entity>('entities');
+    const entityCursor = entitiesCollection.find({});
+
+    while (await entityCursor.hasNext()) {
+      const entity = await entityCursor.next();
+      await this.handleEntity(db, entity);
+    }
+    if (entitiesToUpdate.length) {
+      await this.flushUpdates(db);
+    }
+  },
+
   async up(db: Db) {
     process.stdout.write(`${this.name}...\r\n`);
+    await this.readProperties(db);
+    await this.handleEntities(db);
   },
 };

app/api/migrations/migrations/165-default_empty_metadata/specs/165-default_empty_metadata.spec.ts

@@ -1,41 +1,137 @@
 import { Db } from 'mongodb';
 
 import testingDB from 'api/utils/testing_db';
-import migration from '../index.js';
-import { Fixture } from '../types.js';
-import { fixtures } from './fixtures.js';
+import migration from '../index';
+import { Entity, Fixture } from '../types';
+import { fixtures, correctFixtures } from './fixtures';
 
 let db: Db | null;
 
 const initTest = async (fixture: Fixture) => {
   await testingDB.setupFixturesAndContext(fixture);
   db = testingDB.mongodb!;
+  migration.reindex = false;
+  migration.batchSize = 4;
+
   await migration.up(db);
 };
 
 beforeAll(async () => {
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  jest.spyOn(process.stdout, 'write').mockImplementation((str: string | Uint8Array) => true);
+  // jest.spyOn(process.stdout, 'write').mockImplementation((str: string | Uint8Array) => true);
 });
 
 afterAll(async () => {
   await testingDB.tearDown();
 });
 
 describe('migration test', () => {
-  beforeAll(async () => {
-    await initTest(fixtures);
-  });
-
   it('should have a delta number', () => {
     expect(migration.delta).toBe(165);
   });
 
-  it('should be tested', async () => {
-    expect(true).toBe(false);
+  describe('on a correct database', () => {
+    beforeAll(async () => {
+      await initTest(correctFixtures);
+    });
+
+    it('should do nothing', async () => {
+      const entities = await db!.collection<Entity>('entities').find().toArray();
+      expect(entities).toEqual(correctFixtures.entities);
+    });
+
+    it('should not signal a reindex', async () => {
+      expect(migration.reindex).toBe(false);
+    });
   });
 
-  it('should check if a reindex is needed', async () => {
-    expect(migration.reindex).toBe(undefined);
+  describe('on a faulty database', () => {
+    beforeAll(async () => {
+      await initTest(fixtures);
+    });
+
+    const correctEmptyMetadata = {
+      text: [],
+      select: [],
+      relationship: [],
+    };
+
+    const expectedCorrectMetadata = [correctEmptyMetadata, correctEmptyMetadata];
+
+    it('should not modify correct entities', async () => {
+      const entities = await db!
+        .collection<Entity>('entities')
+        .find({ sharedId: 'correct_entity_sharedId' })
+        .toArray();
+      expect(entities).toEqual(correctFixtures.entities);
+    });
+
+    it.each([
+      {
+        title: 'entity_without_metadata',
+        expectedMetadata: expectedCorrectMetadata,
+      },
+      {
+        title: 'entity_with_undefined_metadata',
+        expectedMetadata: expectedCorrectMetadata,
+      },
+      {
+        title: 'entity_with_null_metadata',
+        expectedMetadata: expectedCorrectMetadata,
+      },
+      {
+        title: 'entity_with_empty_metadata',
+        expectedMetadata: expectedCorrectMetadata,
+      },
+      {
+        title: 'entity_with_undefined_metadata_properties',
+        expectedMetadata: expectedCorrectMetadata,
+      },
+      {
+        title: 'entity_with_null_metadata_properties',
+        expectedMetadata: expectedCorrectMetadata,
+      },
+      {
+        title: 'entity_with_empty_metadata_properties',
+        expectedMetadata: expectedCorrectMetadata,
+      },
+      {
+        title: 'entity_with_assymetric_metadata',
+        expectedMetadata: [
+          {
+            text: [],
+            select: [],
+            relationship: [],
+          },
+          {
+            text: [],
+            select: [],
+            relationship: [],
+          },
+        ],
+      },
+      {
+        title: 'other_template',
+        expectedMetadata: [
+          {
+            text: [],
+            select: [],
+          },
+          {
+            text: [],
+            select: [],
+          },
+        ],
+      },
+    ])('should fix case: $title', async ({ title, expectedMetadata }) => {
+      const sharedId = `${title}_sharedId`;
+      const entities = await db!.collection<Entity>('entities').find({ sharedId }).toArray();
+      const metadata = entities.map(e => e.metadata);
+      expect(metadata).toEqual(expectedMetadata);
+    });
+
+    it('should signal a reindex', async () => {
+      expect(migration.reindex).toBe(true);
+    });
   });
 });