[new script] [programs] Move data values from old data elements to new data elements on specific tracker program's program stage

src/data/DataElementsD2Repository.ts

@@ -1,23 +1,31 @@
 import _ from "lodash";
-import { D2Api, Id } from "types/d2-api";
+import { D2Api, Id, MetadataPick } from "types/d2-api";
 import { NamedRef } from "domain/entities/Base";
 import { DataElementsRepository } from "domain/repositories/DataElementsRepository";
+import { DataElement } from "domain/entities/DataElement";
 
 export class DataElementsD2Repository implements DataElementsRepository {
     constructor(private api: D2Api) {}
 
+    async getByIds(ids: Id[]): Promise<DataElement[]> {
+        return this.getDataElements(ids);
+    }
+
     async getDataElementsNames(ids: Id[]): Promise<NamedRef[]> {
+        return this.getDataElements(ids).then(dataElements =>
+            dataElements.map(de => ({ id: de.id, name: de.name }))
+        );
+    }
+
+    private async getDataElements(ids: Id[]): Promise<D2DataElement[]> {
         const metadata$ = this.api.metadata.get({
             dataElements: {
-                fields: {
-                    id: true,
-                    name: true,
-                },
+                fields: dataElementFields,
                 filter: { id: { in: ids } },
             },
         });
 
-        const { dataElements } = await metadata$.getData();
+        const dataElements = (await metadata$.getData()).dataElements;
         const dataElementsIds = dataElements.map(de => de.id);
         const dataElementsIdsNotFound = _.difference(ids, dataElementsIds);
 
@@ -28,3 +36,13 @@ export class DataElementsD2Repository implements DataElementsRepository {
         }
     }
 }
+
+const dataElementFields = {
+    id: true,
+    name: true,
+    valueType: true,
+} as const;
+
+type D2DataElement = MetadataPick<{
+    dataElements: { fields: typeof dataElementFields };
+}>["dataElements"][number];

src/data/OrgUnitD2Repository.ts

@@ -8,6 +8,32 @@ import { Identifiable } from "domain/entities/Base";
 export class OrgUnitD2Repository implements OrgUnitRepository {
     constructor(private api: D2Api) {}
 
+    async getRoot(): Promise<OrgUnit> {
+        const response = await this.api.metadata
+            .get({
+                organisationUnits: {
+                    fields: {
+                        id: true,
+                        code: true,
+                        name: true,
+                    },
+                    filter: {
+                        level: {
+                            eq: "1",
+                        },
+                    },
+                },
+            })
+            .getData();
+
+        const rootOrgUnit = response.organisationUnits[0];
+        if (!rootOrgUnit) {
+            throw new Error("Root org unit not found");
+        }
+
+        return rootOrgUnit;
+    }
+
     async getByIdentifiables(values: Identifiable[]): Promise<OrgUnit[]> {
         return this.getOrgUnits(values);
     }

src/domain/entities/DataElement.ts

@@ -1,9 +1,7 @@
 import { Id } from "./Base";
-import { Translation } from "./Translation";
 
 export interface DataElement {
     id: Id;
     name: string;
-    formName: string;
-    translations: Translation[];
+    valueType: string;
 }

src/domain/repositories/DataElementsRepository.ts

@@ -1,5 +1,7 @@
 import { Id, NamedRef } from "domain/entities/Base";
+import { DataElement } from "domain/entities/DataElement";
 
 export interface DataElementsRepository {
+    getByIds(ids: Id[]): Promise<DataElement[]>;
     getDataElementsNames(ids: Id[]): Promise<NamedRef[]>;
 }

src/domain/repositories/OrgUnitRepository.ts

@@ -3,4 +3,5 @@ import { OrgUnit } from "domain/entities/OrgUnit";
 
 export interface OrgUnitRepository {
     getByIdentifiables(ids: Identifiable[]): Promise<OrgUnit[]>;
+    getRoot(): Promise<OrgUnit>;
 }

src/domain/usecases/CopyProgramStageDataValuesUseCase.ts

@@ -0,0 +1,175 @@
+import _ from "lodash";
+import fs from "fs";
+import { Id } from "domain/entities/Base";
+import { DataElement } from "domain/entities/DataElement";
+import { DataElementsRepository } from "domain/repositories/DataElementsRepository";
+import { ProgramEventsRepository } from "domain/repositories/ProgramEventsRepository";
+import { OrgUnitRepository } from "domain/repositories/OrgUnitRepository";
+import { ProgramEvent } from "domain/entities/ProgramEvent";
+import log from "utils/log";
+
+export class CopyProgramStageDataValuesUseCase {
+    constructor(
+        private programEventsRepository: ProgramEventsRepository,
+        private orgUnitRepository: OrgUnitRepository,
+        private dataElementsRepository: DataElementsRepository
+    ) {}
+
+    async execute(options: CopyProgramStageDataValuesOptions): Promise<ProgramEvent[]> {
+        const { programStageId, dataElementIdPairs: idPairs, post, saveReport: reportPath } = options;
+
+        const rootOrgUnit = await this.orgUnitRepository.getRoot();
+        const dataElements = await this.dataElementsRepository.getByIds(idPairs.flat());
+        const dataElementPairs = this.mapDataElements(dataElements, idPairs);
+        const sourceIds = idPairs.map(([sourceId, _]) => sourceId);
+        const targetIds = idPairs.map(([_, targetId]) => targetId);
+
+        checkDataElementTypes(dataElementPairs);
+
+        const allEvents = await this.programEventsRepository.get({
+            programStagesIds: [programStageId],
+            orgUnitsIds: [rootOrgUnit.id],
+            orgUnitMode: "DESCENDANTS",
+        });
+
+        const applicableEvents = allEvents.filter(event =>
+            event.dataValues.some(dv => sourceIds.includes(dv.dataElement.id))
+        );
+
+        checkTargetDataValuesAreEmpty(applicableEvents, targetIds);
+
+        const eventsWithNewDataValues = this.copyEventDataValues(
+            applicableEvents,
+            sourceIds,
+            dataElementPairs
+        );
+
+        if (post) {
+            const result = await this.programEventsRepository.save(eventsWithNewDataValues);
+            if (result.type === "success") log.info(JSON.stringify(result, null, 4));
+            else log.error(JSON.stringify(result, null, 4));
+        } else {
+            const payload = { events: eventsWithNewDataValues };
+            const json = JSON.stringify(payload, null, 4);
+            const now = new Date().toISOString().slice(0, 19).replace(/:/g, "-");
+            const payloadPath = `copy-program-stage-data-values-${now}.json`;
+
+            fs.writeFileSync(payloadPath, json);
+            log.info(`Written payload (${eventsWithNewDataValues.length} events): ${payloadPath}`);
+        }
+
+        if (reportPath) {
+            this.saveReport(reportPath, dataElementPairs, programStageId, eventsWithNewDataValues);
+        }
+
+        return eventsWithNewDataValues;
+    }
+
+    private copyEventDataValues(
+        applicableEvents: ProgramEvent[],
+        sourceIds: string[],
+        dataElementPairs: DataElementPair[]
+    ) {
+        return applicableEvents.map(event => ({
+            ...event,
+            dataValues: event.dataValues.flatMap(dv => {
+                if (!sourceIds.includes(dv.dataElement.id)) return [dv];
+                const target = dataElementPairs.find(([source, _]) => source.id === dv.dataElement.id)?.[1];
+
+                if (!target)
+                    throw new Error(`Target data element not found for source id: ${dv.dataElement.id}`);
+
+                return [
+                    dv,
+                    {
+                        ...dv,
+                        dataElement: _.omit(target, "valueType"),
+                    },
+                ];
+            }),
+        }));
+    }
+
+    private mapDataElements(dataElements: DataElement[], pairs: [Id, Id][]): DataElementPair[] {
+        const dataElementPairs = pairs.map(([sourceId, targetId]) => {
+            const sourceElement = dataElements.find(de => de.id === sourceId);
+            const targetElement = dataElements.find(de => de.id === targetId);
+
+            if (!sourceElement || !targetElement)
+                return `Data element not found for pair: [${sourceId}, ${targetId}]`;
+            else return [sourceElement, targetElement];
+        });
+
+        const errors = dataElementPairs.filter(pair => typeof pair === "string");
+        if (!_.isEmpty(errors)) throw new Error(errors.join("\n"));
+
+        return dataElementPairs.filter((pair): pair is DataElementPair => typeof pair !== "string");
+    }
+
+    private saveReport(
+        path: string,
+        dataElementPairs: DataElementPair[],
+        programStageId: string,
+        eventsWithNewDataValues: ProgramEvent[]
+    ) {
+        const dataElementLines = dataElementPairs.map(
+            ([source, target]) =>
+                `Source DataElement: ${source.id} (${source.name}), Target DataElement: ${target.id} (${target.name})`
+        );
+
+        const eventLines = eventsWithNewDataValues.map(event => {
+            const dataValueLines = dataElementPairs.flatMap(([source, target]) => {
+                const sourceValue = event.dataValues.find(dv => dv.dataElement.id === source.id)?.value;
+                const status = sourceValue ? `(${sourceValue})` : undefined;
+                return status ? [`\tCopy ${source.id} to ${target.id} ${status}`] : [];
+            });
+
+            return `Event ID: ${event.id}, OrgUnit ID: ${event.orgUnit.id}\n${dataValueLines.join("\n")}`;
+        });
+
+        const content = [
+            "Program Stage ID: " + programStageId,
+            dataElementLines.join("\n"),
+            "Number of events: " + eventsWithNewDataValues.length,
+            eventLines.join("\n"),
+        ].join("\n\n");
+
+        fs.writeFileSync(path, content);
+        log.info(`Written report: ${path}`);
+    }
+}
+
+function checkDataElementTypes(dePairs: DataElementPair[]) {
+    const typeMismatchErrors = dePairs
+        .filter(([source, target]) => source.valueType !== target.valueType)
+        .map(([source, target]) => `Data elements [${source.id}, ${target.id}] do not have the same type.`);
+
+    if (!_.isEmpty(typeMismatchErrors)) throw new Error(typeMismatchErrors.join("\n"));
+}
+
+function checkTargetDataValuesAreEmpty(events: ProgramEvent[], targetIds: Id[]) {
+    const eventsWithNonEmptyTargetDataValues = _(events)
+        .map(event => {
+            const nonEmpty = event.dataValues
+                .filter(dv => targetIds.includes(dv.dataElement.id))
+                .filter(dv => Boolean(dv.value))
+                .map(dv => `\tTarget DataElement: ${dv.dataElement.id}, Value: ${JSON.stringify(dv.value)}`)
+                .join("\n");
+
+            return _.isEmpty(nonEmpty) ? undefined : `Event ID: ${event.id}, Values: \n${nonEmpty}`;
+        })
+        .compact()
+        .join("\n");
+
+    const error = `Some data values of the destination data elements are not empty:\n${eventsWithNonEmptyTargetDataValues}`;
+    if (eventsWithNonEmptyTargetDataValues) throw new Error(error);
+}
+
+export type CopyProgramStageDataValuesOptions = {
+    programStageId: string;
+    dataElementIdPairs: [Id, Id][]; // [sourceDataElementId, targetDataElementId]
+    post: boolean;
+    saveReport?: string;
+};
+
+type DataElementPair = [DataElement, DataElement];