Add script to recode boolean data values in events

src/data/ProgramsD2Repository.ts

@@ -19,16 +19,30 @@ export class ProgramsD2Repository implements ProgramsRepository {
         this.d2Tracker = new D2Tracker(this.api);
     }
 
-    async get(options: { programTypes?: ProgramType[] }): Async<Program[]> {
+    async get(options: { ids?: Id[]; programTypes?: ProgramType[] }): Async<Program[]> {
         const { programs } = await this.api.metadata
             .get({
                 programs: {
                     fields: {
                         id: true,
                         name: true,
                         programType: true,
+                        programStages: {
+                            id: true,
+                            name: true,
+                            programStageDataElements: {
+                                dataElement: {
+                                    id: true,
+                                    name: true,
+                                    code: true,
+                                    valueType: true,
+                                    optionSet: { id: true, name: true },
+                                },
+                            },
+                        },
                     },
                     filter: {
+                        ...(options.ids ? { id: { in: options.ids } } : {}),
                         ...(options.programTypes ? { programType: { in: options.programTypes } } : {}),
                     },
                 },

src/domain/entities/Program.ts

@@ -1,9 +1,29 @@
-import { Id } from "./Base";
+import { Maybe } from "utils/ts-utils";
+import { Id, NamedRef } from "./Base";
 
 export type ProgramType = "WITH_REGISTRATION" | "WITHOUT_REGISTRATION";
 
 export interface Program {
     id: Id;
     name: string;
     programType: ProgramType;
+    programStages: ProgramStage[];
 }
+
+type ProgramStage = {
+    id: Id;
+    programStageDataElements: ProgramStageDataElement[];
+};
+
+type ProgramStageDataElement = {
+    dataElement: DataElement;
+    displayInReports: boolean;
+};
+
+type DataElement = {
+    id: Id;
+    name: string;
+    code: string;
+    valueType: string;
+    optionSet: Maybe<NamedRef>;
+};

src/domain/entities/ProgramEvent.ts

@@ -21,7 +21,7 @@ export interface ProgramEventToSave {
     program: Ref;
     orgUnit: Ref;
     programStage: Ref;
-    dataValues: EventDataValue[];
+    dataValues: EventDataValueToSave[];
     trackedEntityInstanceId?: Id;
     status: EventStatus;
     date: Timestamp;
@@ -43,6 +43,11 @@ export interface EventDataValue {
     lastUpdated: Timestamp;
 }
 
+export interface EventDataValueToSave {
+    dataElementId: Id;
+    value: string;
+}
+
 export class DuplicatedProgramEvents {
     constructor(
         private options: { ignoreDataElementsIds: Maybe<Id[]>; checkDataElementsIds?: Maybe<Id[]> }

src/domain/repositories/ProgramsRepository.ts

@@ -5,7 +5,7 @@ import { Program, ProgramType } from "domain/entities/Program";
 import { ProgramExport } from "domain/entities/ProgramExport";
 
 export interface ProgramsRepository {
-    get(options: { programTypes?: ProgramType[] }): Async<Program[]>;
+    get(options: { ids?: Id[]; programTypes?: ProgramType[] }): Async<Program[]>;
     export(options: { ids: Id[] }): Async<ProgramExport>;
     import(programExport: ProgramExport): Async<void>;
     runRules(options: RunRulesOptions): Async<void>;

src/domain/usecases/RecodeBooleanDataValuesInEventsUseCase.ts

@@ -0,0 +1,228 @@
+import fs from "fs";
+import _ from "lodash";
+import { promiseMap } from "data/dhis2-utils";
+import { getId, Id, Ref } from "domain/entities/Base";
+import { D2Api } from "types/d2-api";
+import logger from "utils/log";
+import { D2TrackerEvent } from "@eyeseetea/d2-api/api/trackerEvents";
+import { ProgramEventsRepository } from "domain/repositories/ProgramEventsRepository";
+import { ProgramsRepository } from "domain/repositories/ProgramsRepository";
+import { Program } from "domain/entities/Program";
+import { Maybe } from "utils/ts-utils";
+
+type Options = {
+    programId: Id;
+    ternaryOptionSetId: Id;
+    post: boolean;
+};
+
+export class RecodeBooleanDataValuesInEventsUseCase {
+    pageSize = 1000;
+
+    constructor(
+        private api: D2Api,
+        private programsRepository: ProgramsRepository,
+        private eventsRepository: ProgramEventsRepository
+    ) {}
+
+    async execute(options: Options) {
+        const program = await this.getProgram(options.programId);
+        await this.fixEventsInProgram({ ...options, program: program });
+    }
+
+    async getProgram(id: Id): Promise<Program> {
+        const programs = await this.programsRepository.get({ ids: [id] });
+        const program = programs[0];
+        if (!program) {
+            throw new Error(`Program not found: ${id}`);
+        }
+        return program;
+    }
+
+    async fixEventsInProgram(options: { program: Program; post: boolean; ternaryOptionSetId: Id }) {
+        const pageCount = await this.getPageCount(options);
+
+        await promiseMap(_.range(1, pageCount + 1), async page => {
+            await this.fixEventsForPage({
+                ...options,
+                page: page,
+                pageCount: pageCount,
+                ternaryOptionSetId: options.ternaryOptionSetId,
+            });
+        });
+    }
+
+    private async getPageCount(options: { program: Program }) {
+        const response = await this.api.tracker.events
+            .get({
+                ...params,
+                page: 1,
+                pageSize: 0,
+                totalPages: true,
+                program: options.program.id,
+            })
+            .getData();
+
+        const pageCount = Math.ceil((response.total || 0) / this.pageSize);
+        logger.info(`Total: ${response.total} -> pages: ${pageCount} (pageSize: ${this.pageSize})`);
+
+        return pageCount;
+    }
+
+    async fixEventsForPage(options: {
+        program: Program;
+        page: number;
+        pageCount: number;
+        post: boolean;
+        ternaryOptionSetId: Id;
+    }) {
+        const events = await this.getEvents(options);
+        const recodedEvents = this.getRecodedEvents({
+            program: options.program,
+            events: events,
+            ternaryOptionSetId: options.ternaryOptionSetId,
+        });
+        logger.info(`Events to recode: ${recodedEvents.length}`);
+
+        if (_(recodedEvents).isEmpty()) {
+            return;
+        } else if (!options.post) {
+            logger.info(`Add --post to update events`);
+        } else {
+            await this.saveEvents(recodedEvents);
+        }
+    }
+
+    getRecodedEvents(options: {
+        program: Program;
+        events: D2TrackerEvent[];
+        ternaryOptionSetId: Id;
+    }): D2TrackerEvent[] {
+        const dataElementIdsWithTernary = this.getDataElementIdsWithTernaryOptionSet(options);
+
+        return _(options.events)
+            .map((event): Maybe<typeof event> => {
+                return this.fixEvent(event, dataElementIdsWithTernary, options);
+            })
+            .compact()
+            .value();
+    }
+
+    private fixEvent(
+        event: D2TrackerEvent,
+        dataElementIdsWithTernary: Set<string>,
+        options: { program: Program }
+    ): Maybe<D2TrackerEvent> {
+        const updatedDataValues = this.recodeEvent(event, dataElementIdsWithTernary);
+        if (_.isEqual(event.dataValues, updatedDataValues)) return undefined;
+
+        const dataElementIdsUsedInDataValue = _(event.dataValues)
+            .map(dv => dv.dataElement)
+            .uniq()
+            .value();
+
+        const dataElementIdsInProgramStage = _(options.program.programStages)
+            .filter(programStage => programStage.id === event.programStage)
+            .flatMap(programStage => programStage.programStageDataElements)
+            .map(psde => psde.dataElement.id)
+            .uniq()
+            .value();
+
+        const dataElementIdsUsedAndNotCurrentlyAssigned = _.difference(
+            dataElementIdsUsedInDataValue,
+            dataElementIdsInProgramStage
+        );
+
+        if (!_(dataElementIdsUsedAndNotCurrentlyAssigned).isEmpty()) {
+            const tail = dataElementIdsUsedAndNotCurrentlyAssigned.join(" ");
+            const head = `[skip] event.id=${event.event}`;
+            const msg = `${head} [programStage.id=${event.programStage}] has unassigned dataElements: ${tail}`;
+            logger.error(msg);
+            return undefined;
+        } else {
+            return { ...event, dataValues: updatedDataValues };
+        }
+    }
+
+    private recodeEvent(event: D2TrackerEvent, dataElementIdsWithTernary: Set<string>) {
+        return _(event.dataValues)
+            .map((dataValue): typeof dataValue => {
+                if (dataElementIdsWithTernary.has(dataValue.dataElement)) {
+                    // TODO: User options from optionSet
+                    const newValue = ["true", "Yes"].includes(dataValue.value) ? "Yes" : "No";
+                    return { ...dataValue, value: newValue };
+                } else {
+                    return dataValue;
+                }
+            })
+            .value();
+    }
+
+    private getDataElementIdsWithTernaryOptionSet(options: {
+        program: Program;
+        events: D2TrackerEvent[];
+        ternaryOptionSetId: Id;
+    }) {
+        const dataElements = _(options.program.programStages).flatMap(programStage => {
+            return programStage.programStageDataElements.map(programStageDataElement => {
+                return programStageDataElement.dataElement;
+            });
+        });
+
+        const dataElementIdsWithTernaryOptions = new Set(
+            dataElements
+                .filter(dataElement => dataElement.optionSet?.id === options.ternaryOptionSetId)
+                .map(getId)
+                .value()
+        );
+        return dataElementIdsWithTernaryOptions;
+    }
+
+    private async saveEvents(events: D2TrackerEvent[]) {
+        logger.info(`Post events: ${events.length}`);
+        fs.writeFileSync("events.json", JSON.stringify(events, null, 2));
+
+        const response = await this.api.tracker
+            .post(
+                {
+                    async: false,
+                    skipPatternValidation: true,
+                    skipSideEffects: true,
+                    skipRuleEngine: true,
+                    importMode: "COMMIT",
+                },
+                { events: events }
+            )
+            .getData();
+
+        logger.info(`Post result: ${JSON.stringify(response.stats)}`);
+    }
+
+    private async getEvents(options: {
+        program: Ref;
+        page: number;
+        post: boolean;
+        pageCount: number;
+    }): Promise<D2TrackerEvent[]> {
+        logger.info(`Get events: page ${options.page} of ${options.pageCount}`);
+
+        const response = await this.api.tracker.events
+            .get({
+                ...params,
+                page: options.page,
+                pageSize: this.pageSize,
+                program: options.program.id,
+            })
+            .getData();
+
+        logger.info(`Events: ${response.instances.length}`);
+
+        return response.instances;
+    }
+}
+
+const params = {
+    fields: {
+        $all: true,
+    },
+} as const;

src/scripts/commands/events.ts

@@ -18,6 +18,7 @@ import { Id } from "domain/entities/Base";
 import { promiseMap } from "data/dhis2-utils";
 import { DetectExternalOrgUnitUseCase } from "domain/usecases/ProcessEventsOutsideEnrollmentOrgUnitUseCase";
 import { ProgramsD2Repository } from "data/ProgramsD2Repository";
+import { RecodeBooleanDataValuesInEventsUseCase } from "domain/usecases/RecodeBooleanDataValuesInEventsUseCase";
 
 export function getCommand() {
     return subcommands({
@@ -26,6 +27,7 @@ export function getCommand() {
             "move-to-org-unit": moveOrgUnitCmd,
             "update-events": updateEventsDataValues,
             "detect-external-orgunits": detectExternalOrgUnitCmd,
+            "recode-boolean-data-values": recodeBooleanDataValues,
         },
     });
 }
@@ -144,3 +146,34 @@ const updateEventsDataValues = command({
         }
     },
 });
+
+const recodeBooleanDataValues = command({
+    name: "recode-boolean-data-values",
+    description: "Recode boolean data values",
+    args: {
+        ...getApiUrlOptions(),
+        programId: option({
+            type: string,
+            long: "program-id",
+            description: "Program ID to recode",
+        }),
+        ternaryOptionSetId: option({
+            type: string,
+            long: "ternary-optionset-id",
+            description: "ID of the ternary option set (Yes/No/NA) to recode",
+        }),
+        post: flag({
+            long: "post",
+            description: "Fix events",
+            defaultValue: () => false,
+        }),
+    },
+    handler: async args => {
+        const api = getD2ApiFromArgs(args);
+        const eventsRepository = new ProgramEventsD2Repository(api);
+        const programsRepository = new ProgramsD2Repository(api);
+        return new RecodeBooleanDataValuesInEventsUseCase(api, programsRepository, eventsRepository).execute(
+            args
+        );
+    },
+});

src/utils/log.ts

@@ -3,6 +3,7 @@ import { isElementOfUnion, UnionFromValues } from "./ts-utils";
 const logLevels = ["debug", "info", "warn", "error"] as const;
 export type LogLevel = UnionFromValues<typeof logLevels>;
 
+// data layer -> domain layer
 const levelFromEnv = process.env["LOG_LEVEL"] || "";
 const level = isElementOfUnion(levelFromEnv, logLevels) ? levelFromEnv : "info";
 const levelIndex = logLevels.indexOf(level);