Skip to content

Commit

Permalink
temporal
Browse files Browse the repository at this point in the history
  • Loading branch information
@tokland
tokland committed Aug 9, 2024
1 parent 3a31eca commit bf7812f
Show file tree
Hide file tree
Showing 12 changed files with 232 additions and 97 deletions.
4 changes: 2 additions & 2 deletions src/data/EventExportSpreadsheetRepository.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,10 @@ export class EventExportSpreadsheetRepository implements EventExportRepository {
const csvData = _(events)
.flatMap(event => {
return event.dataValues
.filter(dv => dv.dataElementId === options.dataElementId)
.filter(dv => dv.dataElement.id === options.dataElementId)
.map(dv => ({
event: event.id,
dataElement: dv.dataElementId,
dataElement: dv.dataElement.id,
oldValue: dv.oldValue,
value: options.newValue,
}));
Expand Down
138 changes: 109 additions & 29 deletions src/data/ProgramEventsD2Repository.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { EventsGetResponse, PaginatedEventsGetResponse } from "@eyeseetea/d2-api
import { Async } from "domain/entities/Async";
import { ProgramEvent } from "domain/entities/ProgramEvent";
import { GetOptions, ProgramEventsRepository } from "domain/repositories/ProgramEventsRepository";
import { D2Api, EventsPostRequest, EventsPostParams, Ref } from "types/d2-api";
import { D2Api, EventsPostRequest, EventsPostParams, Ref, SelectedPick, D2ProgramSchema } from "types/d2-api";
import { cartesianProduct } from "utils/array";
import logger from "utils/log";
import { getId, Id } from "domain/entities/Base";
Expand Down Expand Up @@ -37,6 +37,25 @@ type Fields = typeof eventFields;

type Event = EventsGetResponse<Fields>["events"][number];

type DataValue = ProgramEvent["dataValues"][number];

type D2DataValue = Event["dataValues"][number];

const programFields = {
id: true,
name: true,
programType: true,
programStages: {
id: true,
name: true,
programStageSections: { dataElements: { id: true } },
},
} as const;

type D2Program = SelectedPick<D2ProgramSchema, typeof programFields>;

type D2ProgramStage = D2Program["programStages"][number];

export class ProgramEventsD2Repository implements ProgramEventsRepository {
constructor(private api: D2Api) {}

Expand All @@ -46,11 +65,7 @@ export class ProgramEventsD2Repository implements ProgramEventsRepository {
const { programs } = await this.api.metadata
.get({
programs: {
fields: {
id: true,
name: true,
programStages: { id: true, name: true },
},
fields: programFields,
},
})
.getData();
Expand All @@ -59,28 +74,93 @@ export class ProgramEventsD2Repository implements ProgramEventsRepository {

const programStagesById = _(programs)
.flatMap(program => program.programStages)
.uniqBy(getId)
.keyBy(getId)
.value();

return d2Events.map(event => ({
created: event.created,
id: event.event,
program: programsById[event.program] || { id: event.program, name: "" },
programStage: programStagesById[event.programStage] || { id: event.programStage, name: "" },
orgUnit: { id: event.orgUnit, name: event.orgUnitName },
trackedEntityInstanceId: (event as D2Event).trackedEntityInstance,
status: event.status,
date: event.eventDate,
dueDate: event.dueDate,
dataValues: event.dataValues.map(dv => ({
dataElementId: dv.dataElement,
value: dv.value,
storedBy: dv.storedBy,
providedElsewhere: dv.providedElsewhere,
lastUpdated: dv.lastUpdated,
})),
}));
const dataElementsById = await this.getDataElementsById(d2Events);

return d2Events.map((event): ProgramEvent => {
const program = programsById[event.program];
if (!program) throw new Error(`Cannot find program ${event.program}`);

return {
created: event.created,
id: event.event,
program: {
id: event.program,
name: program.name,
type: program.programType === "WITH_REGISTRATION" ? "tracker" : "event",
},
programStage: {
id: event.programStage,
name: programStagesById[event.programStage]?.name || "",
},
orgUnit: { id: event.orgUnit, name: event.orgUnitName },
trackedEntityInstanceId: (event as D2Event).trackedEntityInstance,
lastUpdated: event.lastUpdated,
status: event.status,
date: event.eventDate,
dueDate: event.dueDate,
dataValues: this.getDataValuesOrderLikeDataEntryForm(event, programStagesById).map(
(dv): DataValue => {
const dataElement = dataElementsById[dv.dataElement];
if (!dataElement) throw new Error(`Cannot find data element ${dv.dataElement}`);

return {
dataElement: {
id: dv.dataElement,
name: dataElement.formName || dataElement.name,
},
value: dv.value,
storedBy: dv.storedBy,
providedElsewhere: dv.providedElsewhere,
lastUpdated: dv.lastUpdated,
};
}
),
};
});
}

private getDataValuesOrderLikeDataEntryForm(
event: Event,
programStagesById: Record<Id, D2ProgramStage>
): D2DataValue[] {
const programStage = programStagesById[event.programStage];
if (!programStage) throw new Error(`Cannot find program stage ${event.programStage}`);

const indexMapping = _(programStage.programStageSections)
.flatMap(pse => pse.dataElements)
.map((dataElement, index) => [dataElement.id, index] as [Id, number])
.fromPairs()
.value();

return _(event.dataValues)
.sortBy(dv => indexMapping[dv.dataElement] ?? 1000)
.value();
}

private async getDataElementsById(d2Events: Event[]) {
const dataElementIds = _(d2Events)
.flatMap(ev => ev.dataValues)
.map(dv => dv.dataElement)
.uniq()
.value();

const dataElements = await getInChunks(dataElementIds, async dataElementIdsGroup => {
const { dataElements } = await this.api.metadata
.get({
dataElements: {
fields: { id: true, name: true, formName: true },
filter: { id: { in: dataElementIdsGroup } },
},
})
.getData();

return dataElements;
});

return _.keyBy(dataElements, getId);
}

async delete(events: Ref[]): Async<Result> {
Expand Down Expand Up @@ -114,7 +194,7 @@ export class ProgramEventsD2Repository implements ProgramEventsRepository {
eventDate: event.date,
dataValues: event.dataValues.map(dv => {
return {
dataElement: dv.dataElementId,
dataElement: dv.dataElement.id,
value: dv.value,
storedBy: dv.storedBy,
providedElsewhere: dv.providedElsewhere,
Expand Down Expand Up @@ -228,14 +308,14 @@ async function importEvents(api: D2Api, events: EventToPost[], params?: EventsPo

const resList = await promiseMap(_.chunk(events, 100), async eventsGroup => {
const res = await api.events.post(params || {}, { events: eventsGroup }).getData();
if (res.response.status === "SUCCESS") {
if (res.status === "SUCCESS") {
const message = JSON.stringify(
_.pick(res.response, ["status", "imported", "updated", "deleted", "ignored"])
_.pick(res, ["status", "imported", "updated", "deleted", "ignored"])
);
logger.info(`Post events OK: ${message}`);
return true;
} else {
const message = JSON.stringify(res.response, null, 4);
const message = JSON.stringify(res, null, 4);
logger.info(`Post events ERROR: ${message}`);
return false;
}
Expand Down
2 changes: 1 addition & 1 deletion src/data/ProgramEventsExportCsvRepository.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ export class ProgramEventsExportCsvRepository implements ProgramEventsExportRepo
programStage: formatObj(event.programStage),
orgUnit: formatObj(event.orgUnit),
teiId: event.trackedEntityInstanceId || "-",
dataValues: event.dataValues.map(dv => `${dv.dataElementId}=${dv.value}`).join(", "),
dataValues: event.dataValues.map(dv => `${dv.dataElement.id}=${dv.value}`).join(", "),
created: event.created,
date: event.date,
status: event.status,
Expand Down
9 changes: 4 additions & 5 deletions src/data/d2-program-rules/D2ProgramRules.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,9 +235,9 @@ export class D2ProgramRules {
const res = await this.api.events
.post(postOptions, { events })
.getData()
.catch(err => err.response.data as HttpResponse<EventsPostResponse>);
.catch(err => err.response.data as EventsPostResponse);

log.info(`POST events: ${res.response.status}`);
log.info(`POST events: ${res.status}`);
return checkPostEventsResponse(res);
}

Expand All @@ -251,10 +251,9 @@ export class D2ProgramRules {
.getData()
.catch(err => err.response.data as HttpResponse<TeiPostResponse>);

log.info(`POST TEIs: ${res.response.status}`);
log.info(`POST TEIs: ${res.status}`);

if (res.response.status !== "SUCCESS")
log.error(JSON.stringify(res.response.importSummaries, null, 4));
if (res.status !== "SUCCESS") log.error(JSON.stringify(res, null, 4));
}

private async saveReport(reportPath: string, actions: UpdateAction[]) {
Expand Down
9 changes: 4 additions & 5 deletions src/data/dhis2-utils.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import { HttpResponse } from "@eyeseetea/d2-api/api/common";
import { EventsPostResponse } from "@eyeseetea/d2-api/api/events";
import { CancelableResponse } from "@eyeseetea/d2-api/repositories/CancelableResponse";
import { Id } from "domain/entities/Base";
Expand Down Expand Up @@ -29,8 +28,8 @@ export function getData<T>(d2Response: CancelableResponse<T>): Promise<T> {
return d2Response.getData();
}

export function checkPostEventsResponse(res: HttpResponse<EventsPostResponse>): void {
const importMessages = _(res.response.importSummaries || [])
export function checkPostEventsResponse(res: EventsPostResponse): void {
const importMessages = _(res.importSummaries || [])
.map(importSummary =>
importSummary.status !== "SUCCESS"
? _.compact([
Expand All @@ -42,8 +41,8 @@ export function checkPostEventsResponse(res: HttpResponse<EventsPostResponse>):
.compact()
.value();

if (res.status !== "OK") {
const msg = [`POST /events error`, res.message, ...importMessages].join("\n") || "Unknown error";
if (res.status !== "SUCCESS") {
const msg = [`POST /events error`, ...importMessages].join("\n") || "Unknown error";
log.error(msg);
}
}
Expand Down
2 changes: 1 addition & 1 deletion src/data/user-monitoring/common/UserMonitoringFileResourceUtils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ export class UserMonitoringFileResourceUtils {
private static formatDate(date: Date): string {
return date
.toLocaleString()
.replace(/[ :\/\\,-]/g, "_")
.replace(/[ :/\\,-]/g, "_")
.replace("\\", "_");
}

Expand Down
39 changes: 20 additions & 19 deletions src/domain/entities/ProgramEvent.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,13 @@ import { Maybe } from "utils/ts-utils";

export interface ProgramEvent {
id: Id;
program: NamedRef;
program: NamedRef & { type: "event" | "tracker" };
orgUnit: NamedRef;
programStage: NamedRef;
dataValues: EventDataValue[];
trackedEntityInstanceId?: Id;
created: Timestamp;
lastUpdated: Timestamp;
status: EventStatus;
date: Timestamp;
dueDate: Timestamp;
Expand All @@ -21,7 +22,7 @@ export interface ProgramEventToSave {
program: Ref;
orgUnit: Ref;
programStage: Ref;
dataValues: EventDataValue[];
dataValues: Array<{ dataElement: Ref; value: string }>;
trackedEntityInstanceId?: Id;
status: EventStatus;
date: Timestamp;
Expand All @@ -35,21 +36,25 @@ export const orgUnitModes = ["SELECTED", "CHILDREN", "DESCENDANTS"] as const;
export type OrgUnitMode = typeof orgUnitModes[number];

export interface EventDataValue {
dataElementId: Id;
dataElement: NamedRef;
value: string;
storedBy: Username;
oldValue?: string;
providedElsewhere?: boolean;
lastUpdated: Timestamp;
}

export type DuplicatedEvents = { groups: EventsGroup[] };

export type EventsGroup = { events: ProgramEvent[] };

export class DuplicatedProgramEvents {
constructor(
private options: { ignoreDataElementsIds: Maybe<Id[]>; checkDataElementsIds?: Maybe<Id[]> }
) {}

get(events: ProgramEvent[]): ProgramEvent[] {
return _(events)
get(events: ProgramEvent[]): DuplicatedEvents {
const groups = _(events)
.groupBy(event =>
_.compact([
event.orgUnit.id,
Expand All @@ -61,32 +66,28 @@ export class DuplicatedProgramEvents {
.values()
.flatMap(events => this.getDuplicatedEventsForGroup(events))
.value();
}

private getDuplicatedEventsForGroup(eventsGroup: ProgramEvent[]): ProgramEvent[] {
const excludeOldestEvent = (events: ProgramEvent[]) =>
_(events)
.sortBy(ev => ev.created)
.drop(1)
.value();
return { groups: groups };
}

private getDuplicatedEventsForGroup(eventsGroup: ProgramEvent[]): EventsGroup[] {
return _(eventsGroup)
.groupBy(event => this.getEventDataValuesUid(event))
.groupBy(event => this.getEventDataValuesHash(event))
.values()
.filter(events => events.length > 1)
.flatMap(excludeOldestEvent)
.map(events => ({ events: _.sortBy(events, ev => ev.created) }))
.compact()
.value();
}

private getEventDataValuesUid(event: ProgramEvent) {
private getEventDataValuesHash(event: ProgramEvent) {
const { ignoreDataElementsIds, checkDataElementsIds } = this.options;

return _(event.dataValues)
.sortBy(dv => dv.dataElementId)
.filter(dv => (checkDataElementsIds ? checkDataElementsIds.includes(dv.dataElementId) : true))
.reject(dv => (ignoreDataElementsIds ? ignoreDataElementsIds.includes(dv.dataElementId) : false))
.map(dv => [dv.dataElementId, dv.value].join("."))
.sortBy(dv => dv.dataElement.id)
.filter(dv => (checkDataElementsIds ? checkDataElementsIds.includes(dv.dataElement.id) : true))
.reject(dv => (ignoreDataElementsIds ? ignoreDataElementsIds.includes(dv.dataElement.id) : false))
.map(dv => [dv.dataElement.id, dv.value].join("="))
.join();
}
}
2 changes: 1 addition & 1 deletion src/domain/usecases/DeleteProgramDataValuesUseCase.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ export class DeleteProgramDataValuesUseCase {

return events.map((event): typeof event => {
const dataValuesUpdated = event.dataValues.map((dataValue): typeof dataValue => {
return dataElementMatches(dataValue.dataElementId) ? { ...dataValue, value: "" } : dataValue;
return dataElementMatches(dataValue.dataElement.id) ? { ...dataValue, value: "" } : dataValue;
});

return { ...event, dataValues: dataValuesUpdated };
Expand Down
Loading

0 comments on commit bf7812f

Please sign in to comment.