Merge remote-tracking branch 'origin/development' into feature/improv…

…e-get-duplicated-events-8695c239r

README.md

@@ -406,10 +406,6 @@ yarn start users migrate \
 #### Execution:
 
 ```
-yarn install
-
-yarn build
-
 yarn start usermonitoring run-permissions-fixer --config-file config.json
 or
 yarn start usermonitoring run-2fa-reporter --config-file config.json
@@ -418,10 +414,6 @@ yarn start usermonitoring run-2fa-reporter --config-file config.json
 #### Debug:
 
 ```
-yarn install
-
-yarn build:dev
-
 LOG_LEVEL=debug node --inspect-brk dist/index.js usermonitoring run-users-monitoring   --config-file config.json
 LOG_LEVEL=debug node --inspect-brk dist/index.js usermonitoring run-2fa-reporter   --config-file config.json
 ```
@@ -594,10 +586,6 @@ Note: the names are used only to make easy understand and debug the keys.
 #### Execution:
 
 ```bash
-yarn install
-
-yarn build
-
 yarn start usermonitoring run-authorities-monitoring --config-file config.json
 
 # To get the debug logs and store them in a file use:
@@ -629,7 +617,7 @@ A config file with the access info of the server and the message webhook details
 ```
 
 This reports stores data into the `d2-tools.authorities-monitor` datastore. This key needs to be setup before the first run to get a correct report.
-Its possible to leave `usersByAuthority` empty and use the `-s` flag to populate it.
+It's possible to leave `usersByAuthority` empty and use the `-s` flag to populate it.
 
 A sample:
 
@@ -674,6 +662,105 @@ A sample:
 }
 ```
 
+### User Groups Monitoring
+
+This script will compare the metadata of the monitored userGroups with the version stored in the datastore and generate a report of the changes. This report will be sent to the MS Teams channel set in the webhook config section. Then the new version of the metadata will be stored in the datastore.
+
+#### Execution:
+
+```bash
+yarn start usermonitoring run-user-groups-monitoring --config-file config.json
+
+# To get the debug logs and store them in a file use:
+LOG_LEVEL=debug yarn start usermonitoring run-user-groups-monitoring --config-file config.json &> user-groups-monitoring.log
+```
+
+#### Parameters:
+
+-   `--config-file`: Connection and webhook config file.
+-   `-s` | `--set-datastore`: Write usergroups data to datastore, use in script setup. It assumes there is a monitoring config in d2-tools/user-groups-monitoring.
+
+#### Requirements:
+
+A config file with the access info of the server and the message webhook details:
+
+```JSON
+{
+    "URL": {
+        "username": "user",
+        "password": "passwd",
+        "server": "https://dhis.url/"
+    },
+    "WEBHOOK": {
+        "ms_url": "http://webhook.url/",
+        "proxy": "http://proxy.url/",
+        "server_name": "INSTANCE_NAME"
+    }
+}
+```
+
+This reports stores data into the `d2-tools.user-groups-monitoring` datastore. This key needs to be setup before the first run to get a correct report.
+Its possible to leave `monitoredUserGroups` empty and use the `-s` flag to populate it.
+
+The report, potentially, has tree sections for each user group:
+
+-   New entries: JSON with the properties that were unset or empty and changed.
+-   Modified fields: This section has two JSONs, one showing the old values and one with the news.
+-   User assignment changes: This section will show the users lost and added to the group.
+
+If a section is empty it will be omitted.
+
+### User Templates Monitoring
+
+The User Templates Monitoring script is used to compare user templates with the version stored in the datastore and generate a report of the changes. The report includes information on modified fields, and a detailed report on user groups and roles added or lost. This report will be sent to the MS Teams channel set in the webhook config section. The new version of the metadata will be stored in the datastore.
+
+#### Execution:
+
+```bash
+yarn start usermonitoring run-user-templates-monitoring --config-file config.json
+
+# To get the debug logs and store them in a file use:
+LOG_LEVEL=debug yarn start usermonitoring run-user-templates-monitoring --config-file config.json &> user-templates-monitoring.log
+```
+
+#### Parameters:
+
+-   `--config-file`: Connection and webhook config file.
+-   `-s` | `--set-datastore`: Write user templates data to datastore, use in script setup. It assumes there is a monitoring config in d2-tools/user-templates-monitoring.
+
+#### Requirements:
+
+A config file with the access info of the server and the message webhook details:
+
+```JSON
+{
+    "URL": {
+        "username": "user",
+        "password": "passwd",
+        "server": "https://dhis.url/"
+    },
+    "WEBHOOK": {
+        "ms_url": "http://webhook.url/",
+        "proxy": "http://proxy.url/",
+        "server_name": "INSTANCE_NAME"
+    }
+}
+```
+
+#### Report
+
+This reports stores data into the `d2-tools.user-templates-monitoring` datastore. This key needs to be setup before the first run to get a correct report.
+
+Its possible to leave `monitoredUserTemplates` empty and use the `-s` flag to populate it.
+
+The report includes the following sections:
+
+-   New entries: This section lists new properties that didn't exist in the old version.
+-   Modified fields: This section shows the fields that have been modified in the user templates and shows the before/after values.
+-   User Membership changes: This section displays the changes in the template userGroups and userRoles membership.
+
+If a section is empty, it will be omitted from the report.
+
 ## Move Attributes from a Program
 
 Get all the TEIS in the program and move the value from the attribute in the argument `--from-attribute-id` to the attribute `--to-attribute-id`. Then delete the value in `--from-attribute-id`.
@@ -763,3 +850,17 @@ File option can be a file or directory path, if its a directory path the file wi
 CSV headers:
 
 dataElement ID | dataElement Name | categoryOptionCombo ID | categoryOptionCombo Name | Value
+
+## Tracked Entities
+
+### Transfer
+
+Transfer tracked entities to another org unit, using a CSV as source data (expected columns: trackedEntityId, newOrgUnitId):
+
+```console
+shell:~$ yarn start trackedEntities transfer \
+  --url=http://localhost:8080 \
+  --auth="USER:PASSWORD"  \
+  --input-file=transfers.csv \
+  --post
+```

package.json

@@ -19,7 +19,7 @@
     "author": "EyeSeeTea <arnau@eyeseetea.com>",
     "license": "MIT",
     "dependencies": {
-        "@eyeseetea/d2-api": "1.16.0-beta.2",
+        "@eyeseetea/d2-api": "1.16.0-beta.3",
         "@types/json-diff": "^0.7.0",
         "async-parallel": "^1.2.3",
         "cmd-ts": "^0.10.0",
@@ -28,6 +28,7 @@
         "d2-utilizr": "^0.2.16",
         "file-system-cache": "^2.0.0",
         "json-diff": "^0.7.3",
+        "jsonfile": "^6.1.0",
         "lodash": "^4.17.21",
         "loglevel": "^1.8.0",
         "luxon": "3.4.2",

src/data/CategoryOptionD2Repository.ts

@@ -16,7 +16,7 @@ export class CategoryOptionD2Repository implements CategoryOptionRepository {
 
     async saveAll(categoryOptions: CategoryOption[]): Async<Stats> {
         const catOptionsIdsToSave = categoryOptions.map(getId);
-        const stats = await getInChunks<Stats>(catOptionsIdsToSave, async catOptionsIds => {
+        const stats = await getInChunks<string, Stats>(catOptionsIdsToSave, async catOptionsIds => {
             const response = await this.api.metadata
                 .get({
                     categoryOptions: {

src/data/ProgramEventsD2Repository.ts

@@ -174,7 +174,7 @@ export class ProgramEventsD2Repository implements ProgramEventsRepository {
             .keyBy(event => event.id)
             .value();
 
-        const resultsList = await getInChunks<Result>(eventsIdsToSave, async eventIds => {
+        const resultsList = await getInChunks<Id, Result>(eventsIdsToSave, async eventIds => {
             return this.getEvents(eventIds)
                 .then(res => {
                     const postEvents = eventIds.map((eventId): EventToPost => {

src/data/TrackedEntityD2Repository.ts

@@ -1,16 +1,18 @@
 import _ from "lodash";
 import { D2Api } from "types/d2-api";
 import { Async } from "domain/entities/Async";
+import logger from "utils/log";
 
 import { getInChunks } from "./dhis2-utils";
 import { Stats } from "domain/entities/Stats";
 import {
     TrackedEntityFilterParams,
     TrackedEntityRepository,
 } from "domain/repositories/TrackedEntityRepository";
-import { TrackedEntity } from "domain/entities/TrackedEntity";
+import { TrackedEntity, TrackedEntityTransfer } from "domain/entities/TrackedEntity";
 import { D2TrackedEntity } from "./ProgramsD2Repository";
 import { D2Tracker } from "./D2Tracker";
+import { TrackedEntityInstance } from "@eyeseetea/d2-api/api/trackedEntityInstances";
 
 export class TrackedEntityD2Repository implements TrackedEntityRepository {
     private d2Tracker: D2Tracker;
@@ -54,7 +56,7 @@ export class TrackedEntityD2Repository implements TrackedEntityRepository {
             .keyBy(tei => tei.id)
             .value();
 
-        const stats = await getInChunks<Stats>(teisToFetch, async teiIds => {
+        const stats = await getInChunks<string, Stats>(teisToFetch, async teiIds => {
             const trackedEntities = await this.d2Tracker.getFromTracker<D2TrackedEntity>("trackedEntities", {
                 orgUnitIds: undefined,
                 programIds: programsIds,
@@ -94,4 +96,95 @@ export class TrackedEntityD2Repository implements TrackedEntityRepository {
 
         return Stats.combine(stats);
     }
+
+    async transfer(transfers: TrackedEntityTransfer[], options: { post: boolean }): Async<void> {
+        const teis = await this.getTeisFromTransfers(transfers);
+        const teisWithChanges = this.getTeisWithChanges(transfers, teis);
+        this.transferTeis(teisWithChanges, options);
+    }
+
+    private async transferTeis(teis: TrackedEntityInstance[], options: { post: boolean }) {
+        const { api } = this;
+
+        if (!options.post) {
+            logger.info(`Add --post to update tracked entities`);
+            return;
+        }
+
+        logger.info(`Transfer ownership: ${teis.length}`);
+
+        for (const tei of teis) {
+            await this.transferTei(tei);
+        }
+
+        await this.postTeis(teis, api);
+    }
+
+    private async transferTei(tei: TrackedEntityInstance) {
+        const programId = tei.enrollments[0]?.program;
+
+        if (!programId) {
+            logger.warn(`Tei without enrollments: ${tei.trackedEntityInstance}`);
+            return;
+        }
+
+        logger.debug(`Transfer ownership: trackedEntity.id=${tei.trackedEntityInstance}`);
+
+        const res = await this.api
+            .put<{ status: string }>(
+                `tracker/ownership/transfer?trackedEntityInstance=${tei.trackedEntityInstance}&ou=${tei.orgUnit}&program=${programId}`
+            )
+            .getData();
+
+        if (res.status !== "OK") {
+            logger.error(`Transfer ownership failed: ${JSON.stringify(res)}`);
+        }
+    }
+
+    private async postTeis(teis: TrackedEntityInstance[], api: D2Api) {
+        logger.info(`Import trackedEntities: ${teis.length}`);
+        const res = await api.trackedEntityInstances.post({}, { trackedEntityInstances: teis }).getData();
+        logger.info(`Tracked entities updated: ${res.status} - updated=${res.updated}`);
+    }
+
+    private getTeisWithChanges(transfers: TrackedEntityTransfer[], teis: TrackedEntityInstance[]) {
+        const rowsByTeiId = _.keyBy(transfers, row => row.trackedEntityId);
+
+        const teis2 = _(teis)
+            .sortBy(tei => tei.trackedEntityInstance)
+            .map(tei => {
+                const row = rowsByTeiId[tei.trackedEntityInstance];
+                if (!row) throw new Error("internal");
+                const orgUnitId = row.newOrgUnitId;
+                if (!orgUnitId) throw new Error("internal");
+                const hasChanges = tei.orgUnit !== orgUnitId;
+
+                return hasChanges ? { ...tei, orgUnit: orgUnitId } : undefined;
+            })
+            .compact()
+            .value();
+
+        logger.info(`trackedEntities that need to be transfered: ${teis2.length}`);
+        return teis2;
+    }
+
+    private async getTeisFromTransfers(transfers: TrackedEntityTransfer[]) {
+        logger.debug(`Get trackedEntities: ${transfers.length}`);
+
+        const teis = await getInChunks(transfers, async rowsChunk => {
+            const res = await this.api.trackedEntityInstances
+                .getAll({
+                    totalPages: true,
+                    ouMode: "ALL",
+                    trackedEntityInstance: rowsChunk.map(row => row.trackedEntityId).join(";"),
+                })
+                .getData();
+
+            return res.trackedEntityInstances;
+        });
+
+        logger.debug(`trackedEntities from server: ${teis.length}`);
+
+        return teis;
+    }
 }

src/data/UserD2Repository.ts

@@ -70,7 +70,7 @@ export class UserD2Repository implements UserRepository {
 
     async saveAll(users: User[]): Async<Stats> {
         const userToSaveIds = users.map(user => user.id);
-        const stats = await getInChunks<Stats>(userToSaveIds, async userIds => {
+        const stats = await getInChunks<string, Stats>(userToSaveIds, async userIds => {
             return this.api.metadata
                 .get({
                     users: {

src/data/d2-program-rules/D2ProgramRules.ts

@@ -235,7 +235,7 @@ export class D2ProgramRules {
         const res = await this.api.events
             .post(postOptions, { events })
             .getData()
-            .catch(err => err.response.data as EventsPostResponse);
+            .catch(err => err.data as EventsPostResponse);
 
         log.info(`POST events: ${res.status}`);
         return checkPostEventsResponse(res);
@@ -249,7 +249,7 @@ export class D2ProgramRules {
         const res = await this.api.trackedEntityInstances
             .post(postOptions, { trackedEntityInstances: teis })
             .getData()
-            .catch(err => err.response.data as HttpResponse<TeiPostResponse>);
+            .catch(err => err.data as HttpResponse<TeiPostResponse>);
 
         log.info(`POST TEIs: ${res.status}`);
 

src/data/dhis2-utils.ts

@@ -1,6 +1,5 @@
 import { EventsPostResponse } from "@eyeseetea/d2-api/api/events";
 import { CancelableResponse } from "@eyeseetea/d2-api/repositories/CancelableResponse";
-import { Id } from "domain/entities/Base";
 import _ from "lodash";
 import { MetadataResponse } from "../types/d2-api";
 import log from "utils/log";
@@ -47,7 +46,7 @@ export function checkPostEventsResponse(res: EventsPostResponse): void {
     }
 }
 
-export async function getInChunks<T>(ids: Id[], getter: (idsGroup: Id[]) => Promise<T[]>): Promise<T[]> {
+export async function getInChunks<T, U>(ids: T[], getter: (idsGroup: T[]) => Promise<U[]>): Promise<U[]> {
     const objsCollection = await promiseMap(_.chunk(ids, 300), idsGroup => getter(idsGroup));
     return _.flatten(objsCollection);
 }