diff --git a/packages/data-models/flowTypes.ts b/packages/data-models/flowTypes.ts index 51f8f6fabf..e7b0a95a96 100644 --- a/packages/data-models/flowTypes.ts +++ b/packages/data-models/flowTypes.ts @@ -432,6 +432,7 @@ export namespace FlowTypes { "pop_up", "process_template", "reset_app", + "reset_data", "save_to_device", "screen_orientation", "set_field", diff --git a/packages/shared/src/utils/object-utils.ts b/packages/shared/src/utils/object-utils.ts index 11a2d0a9c4..2174bb77d4 100644 --- a/packages/shared/src/utils/object-utils.ts +++ b/packages/shared/src/utils/object-utils.ts @@ -100,8 +100,7 @@ export function isEqual(a: any, b: any) { if (!isEqual(Object.keys(aSorted), Object.keys(bSorted))) return false; return isEqual(Object.values(aSorted), Object.values(bSorted)); } - // could not compare (e.g. symbols, date objects, functions, buffers etc) - console.warn(`[isEqual] could not compare`, a, b); + // NOTE - does not compare symbols, date objects, functions, buffers etc. return false; } diff --git a/src/app/shared/components/template/services/instance/template-action.registry.ts b/src/app/shared/components/template/services/instance/template-action.registry.ts index 0369129867..a97c3e3e64 100644 --- a/src/app/shared/components/template/services/instance/template-action.registry.ts +++ b/src/app/shared/components/template/services/instance/template-action.registry.ts @@ -3,10 +3,8 @@ import clone from "clone"; import { FlowTypes } from "data-models"; export type IActionId = FlowTypes.TemplateRowAction["action_id"]; -export type IActionHandlers = Record< - IActionId, - (action: FlowTypes.TemplateRowAction) => Promise ->; +export type IActionHandler = (action: FlowTypes.TemplateRowAction) => Promise; +export type IActionHandlers = Record; @Injectable({ providedIn: "root" }) /** diff --git a/src/app/shared/components/template/services/template-field.service.spec.ts b/src/app/shared/components/template/services/template-field.service.spec.ts index dcfd73f763..8006345abe 100644 --- a/src/app/shared/components/template/services/template-field.service.spec.ts +++ b/src/app/shared/components/template/services/template-field.service.spec.ts @@ -4,7 +4,7 @@ import { TemplateFieldService } from "./template-field.service"; import type { PromiseExtended } from "dexie"; import { booleanStringToBoolean } from "src/app/shared/utils"; import { ErrorHandlerService } from "src/app/shared/services/error-handler/error-handler.service"; -import { MockErrorHandlerService } from "src/app/shared/services/error-handler/error-handler.service.spec"; +import { MockErrorHandlerService } from "src/app/shared/services/error-handler/error-handler.service.mock.spec"; /** Mock calls for field values from the template field service to return test data */ export class MockTemplateFieldService implements Partial { diff --git a/src/app/shared/services/data/app-data.service.spec.ts b/src/app/shared/services/data/app-data.service.spec.ts index 8394190d5d..2ea55d24cf 100644 --- a/src/app/shared/services/data/app-data.service.spec.ts +++ b/src/app/shared/services/data/app-data.service.spec.ts @@ -7,7 +7,7 @@ import { AppDataService, IAppDataCache } from "./app-data.service"; import { FlowTypes } from "../../model"; import { MockAppDataVariableService } from "./app-data-variable.service.spec"; import { ErrorHandlerService } from "../error-handler/error-handler.service"; -import { MockErrorHandlerService } from "../error-handler/error-handler.service.spec"; +import { MockErrorHandlerService } from "../error-handler/error-handler.service.mock.spec"; import { DbService } from "../db/db.service"; import { MockDbService } from "../db/db.service.spec"; import { Injectable } from "@angular/core"; diff --git a/src/app/shared/services/dynamic-data/adapters/persistedMemory.ts b/src/app/shared/services/dynamic-data/adapters/persistedMemory.ts index 4378dd7a81..523fdfa694 100644 --- a/src/app/shared/services/dynamic-data/adapters/persistedMemory.ts +++ b/src/app/shared/services/dynamic-data/adapters/persistedMemory.ts @@ -135,7 +135,7 @@ export class PersistedMemoryAdapter { this.persistStateToDB(); } - public async delete(flow_type: FlowTypes.FlowType, flow_name: string) { + public delete(flow_type: FlowTypes.FlowType, flow_name: string) { if (this.get(flow_type, flow_name)) { delete this.state[flow_type][flow_name]; this.persistStateToDB(); diff --git a/src/app/shared/services/dynamic-data/dynamic-data.actions.spec.ts b/src/app/shared/services/dynamic-data/dynamic-data.actions.spec.ts new file mode 100644 index 0000000000..cbc797763f --- /dev/null +++ b/src/app/shared/services/dynamic-data/dynamic-data.actions.spec.ts @@ -0,0 +1,200 @@ +import { TestBed } from "@angular/core/testing"; + +import { HttpClientTestingModule } from "@angular/common/http/testing"; +import { MockAppDataService } from "../data/app-data.service.spec"; +import { AppDataService } from "../data/app-data.service"; + +import { IActionSetDataParams } from "./dynamic-data.actions"; +import { DynamicDataService } from "./dynamic-data.service"; +import ActionFactory from "./dynamic-data.actions"; +import { firstValueFrom } from "rxjs"; +import { FlowTypes } from "packages/data-models"; +import { DeploymentService } from "../deployment/deployment.service"; +import { MockDeploymentService } from "../deployment/deployment.service.spec"; +import { TemplateActionRegistry } from "../../components/template/services/instance/template-action.registry"; + +const TEST_DATA_ROWS = [ + { id: "id_0", number: 0, string: "hello", _meta: "original" }, + { id: "id_1", number: 1, string: "hello", boolean: true, _meta: "original" }, +]; + +/******************************************************************************** + * Test Utilities + *******************************************************************************/ + +/** Generate a rows to trigger set_data action with included params */ +function getTestActionRow(params: IActionSetDataParams) { + params._list_id = "test_flow"; + const actionRow: FlowTypes.TemplateRowAction = { + action_id: "set_data", + trigger: "click", + args: [], + params, + }; + return actionRow; +} + +/** + * Trigger the set_data action with included params and return first update + * to corresponding data_list + * * */ +async function triggerTestSetDataAction(service: DynamicDataService, params: IActionSetDataParams) { + const actionRow = getTestActionRow(params); + const actions = new ActionFactory(service); + await actions.set_data(actionRow); + const obs = await service.query$("data_list", "test_flow"); + const data = await firstValueFrom(obs); + return data; +} + +/******************************************************************************** + * Tests + * yarn ng test --include src/app/shared/services/dynamic-data/dynamic-data.actions.spec.ts + *******************************************************************************/ +describe("DynamicDataService Actions", () => { + let service: DynamicDataService; + let actions: ActionFactory; + + beforeEach(async () => { + TestBed.configureTestingModule({ + imports: [HttpClientTestingModule], + providers: [ + DynamicDataService, + { + provide: AppDataService, + useValue: new MockAppDataService({ + data_list: { + test_flow: { + flow_name: "test_flow", + flow_type: "data_list", + // Make deep clone of data to avoid data overwrite issues + rows: JSON.parse(JSON.stringify(TEST_DATA_ROWS)), + }, + }, + }), + }, + { + provide: DeploymentService, + useValue: new MockDeploymentService({ name: "test" }), + }, + { + provide: TemplateActionRegistry, + useValue: { register: () => null }, + }, + ], + }); + + // HACK - polyfill not loaded for rxdb dev plugin so manually fill global before running tests + window.global = window; + + service = TestBed.inject(DynamicDataService); + await service.ready(); + // Ensure any data previously persisted is cleared + await service.resetFlow("data_list", "test_flow"); + actions = new ActionFactory(service); + }); + + /************************************************************* + * Main Tests + ************************************************************/ + it("set_data by _id", async () => { + const params: IActionSetDataParams = { _id: "id_1", string: "updated string" }; + const data = await triggerTestSetDataAction(service, params); + expect(data[0].string).toEqual("hello"); + expect(data[1].string).toEqual("updated string"); + }); + + it("set_data by _index", async () => { + const params: IActionSetDataParams = { _index: 1, string: "updated string" }; + const data = await triggerTestSetDataAction(service, params); + expect(data[0].string).toEqual("hello"); + expect(data[1].string).toEqual("updated string"); + }); + + it("set_data bulk", async () => { + const params: IActionSetDataParams = { string: "updated string" }; + const data = await triggerTestSetDataAction(service, params); + expect(data[0].string).toEqual("updated string"); + expect(data[1].string).toEqual("updated string"); + }); + + it("set_data with item ref (single)", async () => { + const params: IActionSetDataParams = { _id: "id_1", number: "@item.number + 100" }; + const data = await triggerTestSetDataAction(service, params); + expect(data[0].number).toEqual(0); + expect(data[1].number).toEqual(101); + }); + + it("set_data with item ref (bulk)", async () => { + const params: IActionSetDataParams = { number: "@item.number + 100" }; + const data = await triggerTestSetDataAction(service, params); + expect(data[0].number).toEqual(100); + expect(data[1].number).toEqual(101); + }); + + it("set_data ignores updates for unchanged data", async () => { + const params: IActionSetDataParams = { _list_id: "test_flow", number: 1 }; + const updates = await actions["generateUpdateList"](params); + expect(updates).toEqual([{ id: "id_0", number: 1 }]); + }); + + it("set_data prevents update to metadata fields", async () => { + const params: IActionSetDataParams = { _meta: "updated", string: "updated" }; + const data = await triggerTestSetDataAction(service, params); + expect(data[0].string).toEqual("updated"); + expect(data[0]._meta).toEqual("original"); + }); + + it("set_data ignores evaluation when _updates provided", async () => { + const params: IActionSetDataParams = { _updates: [{ id: "id_0", number: "@item.number" }] }; + const data = await triggerTestSetDataAction(service, params); + // test case illustrative only of not parsing data (would have been parsed independently) + expect(data[0].number).toEqual("@item.number"); + expect(data[1].number).toEqual(1); + }); + + it("reset_data action restores data to initial", async () => { + const updatedData = await triggerTestSetDataAction(service, { string: "updated string" }); + expect(updatedData[0].string).toEqual("updated string"); + const resetActionBase = getTestActionRow({}); + await actions.reset_data({ ...resetActionBase, action_id: "reset_data" }); + const obs = await service.query$("data_list", "test_flow"); + const resetData = await firstValueFrom(obs); + expect(resetData[0].string).toEqual("hello"); + }); + + /************************************************************* + * Misc + ************************************************************/ + + it("Coerces string params to correct data type", async () => { + // NOTE - there is no specific code that casts variables, but RXDB will infer from schema + const params: IActionSetDataParams = { number: "300" }; + const data = await triggerTestSetDataAction(service, params); + expect(data[0].number).toEqual(300); + }); + + /************************************************************* + * Quality Control + ************************************************************/ + + it("throws error if provided _id does not exist", async () => { + const params = getTestActionRow({ + _id: "missing_id", + string: "sets an item correctly a given id", + }); + await expectAsync(actions.set_data(params)).toBeRejectedWithError( + `[Update Fail] no doc exists\ndata_list: test_flow\n_id: missing_id` + ); + }); + + it("throws error if provided _index does not exist", async () => { + const params = getTestActionRow({ + _index: 10, + string: "sets an item correctly a given id", + }); + await expectAsync(actions.set_data(params)).toBeRejectedWithError( + `[Update Fail] no doc exists\ndata_list: test_flow\n_index: 10` + ); + }); +}); diff --git a/src/app/shared/services/dynamic-data/dynamic-data.actions.ts b/src/app/shared/services/dynamic-data/dynamic-data.actions.ts new file mode 100644 index 0000000000..3ead2ccf0e --- /dev/null +++ b/src/app/shared/services/dynamic-data/dynamic-data.actions.ts @@ -0,0 +1,141 @@ +import { IActionHandler } from "../../components/template/services/instance/template-action.registry"; +import type { DynamicDataService } from "./dynamic-data.service"; +import { firstValueFrom } from "rxjs"; +import { isObjectLiteral } from "packages/shared/src/utils/object-utils"; +import { FlowTypes } from "packages/data-models"; +import { MangoQuery } from "rxdb"; +import { + coerceDataUpdateTypes, + evaluateDynamicDataUpdate, + isItemChanged, +} from "./dynamic-data.utils"; + +/** Metadata passed to set_data action to specify data for update **/ +interface IActionSetDataParamsMeta { + /** Reference to source data_list id. All rows in list will be updated */ + _list_id?: string; + + /** ID of data_list row for single update */ + _id?: string; + + /** Index of data_list row for single update */ + _index?: number; + + /** List of compiled data updates if computed outside of action (e.g. data_items) */ + _updates?: FlowTypes.Data_listRow[]; +} + +/** Key-value pairs to update. These support reference to self `@item` context */ +export type IActionSetDataParams = IActionSetDataParamsMeta & Record; + +class DynamicDataActionFactory { + constructor(private service: DynamicDataService) {} + + /** + * Similar syntax can be used for multiple items. + * + * or from outside a loop can specify + * click | set_data | _list: @data.example_list, completed:true; + */ + public set_data: IActionHandler = async ({ params }: { params?: IActionSetDataParams }) => { + const { _list_id, _updates } = await this.parseParams(params); + // Hack, no current method for bulk update so make successive (changes debounced in component) + for (const { id, ...writeableProps } of _updates) { + await this.service.update("data_list", _list_id, id, writeableProps); + } + }; + + public reset_data: IActionHandler = async ({ params }: { params?: IActionSetDataParams }) => { + const { _list_id } = await this.parseParams(params); + return this.service.resetFlow("data_list", _list_id); + }; + + /** Parse action parameters to generate a list of updates to apply */ + private async parseParams(params: IActionSetDataParams) { + if (isObjectLiteral(params)) { + const parsed = this.hackParseTemplatedParams(params); + let { _updates, _list_id } = parsed; + // handle parse from item reference string + if (_list_id) { + if (!_updates) { + _updates = await this.generateUpdateList(parsed); + } + return { _updates, _list_id }; + } + } + + // throw error if args not parsed correctly + console.error(params); + throw new Error("[set_data] could not parse params"); + } + + private async generateUpdateList(params: IActionSetDataParams) { + // remove metadata from rest of update + const { _id, _index, _list_id, _updates, ...update } = params; + const query: MangoQuery = {}; + if (_id) { + query.selector = { id: _id }; + } + let ref = await this.service.query$("data_list", _list_id, query); + let items = await firstValueFrom(ref); + if (items.length === 0) { + const msg = `[Update Fail] no doc exists\ndata_list: ${_list_id}\n_id: ${_id}`; + throw new Error(msg); + } + // NOTE - RXDB doesn't support querying by index or pagination so still retrieve all and then reduce + if (typeof _index === "number") { + const targetItem = items[_index]; + if (!targetItem) { + const msg = `[Update Fail] no doc exists\ndata_list: ${_list_id}\n_index: ${_index}`; + throw new Error(msg); + } + items = [items[_index]]; + } + + const cleanedUpdate = this.removeUpdateMetadata(update); + + // Evaluate item updates for any `@item` self-references + const evaluated = evaluateDynamicDataUpdate(items, cleanedUpdate); + + // Coerce updates to correct data types (inline parameter_list values parsed as strings) + const schema = this.service.getSchema("data_list", _list_id); + const coerced = coerceDataUpdateTypes(schema?.jsonSchema?.properties, evaluated); + + // Filter to only include updates that will change original item + return coerced.filter((data, i) => isItemChanged(items[i], data)); + } + + private removeUpdateMetadata(update: Record) { + for (const key of Object.keys(update)) { + if (key.startsWith("_")) delete update[key]; + } + return update; + } + + /** + * Any params provided by templating system will already be partially parsed to replace dynamic references + * such as @local or @field. This also converts @item references to `this.item` + * Revert the item changes and also convert string parameter values to number where required + * + * TODO - this method should be removed and `items` made to update their own `this` context + **/ + private hackParseTemplatedParams(params: IActionSetDataParams) { + const parsed: IActionSetDataParams = {}; + // HACK - un-parse @item references that the templating system converts to `this.item` + for (const [key, value] of Object.entries(params)) { + if (typeof value === "string") { + parsed[key] = value.replace(/this\.item/g, "@item"); + } else { + parsed[key] = value; + } + } + // convert _index param which may be passed as string from template if defined inline + // NOTE - RXDB will automatically cast all other string values to correct type due to inferred schema + if (typeof params._index === "string") { + parsed._index = Number(params._index); + } + return parsed; + } +} + +export default DynamicDataActionFactory; diff --git a/src/app/shared/services/dynamic-data/dynamic-data.service.mock.spec.ts b/src/app/shared/services/dynamic-data/dynamic-data.service.mock.spec.ts new file mode 100644 index 0000000000..939e34393d --- /dev/null +++ b/src/app/shared/services/dynamic-data/dynamic-data.service.mock.spec.ts @@ -0,0 +1,16 @@ +import { FlowTypes } from "packages/data-models"; +import { DynamicDataService } from "./dynamic-data.service"; +import { of } from "rxjs"; + +/** Mock implementation used in other tests */ +export class MockDynamicDataService implements Partial { + constructor(private mockQueryData = {}) {} + // mock query just returns data provided as observable + public async query$(flow_type: FlowTypes.FlowType, flow_name: string) { + return of(this.mockQueryData[flow_type]?.[flow_name]); + } + + public async ready() { + return true; + } +} diff --git a/src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts b/src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts index e6cde5bff1..95fafc4e6b 100644 --- a/src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts +++ b/src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts @@ -2,9 +2,11 @@ import { TestBed } from "@angular/core/testing"; import { HttpClientTestingModule } from "@angular/common/http/testing"; import { firstValueFrom } from "rxjs"; -import { DynamicDataService, ISetItemContext } from "./dynamic-data.service"; +import { DynamicDataService } from "./dynamic-data.service"; import { AppDataService } from "../data/app-data.service"; import { MockAppDataService } from "../data/app-data.service.spec"; +import { DeploymentService } from "../deployment/deployment.service"; +import { MockDeploymentService } from "../deployment/deployment.service.spec"; const TEST_DATA_ROWS = [ { id: "id1", number: 1, string: "hello", boolean: true, _meta_field: { test: "hello" } }, @@ -13,12 +15,6 @@ const TEST_DATA_ROWS = [ ]; type ITestRow = (typeof TEST_DATA_ROWS)[number]; -const SET_ITEM_CONTEXT: ISetItemContext = { - flow_name: "test_flow", - itemDataIDs: ["id1", "id2"], - currentItemId: "id1", -}; - /** * Call standalone tests via: * yarn ng test --include src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts @@ -44,6 +40,10 @@ describe("DynamicDataService", () => { }, }), }, + { + provide: DeploymentService, + useValue: new MockDeploymentService({ name: "test" }), + }, ], }); @@ -51,7 +51,6 @@ describe("DynamicDataService", () => { window.global = window; service = TestBed.inject(DynamicDataService); - TestBed.inject(AppDataService); await service.ready(); // Ensure any data previously persisted is cleared await service.resetFlow("data_list", "test_flow"); @@ -109,56 +108,12 @@ describe("DynamicDataService", () => { expect(res.length).toEqual(20); }); - it("sets an item correctly for current item", async () => { - await service.setItem({ - context: SET_ITEM_CONTEXT, - writeableProps: { string: "sets an item correctly for current item" }, - }); - const obs = await service.query$("data_list", "test_flow"); - const data = await firstValueFrom(obs); - expect(data[0].string).toEqual("sets an item correctly for current item"); - expect(data[1].string).toEqual("goodbye"); - }); - - it("sets an item correctly for a given _id", async () => { - await service.setItem({ - context: SET_ITEM_CONTEXT, - _id: "id2", - writeableProps: { string: "sets an item correctly for a given _id" }, - }); - const obs = await service.query$("data_list", "test_flow"); - const data = await firstValueFrom(obs); - expect(data[0].string).toEqual("hello"); - expect(data[1].string).toEqual("sets an item correctly for a given _id"); - }); - - it("sets an item correctly for a given _index", async () => { - await service.setItem({ - context: SET_ITEM_CONTEXT, - _index: 1, - writeableProps: { string: "sets an item correctly for a given _index" }, - }); - const obs = await service.query$("data_list", "test_flow"); - const data = await firstValueFrom(obs); - expect(data[0].string).toEqual("hello"); - expect(data[1].string).toEqual("sets an item correctly for a given _index"); - }); - it("supports reading data with protected fields", async () => { const obs = await service.query$("data_list", "test_flow"); const data = await firstValueFrom(obs); expect(data[0]["_meta_field"]).toEqual({ test: "hello" }); }); - it("ignores writes to protected fields", async () => { - await service.setItem({ - context: SET_ITEM_CONTEXT, - writeableProps: { _meta_field: "updated", string: "updated" }, - }); - const obs = await service.query$("data_list", "test_flow"); - const data = await firstValueFrom(obs); - expect(data[0]["string"]).toEqual("updated"); - expect(data[0]["_meta_field"]).toEqual({ test: "hello" }); - }); + it("adds metadata (row_index) to docs", async () => { const obs = await service.query$("data_list", "test_flow"); const data = await firstValueFrom(obs); diff --git a/src/app/shared/services/dynamic-data/dynamic-data.service.ts b/src/app/shared/services/dynamic-data/dynamic-data.service.ts index 356ccfd645..d8df8ab769 100644 --- a/src/app/shared/services/dynamic-data/dynamic-data.service.ts +++ b/src/app/shared/services/dynamic-data/dynamic-data.service.ts @@ -11,6 +11,7 @@ import { PersistedMemoryAdapter } from "./adapters/persistedMemory"; import { ReactiveMemoryAdapter, REACTIVE_SCHEMA_BASE } from "./adapters/reactiveMemory"; import { TemplateActionRegistry } from "../../components/template/services/instance/template-action.registry"; import { TopLevelProperty } from "rxdb/dist/types/types"; +import ActionsFactory from "./dynamic-data.actions"; import { DeploymentService } from "../deployment/deployment.service"; type IDocWithMeta = { id: string; APP_META?: Record }; @@ -52,6 +53,10 @@ export class DynamicDataService extends AsyncServiceBase { ) { super("Dynamic Data"); this.registerInitFunction(this.initialise); + // register action handlers + const { set_data, reset_data } = new ActionsFactory(this); + this.templateActionRegistry.register({ set_data, reset_data }); + // HACK - Legacy `set_item` action still managed here (will be removed in #2454) this.registerTemplateActionHandlers(); } @@ -140,12 +145,16 @@ export class DynamicDataService extends AsyncServiceBase { const existingDoc = await this.db.getDoc(collectionName, row_id); if (existingDoc) { const data = existingDoc.toMutableJSON(); - update = deepMergeObjects(data, update); + const mergedUpdate = deepMergeObjects(data, update); + // update memory db + await this.db.updateDoc({ collectionName, id: row_id, data: mergedUpdate }); + // update persisted db + this.writeCache.update({ flow_name, flow_type, id: row_id, data: mergedUpdate }); + } else { + throw new Error( + `[Update Fail] no doc exists for ${flow_type}:${flow_name} with row_id: ${row_id}` + ); } - // update memory db - await this.db.updateDoc({ collectionName, id: row_id, data: update }); - // update persisted db - this.writeCache.update({ flow_name, flow_type, id: row_id, data: update }); } } @@ -157,14 +166,20 @@ export class DynamicDataService extends AsyncServiceBase { await lastValueFrom(this.collectionCreators[collectionName]); } // Ensure any persisted data deleted - await this.writeCache.delete(flow_type, flow_name); + this.writeCache.delete(flow_type, flow_name); // Remove in-memory db if exists const existingCollection = this.db.getCollection(collectionName); if (existingCollection) { - await this.db.removeCollection(collectionName); + // Empty existing data and re-seed initial data + const docs = await existingCollection.find().exec(); + await existingCollection.bulkRemove(docs.map((d) => d.id)); + // Re-seed initial data + const initialData = await this.getInitialData(flow_type, flow_name); + await existingCollection.bulkInsert(initialData); + } else { + await this.createCollection(flow_type, flow_name); } - await this.createCollection(flow_type, flow_name); } /** Access full state of all persisted data layers */ @@ -174,6 +189,11 @@ export class DynamicDataService extends AsyncServiceBase { return this.writeCache.state; } + public getSchema(flow_type: FlowTypes.FlowType, flow_name: string) { + const collectionName = this.normaliseCollectionName(flow_type, flow_name); + return this.db.getCollection(collectionName)?.schema; + } + /** Ensure a collection exists, creating if not and populating with corresponding list data */ private async ensureCollection(flow_type: FlowTypes.FlowType, flow_name: string) { const collectionName = this.normaliseCollectionName(flow_type, flow_name); @@ -196,17 +216,10 @@ export class DynamicDataService extends AsyncServiceBase { if (initialData.length === 0) { throw new Error(`No data exists for collection [${flow_name}], cannot initialise`); } - // add index property to each element before insert, for sorting queried data by original order - const initialDataWithMeta = initialData.map((el) => { - return { - ...el, - row_index: initialData.indexOf(el), - }; - }); - const schema = this.inferSchema(initialDataWithMeta[0]); + const schema = this.inferSchema(initialData[0]); await this.db.createCollection(collectionName, schema); - await this.db.bulkInsert(collectionName, initialDataWithMeta); + await this.db.bulkInsert(collectionName, initialData); // notify any observers that collection has been created this.collectionCreators[collectionName].next(collectionName); this.collectionCreators[collectionName].complete(); @@ -224,7 +237,10 @@ export class DynamicDataService extends AsyncServiceBase { const mergedData = this.mergeData(flowData?.rows, writeDataArray); // HACK - rxdb can't write any fields prefixed with `_` so extract all to top-level APP_META key const cleaned = mergedData.map((el) => this.extractMeta(el)); - return cleaned; + + // add index property to each element before insert, for sorting queried data by original order + const initialDataWithMeta = cleaned.map((el) => ({ ...el, row_index: cleaned.indexOf(el) })); + return initialDataWithMeta; } /** When working with rxdb collections only alphanumeric lower case names allowed */ diff --git a/src/app/shared/services/dynamic-data/dynamic-data.utils.spec.ts b/src/app/shared/services/dynamic-data/dynamic-data.utils.spec.ts new file mode 100644 index 0000000000..4fb0c7e5e7 --- /dev/null +++ b/src/app/shared/services/dynamic-data/dynamic-data.utils.spec.ts @@ -0,0 +1,67 @@ +import { FlowTypes } from "packages/data-models"; +import { + coerceDataUpdateTypes, + evaluateDynamicDataUpdate, + isItemChanged, +} from "./dynamic-data.utils"; +import { JsonSchema } from "rxdb"; + +const itemRows: FlowTypes.Data_listRow[] = [ + { id: "id_0", number: 0, string: "hello" }, + { id: "id_1", number: 1, string: "hello", boolean: true }, +]; + +/******************************************************************************** + * Tests + * yarn ng test --include src/app/shared/services/dynamic-data/dynamic-data.utils.spec.ts + *******************************************************************************/ +describe("DynamicDataService Utils", () => { + /************************************************************* + * Main Tests + ************************************************************/ + it("evaluateDynamicDataUpdate", async () => { + const res = evaluateDynamicDataUpdate(itemRows, { number: "@item.number+10" }); + expect(res).toEqual([ + { id: "id_0", number: 10 }, + { id: "id_1", number: 11 }, + ]); + }); + + it("isItemChanged true", async () => { + const res1 = isItemChanged({ id: "id_1", number: 1, string: "hello" }, { number: 2 }); + expect(res1).toEqual(true); + const res2 = isItemChanged({ id: "id_1", number: 1 }, { number: 1 }); + expect(res2).toEqual(false); + }); + + it("coerceDataUpdateTypes", () => { + const schemaMapping: Record = { + number: { type: "number" }, + boolean: { type: "boolean" }, + text: { type: "string" }, + }; + const res1 = coerceDataUpdateTypes(schemaMapping, [ + { + number: "1", + boolean: "true", + text: "hello", + additional: "string", + }, + ]); + expect(res1).toEqual([ + { + number: 1, + boolean: true, + text: "hello", + additional: "string", + }, + ]); + // does not coerce missing or undefined properties + const res2 = coerceDataUpdateTypes(schemaMapping, [ + { + number: undefined, + }, + ]); + expect(res2).toEqual([{ number: undefined }]); + }); +}); diff --git a/src/app/shared/services/dynamic-data/dynamic-data.utils.ts b/src/app/shared/services/dynamic-data/dynamic-data.utils.ts new file mode 100644 index 0000000000..695ec62a36 --- /dev/null +++ b/src/app/shared/services/dynamic-data/dynamic-data.utils.ts @@ -0,0 +1,100 @@ +import type { FlowTypes } from "packages/data-models/flowTypes"; +import { TemplatedData } from "packages/shared/src/models/templatedData/templatedData"; +import { AppDataEvaluator } from "packages/shared/src/models/appDataEvaluator/appDataEvaluator"; +import { isEqual, isObjectLiteral } from "packages/shared/src/utils/object-utils"; +import { JsonSchema, JsonSchemaTypes } from "rxdb"; +import { booleanStringToBoolean } from "../../utils"; + +/** + * Given an update to apply to a list of items check whether the update self-references + * `@item.some_field` within any part of the update, and if so evaluated for each + * individual item context. Return list of updates to apply, evaluated for item context + */ +export function evaluateDynamicDataUpdate( + items: FlowTypes.Data_listRow[], + update: Record +) { + if (hasDynamicDataItemReferences(update)) { + const evaluator = new AppDataEvaluator(); + return items.map((item) => { + evaluator.setExecutionContext({ item }); + const evaluated = evaluator.evaluate(update); + // ensure target item id also included in update + return { ...evaluated, id: item.id }; + }); + } else { + return items.map((item) => { + return { ...update, id: item.id }; + }); + } +} + +/** Check whether the update does include any `@item` references */ +function hasDynamicDataItemReferences(data: any) { + const contextVariables = new TemplatedData().listContextVariables(data, ["item"]); + return contextVariables.item; +} + +/** + * Compare an item with update snapshot and determine whether the update contains + * any changes. As updates are partials only compare properties in update against item equivalent + */ +export function isItemChanged( + item: FlowTypes.Data_listRow, + update: Partial +) { + const isChanged = Object.entries(update).find(([key, value]) => !isEqual(item[key], value)); + return isChanged ? true : false; +} + +/** + * Use rxdb schema to cast data updates to the correct data types + * Used to convert parameter_list values stored as strings to number or boolean where required + * */ +export function coerceDataUpdateTypes(schemaProperties: Record, data: any[]) { + // create a general mapping to convert any listed schema property from a string value + // to the correct schema value type + const propertyMapping: Record any> = {}; + for (const [key, { type }] of Object.entries(schemaProperties)) { + // do not map metadata fields (can't be set by user input) + if (!key.startsWith("_")) { + propertyMapping[key] = typeMappings[type as JsonSchemaTypes]; + } + } + // coerce data values using mapping + const coerced = []; + for (const el of data) { + if (isObjectLiteral(el)) { + for (const [key, value] of Object.entries(el)) { + // only map values if they are strings and have a defined mapping function + if (propertyMapping[key] && typeof value === "string") { + el[key] = propertyMapping[key](value); + } + } + } + coerced.push(el); + } + return coerced; +} + +/** + * Mapping functions used to coerce string values to RXDB schema types + * This is a subset of mappings that could alternatively be provided by + * https://ajv.js.org/coercion.html + * **/ +const typeMappings: Record any | undefined> = { + // do not attempt to coerce arrays (data_list does not store array values) + array: undefined, + // convert boolean string to boolean + boolean: (v) => booleanStringToBoolean(v), + // convert string integer to integer + integer: (v) => parseInt(v, 10), + // do not attempt to coerce null schema + null: undefined, + // convert string number to number + number: (v) => Number(v), + // do not attempt to coerce objects (data_list does not store object values) + object: undefined, + // as incoming values will be formatted as strings do not provide any additional mapping + string: undefined, +}; diff --git a/src/app/shared/services/error-handler/error-handler.service.mock.spec.ts b/src/app/shared/services/error-handler/error-handler.service.mock.spec.ts new file mode 100644 index 0000000000..7f3d020570 --- /dev/null +++ b/src/app/shared/services/error-handler/error-handler.service.mock.spec.ts @@ -0,0 +1,11 @@ +import { ErrorHandlerService } from "./error-handler.service"; + +/** Mock calls for sheets from the appData service to return test data */ +export class MockErrorHandlerService implements Partial { + public logError(error: Error): Promise { + throw error; + } + public handleError(error: Error): Promise { + throw error; + } +} diff --git a/src/app/shared/services/error-handler/error-handler.service.spec.ts b/src/app/shared/services/error-handler/error-handler.service.spec.ts index 96afdbf7f1..f377e0c6b3 100644 --- a/src/app/shared/services/error-handler/error-handler.service.spec.ts +++ b/src/app/shared/services/error-handler/error-handler.service.spec.ts @@ -3,16 +3,6 @@ import { TestBed } from "@angular/core/testing"; import { ErrorHandlerService } from "./error-handler.service"; import { FirebaseService } from "../firebase/firebase.service"; -/** Mock calls for sheets from the appData service to return test data */ -export class MockErrorHandlerService implements Partial { - public logError(error: Error): Promise { - throw error; - } - public handleError(error: Error): Promise { - throw error; - } -} - describe("ErrorHandlerService", () => { let service: ErrorHandlerService;