From bf915e200fd1208a5709d0e355d9296de90d83a4 Mon Sep 17 00:00:00 2001 From: chrismclarke Date: Tue, 10 Dec 2024 11:51:07 -0800 Subject: [PATCH 01/10] chore: code tidying --- packages/data-models/flowTypes.ts | 13 +++++++++ .../template/processors/itemPipe.ts | 9 ++++-- .../dynamic-data/dynamic-data.actions.ts | 29 ++++++++++++++++++- 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/packages/data-models/flowTypes.ts b/packages/data-models/flowTypes.ts index e7b0a95a96..87490ac711 100644 --- a/packages/data-models/flowTypes.ts +++ b/packages/data-models/flowTypes.ts @@ -382,6 +382,19 @@ export namespace FlowTypes { export type IDynamicPrefix = (typeof DYNAMIC_PREFIXES)[number]; + /** + * Namespaced hashmap of dynamic variables. Values may include evaluated value, + * or may include variable metadata depending on use context + * @example + * ``` + * { + * local: {my_local_var: {} + * field: {some_field_var: {} + * } + * ``` + * */ + export type IDynamicContext = { [key in IDynamicPrefix]?: { [fieldName: string]: any } }; + /** Data passed back from regex match, e.g. expression @local.someField => type:local, fieldName: someField */ export interface TemplateRowDynamicEvaluator { fullExpression: string; diff --git a/src/app/shared/components/template/processors/itemPipe.ts b/src/app/shared/components/template/processors/itemPipe.ts index 63fcfc4711..656eeefd9d 100644 --- a/src/app/shared/components/template/processors/itemPipe.ts +++ b/src/app/shared/components/template/processors/itemPipe.ts @@ -1,3 +1,4 @@ +import type { FlowTypes } from "packages/data-models/flowTypes"; import { JSEvaluator } from "packages/shared/src/models/jsEvaluator/jsEvaluator"; import { shuffleArray } from "src/app/shared/utils"; @@ -22,13 +23,17 @@ export class ItemDataPipe { if (!sortField) return items; return items.sort((a, b) => (a[sortField] > b[sortField] ? 1 : -1)); }, - filter: (items: any[] = [], expression: string) => { + filter: ( + items: any[] = [], + expression: string, + contextVariables: FlowTypes.IDynamicContext = {} + ) => { if (!expression) return; return items.filter((item) => { // NOTE - expects all non-item condition to be evaluated // e.g. `@item.field > @local.some_value` already be evaluated to `this.item.field > "local value"` const evaluator = new JSEvaluator(); - const evaluated = evaluator.evaluate(expression, { item }); + const evaluated = evaluator.evaluate(expression, { ...contextVariables, item }); return evaluated; }); }, diff --git a/src/app/shared/services/dynamic-data/dynamic-data.actions.ts b/src/app/shared/services/dynamic-data/dynamic-data.actions.ts index 3ead2ccf0e..1e6fd922a9 100644 --- a/src/app/shared/services/dynamic-data/dynamic-data.actions.ts +++ b/src/app/shared/services/dynamic-data/dynamic-data.actions.ts @@ -10,8 +10,17 @@ import { isItemChanged, } from "./dynamic-data.utils"; +interface IActionSetDataOperatorParams { + // TODO - ideally use same itemPipe operators + // (although filter will need to be updated to include dynamic context) + _filter?: string; + _limit?: number; + _reverse?: boolean; + _sort?: string; +} + /** Metadata passed to set_data action to specify data for update **/ -interface IActionSetDataParamsMeta { +interface IActionSetDataParamsMeta extends IActionSetDataOperatorParams { /** Reference to source data_list id. All rows in list will be updated */ _list_id?: string; @@ -92,6 +101,8 @@ class DynamicDataActionFactory { items = [items[_index]]; } + const filteredUpdate = {}; + const cleanedUpdate = this.removeUpdateMetadata(update); // Evaluate item updates for any `@item` self-references @@ -105,6 +116,22 @@ class DynamicDataActionFactory { return coerced.filter((data, i) => isItemChanged(items[i], data)); } + private applyUpdateOperations(params: IActionSetDataParams) { + // TODO - decide when to evaluate... e.g. filter: @item.id > @local.value + const { _filter, _reverse, _sort, _limit } = params; + const parsedOps: IActionSetDataOperatorParams = {}; + for (const [operator, arg] of Object.entries({ _filter, _reverse, _sort, _limit })) { + if (arg !== undefined) { + let parsedArg = ""; + // TODO - understand if dynamic or numeric string... + // TODO - better to convert these params in parser not at runtime!!! (same for _index previously) + // using some sort of parseNumericalString or similar regex lookup... same for boolean strings and other similar + if (operator === "_limit" && typeof parsedArg === "string") { + } + } + } + } + private removeUpdateMetadata(update: Record) { for (const key of Object.keys(update)) { if (key.startsWith("_")) delete update[key]; From 52c48d2d1d1c818f99cbc774d7c0adc1a2645032 Mon Sep 17 00:00:00 2001 From: chrismclarke Date: Tue, 10 Dec 2024 11:57:01 -0800 Subject: [PATCH 02/10] Merge branch 'master' of https://github.com/idemsinternational/open-app-builder into feat/set-data-action-operators From 7b66fcaed16d31dc407c8be32e9c435ffc249ed0 Mon Sep 17 00:00:00 2001 From: chrismclarke Date: Wed, 11 Dec 2024 16:07:51 -0800 Subject: [PATCH 03/10] chore: test tidying --- .../dynamic-data/dynamic-data.actions.spec.ts | 74 +++++++++++++------ .../dynamic-data/dynamic-data.service.spec.ts | 28 ++++--- 2 files changed, 69 insertions(+), 33 deletions(-) diff --git a/src/app/shared/services/dynamic-data/dynamic-data.actions.spec.ts b/src/app/shared/services/dynamic-data/dynamic-data.actions.spec.ts index 4618cc60c8..fd87793be0 100644 --- a/src/app/shared/services/dynamic-data/dynamic-data.actions.spec.ts +++ b/src/app/shared/services/dynamic-data/dynamic-data.actions.spec.ts @@ -13,20 +13,38 @@ import { DeploymentService } from "../deployment/deployment.service"; import { MockDeploymentService } from "../deployment/deployment.service.spec"; import { TemplateActionRegistry } from "../../components/template/services/instance/template-action.registry"; -const TEST_DATA_ROWS = [ - { id: "id_0", number: 0, string: "hello", _meta: "original" }, - { id: "id_1", number: 1, string: "hello", boolean: true, _meta: "original" }, +type ITestRow = { id: string; number: number; string: string; _meta_field?: any }; + +const TEST_DATA_ROWS = (): FlowTypes.Data_listRow[] => [ + { id: "id_0", number: 0, string: "hello", _meta_field: "original" }, + { id: "id_1", number: 1, string: "hello", _meta_field: "original" }, ]; +const TEST_DATA_LIST = (): FlowTypes.Data_list => ({ + flow_name: "test_flow", + flow_type: "data_list", + // Make deep clone of data to avoid data overwrite issues + rows: TEST_DATA_ROWS(), + // Metadata would be extracted from parser based on data or defined schema + _metadata: { + boolean: { type: "boolean" }, + number: { type: "number" }, + _meta_field: { type: "object" }, + }, +}); + /******************************************************************************** * Test Utilities *******************************************************************************/ /** Generate a rows to trigger set_data action with included params */ -function getTestActionRow(params: IActionSetDataParams) { +function getTestActionRow( + params: IActionSetDataParams, + action_id: FlowTypes.TemplateRowAction["action_id"] +) { params._list_id = "test_flow"; const actionRow: FlowTypes.TemplateRowAction = { - action_id: "set_data", + action_id, trigger: "click", args: [], params, @@ -39,7 +57,7 @@ function getTestActionRow(params: IActionSetDataParams) { * to corresponding data_list * * */ async function triggerTestSetDataAction(service: DynamicDataService, params: IActionSetDataParams) { - const actionRow = getTestActionRow(params); + const actionRow = getTestActionRow(params, "set_data"); const actions = new ActionFactory(service); await actions.set_data(actionRow); const obs = await service.query$("data_list", "test_flow"); @@ -47,6 +65,15 @@ async function triggerTestSetDataAction(service: DynamicDataService, params: IAc return data; } +async function triggerAddDataAction(service: DynamicDataService, params: IActionSetDataParams) { + const actionRow = getTestActionRow(params, "add_data"); + const actions = new ActionFactory(service); + await actions.add_data(actionRow); + const obs = await service.query$("data_list", "test_flow"); + const data = await firstValueFrom(obs); + return data; +} + /******************************************************************************** * Tests * yarn ng test --include src/app/shared/services/dynamic-data/dynamic-data.actions.spec.ts @@ -64,12 +91,7 @@ describe("DynamicDataService Actions", () => { provide: AppDataService, useValue: new MockAppDataService({ data_list: { - test_flow: { - flow_name: "test_flow", - flow_type: "data_list", - // Make deep clone of data to avoid data overwrite issues - rows: JSON.parse(JSON.stringify(TEST_DATA_ROWS)), - }, + test_flow: TEST_DATA_LIST(), }, }), }, @@ -139,10 +161,10 @@ describe("DynamicDataService Actions", () => { }); it("set_data prevents update to metadata fields", async () => { - const params: IActionSetDataParams = { _meta: "updated", string: "updated" }; + const params: IActionSetDataParams = { _meta_field: "updated", string: "updated" }; const data = await triggerTestSetDataAction(service, params); expect(data[0].string).toEqual("updated"); - expect(data[0]._meta).toEqual("original"); + expect(data[0]._meta_field).toEqual("original"); }); it("set_data ignores evaluation when _updates provided", async () => { @@ -156,7 +178,7 @@ describe("DynamicDataService Actions", () => { it("reset_data action restores data to initial", async () => { const updatedData = await triggerTestSetDataAction(service, { string: "updated string" }); expect(updatedData[0].string).toEqual("updated string"); - const resetActionBase = getTestActionRow({}); + const resetActionBase = getTestActionRow({}, "reset_data"); await actions.reset_data({ ...resetActionBase, action_id: "reset_data" }); const obs = await service.query$("data_list", "test_flow"); const resetData = await firstValueFrom(obs); @@ -179,20 +201,26 @@ describe("DynamicDataService Actions", () => { ************************************************************/ it("throws error if provided _id does not exist", async () => { - const params = getTestActionRow({ - _id: "missing_id", - string: "sets an item correctly a given id", - }); + const params = getTestActionRow( + { + _id: "missing_id", + string: "sets an item correctly a given id", + }, + "set_data" + ); await expectAsync(actions.set_data(params)).toBeRejectedWithError( `[Update Fail] no doc exists\ndata_list: test_flow\n_id: missing_id` ); }); it("throws error if provided _index does not exist", async () => { - const params = getTestActionRow({ - _index: 10, - string: "sets an item correctly a given id", - }); + const params = getTestActionRow( + { + _index: 10, + string: "sets an item correctly a given id", + }, + "set_data" + ); await expectAsync(actions.set_data(params)).toBeRejectedWithError( `[Update Fail] no doc exists\ndata_list: test_flow\n_index: 10` ); diff --git a/src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts b/src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts index bbf8754c78..85090f72b0 100644 --- a/src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts +++ b/src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts @@ -7,13 +7,28 @@ import { AppDataService } from "../data/app-data.service"; import { MockAppDataService } from "../data/app-data.service.mock.spec"; import { DeploymentService } from "../deployment/deployment.service"; import { MockDeploymentService } from "../deployment/deployment.service.spec"; +import { FlowTypes } from "packages/data-models"; -const TEST_DATA_ROWS = [ +type ITestRow = { id: string; number: number; string: string; boolean: boolean; _meta_field?: any }; + +const TEST_DATA_ROWS = (): FlowTypes.Data_listRow[] => [ { id: "id1", number: 1, string: "hello", boolean: true, _meta_field: { test: "hello" } }, { id: "id2", number: 2, string: "goodbye", boolean: false }, { id: "id0", number: 3, string: "goodbye", boolean: false }, ]; -type ITestRow = (typeof TEST_DATA_ROWS)[number]; + +const TEST_DATA_LIST = (): FlowTypes.Data_list => ({ + flow_name: "test_flow", + flow_type: "data_list", + // Make deep clone of data to avoid data overwrite issues + rows: TEST_DATA_ROWS(), + // Metadata would be extracted from parser based on data or defined schema + _metadata: { + boolean: { type: "boolean" }, + number: { type: "number" }, + _meta_field: { type: "object" }, + }, +}); /** * Call standalone tests via: @@ -30,14 +45,7 @@ describe("DynamicDataService", () => { { provide: AppDataService, useValue: new MockAppDataService({ - data_list: { - test_flow: { - flow_name: "test_flow", - flow_type: "data_list", - // Make deep clone of data to avoid data overwrite issues - rows: JSON.parse(JSON.stringify(TEST_DATA_ROWS)), - }, - }, + data_list: { test_flow: TEST_DATA_LIST() }, }), }, { From 7eac128d16ddcc27c716d66bbbf84d1cf2758c12 Mon Sep 17 00:00:00 2001 From: chrismclarke Date: Fri, 13 Dec 2024 13:40:05 -0800 Subject: [PATCH 04/10] feat: add_data and remove_data actions --- package.json | 2 + packages/data-models/flowTypes.ts | 10 ++-- .../dynamic-data/dynamic-data.actions.ts | 58 +++++++++++++++---- src/app/shared/utils/utils.ts | 6 ++ yarn.lock | 18 ++++++ 5 files changed, 78 insertions(+), 16 deletions(-) diff --git a/package.json b/package.json index 1d96b4ad74..ad25ca2165 100644 --- a/package.json +++ b/package.json @@ -109,6 +109,7 @@ "stacktrace-js": "^2.0.2", "swiper": "^8.4.7", "trusted-types": "^2.0.0", + "uuid": "^11.0.3", "zone.js": "~0.14.2" }, "devDependencies": { @@ -141,6 +142,7 @@ "@types/nouislider": "^15.0.0", "@types/qrcode": "^1.5.5", "@types/swiper": "~4.2.0", + "@types/uuid": "^10.0.0", "@typescript-eslint/eslint-plugin": "^6.13.1", "@typescript-eslint/parser": "^6.13.1", "codelyzer": "^6.0.2", diff --git a/packages/data-models/flowTypes.ts b/packages/data-models/flowTypes.ts index 9fb971b3cc..389d9a0914 100644 --- a/packages/data-models/flowTypes.ts +++ b/packages/data-models/flowTypes.ts @@ -396,6 +396,9 @@ export namespace FlowTypes { | "sent" // notification sent | "uncompleted"; + const DATA_ACTIONS_LIST = ["add_data", "remove_data", "set_data"] as const; + const ITEMS_ACTIONS_LIST = ["remove_item", "set_item", "set_items"] as const; + // TODO document '' action for stop propagation // note - to keep target nav within component stack go_to is actually just a special case of pop_up // TODO - 2021-03-11 - most of list needs reconsideration/implementation @@ -425,11 +428,6 @@ export namespace FlowTypes { "save_to_device", "screen_orientation", "set_field", - /** NOTE - only available from with data_items loop */ - "set_item", - /** NOTE - only available from with data_items loop */ - "set_items", - "set_data", "set_local", "share", "style", @@ -440,6 +438,8 @@ export namespace FlowTypes { "track_event", "trigger_actions", "user", + ...DATA_ACTIONS_LIST, + ...ITEMS_ACTIONS_LIST, ] as const; export interface TemplateRowAction { diff --git a/src/app/shared/services/dynamic-data/dynamic-data.actions.ts b/src/app/shared/services/dynamic-data/dynamic-data.actions.ts index 1e6fd922a9..c88004a0d6 100644 --- a/src/app/shared/services/dynamic-data/dynamic-data.actions.ts +++ b/src/app/shared/services/dynamic-data/dynamic-data.actions.ts @@ -9,6 +9,7 @@ import { evaluateDynamicDataUpdate, isItemChanged, } from "./dynamic-data.utils"; +import { generateUUID } from "../../utils"; interface IActionSetDataOperatorParams { // TODO - ideally use same itemPipe operators @@ -37,6 +38,11 @@ interface IActionSetDataParamsMeta extends IActionSetDataOperatorParams { /** Key-value pairs to update. These support reference to self `@item` context */ export type IActionSetDataParams = IActionSetDataParamsMeta & Record; +export type IActionRemoveDataParams = { + _list_id: string; + _id: string; +}; + class DynamicDataActionFactory { constructor(private service: DynamicDataService) {} @@ -47,7 +53,12 @@ class DynamicDataActionFactory { * click | set_data | _list: @data.example_list, completed:true; */ public set_data: IActionHandler = async ({ params }: { params?: IActionSetDataParams }) => { - const { _list_id, _updates } = await this.parseParams(params); + const parsed = await this.parseParams(params); + // if called from set_item will already include list of updates to apply, if not generate + if (!parsed._updates) { + parsed._updates = await this.generateUpdateList(parsed); + } + const { _list_id, _updates } = parsed; // Hack, no current method for bulk update so make successive (changes debounced in component) for (const { id, ...writeableProps } of _updates) { await this.service.update("data_list", _list_id, id, writeableProps); @@ -59,18 +70,37 @@ class DynamicDataActionFactory { return this.service.resetFlow("data_list", _list_id); }; + public remove_data: IActionHandler = async ({ params }: { params?: IActionRemoveDataParams }) => { + const { _id, _list_id } = params; + return this.service.remove("data_list", _list_id, [_id]); + }; + + public add_data: IActionHandler = async ({ params }: { params?: IActionSetDataParams }) => { + const { _list_id, ...data } = await this.parseParams(params); + // assign a row_index to push user_generated docs to bottom of list + let row_index = await this.service.getCount("data_list", _list_id); + data.id = generateUUID(); + // add metadata to track user created + data._user_created = true; + data.row_index = row_index; + // HACK - use the same dynamic data evaluator as set_data action + // This requires passing an item list, so just create an ad-hoc list with a single item + const [evaluated] = evaluateDynamicDataUpdate([{ id: data.id }], data); + // TODO - add support for evaluating @list statements + const schema = this.service.getSchema("data_list", _list_id); + const [coerced] = coerceDataUpdateTypes(schema?.jsonSchema?.properties, [evaluated]); + return this.service.insert("data_list", _list_id, coerced); + }; + /** Parse action parameters to generate a list of updates to apply */ private async parseParams(params: IActionSetDataParams) { if (isObjectLiteral(params)) { const parsed = this.hackParseTemplatedParams(params); - let { _updates, _list_id } = parsed; - // handle parse from item reference string - if (_list_id) { - if (!_updates) { - _updates = await this.generateUpdateList(parsed); - } - return { _updates, _list_id }; + if (!parsed._list_id) { + console.error(params); + throw new Error("[Data Actions] could not parse list id"); } + return parsed; } // throw error if args not parsed correctly @@ -101,9 +131,15 @@ class DynamicDataActionFactory { items = [items[_index]]; } - const filteredUpdate = {}; + return this.parseUpdateData(update, items, _list_id); + } - const cleanedUpdate = this.removeUpdateMetadata(update); + private parseUpdateData( + updateData: Record, + items: FlowTypes.Data_listRow[], + _list_id: string + ) { + const cleanedUpdate = this.removeUpdateMetadata(updateData); // Evaluate item updates for any `@item` self-references const evaluated = evaluateDynamicDataUpdate(items, cleanedUpdate); @@ -113,7 +149,7 @@ class DynamicDataActionFactory { const coerced = coerceDataUpdateTypes(schema?.jsonSchema?.properties, evaluated); // Filter to only include updates that will change original item - return coerced.filter((data, i) => isItemChanged(items[i], data)); + return coerced.filter((el, i) => isItemChanged(items[i], el)); } private applyUpdateOperations(params: IActionSetDataParams) { diff --git a/src/app/shared/utils/utils.ts b/src/app/shared/utils/utils.ts index 6c9723d355..2b2c9daf1c 100644 --- a/src/app/shared/utils/utils.ts +++ b/src/app/shared/utils/utils.ts @@ -7,6 +7,7 @@ import { FlowTypes } from "../model"; import { objectToArray } from "../components/template/utils"; import marked from "marked"; import { markedSmartypantsLite } from "marked-smartypants-lite"; +import { v4 as uuidV4 } from "uuid"; /** * Generate a random string of characters in base-36 (a-z and 0-9 characters) @@ -17,6 +18,11 @@ export function generateRandomId() { return Math.random().toString(36).substring(2); } +/** Generate a uuid with v4 specification */ +export function generateUUID() { + return uuidV4(); +} + /** * generate a string representation of the current datetime in local (unspecified) timezone * @returns 2020-12-22T18:15:20 diff --git a/yarn.lock b/yarn.lock index 8a043ef189..9df397df48 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8068,6 +8068,13 @@ __metadata: languageName: node linkType: hard +"@types/uuid@npm:^10.0.0": + version: 10.0.0 + resolution: "@types/uuid@npm:10.0.0" + checksum: e3958f8b0fe551c86c14431f5940c3470127293280830684154b91dc7eb3514aeb79fe3216968833cf79d4d1c67f580f054b5be2cd562bebf4f728913e73e944 + languageName: node + linkType: hard + "@types/validator@npm:*, @types/validator@npm:^13.6.1, @types/validator@npm:^13.7.17": version: 13.11.9 resolution: "@types/validator@npm:13.11.9" @@ -15142,6 +15149,7 @@ __metadata: "@types/nouislider": ^15.0.0 "@types/qrcode": ^1.5.5 "@types/swiper": ~4.2.0 + "@types/uuid": ^10.0.0 "@typescript-eslint/eslint-plugin": ^6.13.1 "@typescript-eslint/parser": ^6.13.1 bootstrap-datepicker: ^1.10.0 @@ -15209,6 +15217,7 @@ __metadata: swiper: ^8.4.7 trusted-types: ^2.0.0 typescript: 5.2.2 + uuid: ^11.0.3 zone.js: ~0.14.2 languageName: unknown linkType: soft @@ -26977,6 +26986,15 @@ __metadata: languageName: node linkType: hard +"uuid@npm:^11.0.3": + version: 11.0.3 + resolution: "uuid@npm:11.0.3" + bin: + uuid: dist/esm/bin/uuid + checksum: 646181c77e8b8df9bd07254faa703943e1c4d5ccde7d080312edf12f443f6c5750801fd9b27bf2e628594182165e6b1b880c0382538f7eca00b26622203741dc + languageName: node + linkType: hard + "uuid@npm:^8.0.0, uuid@npm:^8.3.2": version: 8.3.2 resolution: "uuid@npm:8.3.2" From 6aac403c793aacf35b0bd7601a288628eb1199da Mon Sep 17 00:00:00 2001 From: chrismclarke Date: Fri, 13 Dec 2024 13:40:21 -0800 Subject: [PATCH 05/10] feat: remove_item action --- .../components/data-items/data-items.utils.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/app/shared/components/template/components/data-items/data-items.utils.ts b/src/app/shared/components/template/components/data-items/data-items.utils.ts index 50dd55081c..09d27bbf5d 100644 --- a/src/app/shared/components/template/components/data-items/data-items.utils.ts +++ b/src/app/shared/components/template/components/data-items/data-items.utils.ts @@ -1,4 +1,5 @@ import { FlowTypes } from "packages/data-models"; +import { IActionRemoveDataParams } from "src/app/shared/services/dynamic-data/dynamic-data.actions"; import { ISetItemContext } from "src/app/shared/services/dynamic-data/dynamic-data.service"; /** @@ -44,6 +45,16 @@ export function updateItemMeta( if (a.action_id === "set_item") { a.args = [setItemContext]; } + // re-map remove_item to remove_data action + // TODO - set_item and set_items should also be remapped + if (a.action_id === "remove_item") { + a.action_id = "remove_data"; + const removeDataParams: IActionRemoveDataParams = { + _id: itemId, + _list_id: dataListName, + }; + a.params = removeDataParams; + } if (a.action_id === "set_items") { console.warn( "[Deprecated] set_items should not be used from within an items loop", From 1ba05e1dabf57da06fc287feee5b449cb88e0313 Mon Sep 17 00:00:00 2001 From: chrismclarke Date: Fri, 13 Dec 2024 13:41:37 -0800 Subject: [PATCH 06/10] refactor: doc metadata handling --- .../dynamic-data/adapters/persistedMemory.ts | 26 +++-- .../dynamic-data/adapters/reactiveMemory.ts | 56 ++++++++-- .../dynamic-data/dynamic-data.service.ts | 100 +++++++++--------- 3 files changed, 117 insertions(+), 65 deletions(-) diff --git a/src/app/shared/services/dynamic-data/adapters/persistedMemory.ts b/src/app/shared/services/dynamic-data/adapters/persistedMemory.ts index 894eaaf264..81dd9a96a4 100644 --- a/src/app/shared/services/dynamic-data/adapters/persistedMemory.ts +++ b/src/app/shared/services/dynamic-data/adapters/persistedMemory.ts @@ -37,7 +37,7 @@ const SCHEMA: RxJsonSchema = { title: "base schema for id-primary key data", // NOTE - important to start at 0 and not timestamp (e.g. 20221220) as will check // for migration strategies for each version which is hugely inefficient - version: 2, + version: 3, primaryKey: "id", type: "object", properties: { @@ -48,13 +48,12 @@ const SCHEMA: RxJsonSchema = { flow_name: { type: "string", maxLength: 64 }, flow_type: { type: "string", maxLength: 64 }, row_id: { type: "string", maxLength: 64 }, - row_index: { type: "integer", minimum: 0, maximum: 10000, multipleOf: 1, final: true }, data: { type: "object", }, }, - required: ["id", "flow_type", "flow_name", "row_id", "data", "row_index"], - indexes: ["flow_type", "flow_name", "row_id", "row_index"], + required: ["id", "flow_type", "flow_name", "row_id", "data"], + indexes: ["flow_type", "flow_name", "row_id"], }; const MIGRATIONS: MigrationStrategies = { // As part of RXDb v14 update all data requires migrating to change metadata fields (no doc data changes) @@ -64,6 +63,10 @@ const MIGRATIONS: MigrationStrategies = { const newDoc = { ...oldDoc, row_index: 0 }; return newDoc; }, + // remove row_index from persisted memory as user writes will never modify + 3: (doc) => { + return doc; + }, }; interface IDataUpdate { @@ -136,11 +139,20 @@ export class PersistedMemoryAdapter { this.persistStateToDB(); } - public delete(flow_type: FlowTypes.FlowType, flow_name: string) { - if (this.get(flow_type, flow_name)) { + public delete(flow_type: FlowTypes.FlowType, flow_name: string, ids?: string[]) { + const stateRef = this.get(flow_type, flow_name); + if (!stateRef) return; + // delete individuals + if (ids) { + for (const id of ids) { + delete this.state[flow_type][flow_name][id]; + } + } + // delete all + else { delete this.state[flow_type][flow_name]; - this.persistStateToDB(); } + this.persistStateToDB(); } /** Trigger persist handler. Requests will be debounced and notified when complete */ diff --git a/src/app/shared/services/dynamic-data/adapters/reactiveMemory.ts b/src/app/shared/services/dynamic-data/adapters/reactiveMemory.ts index 59d8f62882..636aeb9885 100644 --- a/src/app/shared/services/dynamic-data/adapters/reactiveMemory.ts +++ b/src/app/shared/services/dynamic-data/adapters/reactiveMemory.ts @@ -20,7 +20,11 @@ import { RxDBMigrationPlugin } from "rxdb/plugins/migration"; addRxPlugin(RxDBMigrationPlugin); import { RxDBUpdatePlugin } from "rxdb/plugins/update"; addRxPlugin(RxDBUpdatePlugin); -import { BehaviorSubject } from "rxjs"; +import { map } from "rxjs"; +import { of } from "rxjs/internal/observable/of"; +import { FlowTypes } from "packages/data-models"; + +type IDocWithMeta = FlowTypes.Data_listRow & { APP_META?: Record }; /** * Create a base schema for data @@ -31,7 +35,7 @@ export const REACTIVE_SCHEMA_BASE: RxJsonSchema = { title: "base schema for id-primary key data", // NOTE - important to start at 0 and not timestamp (e.g. 20221220) as will check // for migration strategies for each version which is hugely inefficient - version: 1, + version: 2, primaryKey: "id", type: "object", properties: { @@ -40,6 +44,7 @@ export const REACTIVE_SCHEMA_BASE: RxJsonSchema = { maxLength: 128, // <- the primary key must have set maxLength }, row_index: { type: "integer", minimum: 0, maximum: 10000, multipleOf: 1, final: true }, + APP_META: { type: "object" }, }, required: ["id"], indexes: ["row_index"], @@ -49,6 +54,10 @@ const MIGRATIONS: MigrationStrategies = { const newDoc = { ...oldDoc, row_index: 0 }; return newDoc; }, + 2: (oldDoc) => { + // adds optional APP_META property to schema, but not required + return oldDoc; + }, }; interface IDataUpdate { @@ -86,17 +95,26 @@ export class ReactiveMemoryAdapter { return undefined; } const matchedDocs = await collection.findByIds([docId]).exec(); - const existingDoc: RxDocument = matchedDocs.get(docId); + const existingDoc: RxDocument = matchedDocs.get(docId); return existingDoc; } - public query(name: string, queryObj?: MangoQuery) { + public query(name: string, queryObj?: MangoQuery) { const collection = this.getCollection(name); if (!collection) { console.error("No db entry", name); - return new BehaviorSubject([]); + return of([]); } - return collection.find(queryObj).$; + // we need mutable json so that we can replace dynamic references as required + // ensure any previously extracted metadata fields are repopulated + return collection.find(queryObj).$.pipe( + map((docs) => + docs.map((d: RxDocument) => { + const data = d.toMutableJSON(); + return this.populateMeta(data); + }) + ) + ); } /** @@ -113,6 +131,8 @@ export class ReactiveMemoryAdapter { if (data.length > 0) { await collection.bulkRemove(data.map((d) => d.id)); } + // Use a pre-insert hook to extract any metadata fields that are unsupported by rxdb + collection.preInsert((doc) => this.extractMeta(doc), true); return collection; } @@ -152,4 +172,28 @@ export class ReactiveMemoryAdapter { public async removeCollection(collectionName: string) { await this.db.collections[collectionName].remove(); } + + /** + * Iterate over a document's key-value pairs and populate any properties starting with + * an underscore to a single top-level APP_META property + */ + private extractMeta(doc: IDocWithMeta) { + const APP_META: Record = {}; + const rxdbMetaKeys = ["_attachments", "_deleted", "_meta", "_rev"]; + for (const [key, value] of Object.entries(doc)) { + if (key.startsWith("_") && !rxdbMetaKeys.includes(key)) { + APP_META[key] = value; + delete doc[key]; + } + } + if (Object.keys(APP_META).length > 0) { + doc.APP_META = APP_META; + } + return doc; + } + /** Populate any previously extracted APP_META properties back to document */ + private populateMeta(doc: IDocWithMeta) { + const { APP_META, ...data } = doc; + return { ...data, ...APP_META }; + } } diff --git a/src/app/shared/services/dynamic-data/dynamic-data.service.ts b/src/app/shared/services/dynamic-data/dynamic-data.service.ts index 633d72b4f2..537c6a0fcb 100644 --- a/src/app/shared/services/dynamic-data/dynamic-data.service.ts +++ b/src/app/shared/services/dynamic-data/dynamic-data.service.ts @@ -1,6 +1,6 @@ import { Injectable } from "@angular/core"; -import { addRxPlugin, MangoQuery, RxDocument } from "rxdb"; -import { firstValueFrom, lastValueFrom, map, AsyncSubject } from "rxjs"; +import { addRxPlugin, MangoQuery } from "rxdb"; +import { firstValueFrom, lastValueFrom, AsyncSubject } from "rxjs"; import { FlowTypes } from "data-models"; import { environment } from "src/environments/environment"; @@ -10,12 +10,9 @@ import { arrayToHashmap, deepMergeObjects } from "../../utils"; import { PersistedMemoryAdapter } from "./adapters/persistedMemory"; import { ReactiveMemoryAdapter, REACTIVE_SCHEMA_BASE } from "./adapters/reactiveMemory"; import { TemplateActionRegistry } from "../../components/template/services/instance/template-action.registry"; -import { TopLevelProperty } from "rxdb/dist/types/types"; import ActionsFactory from "./dynamic-data.actions"; import { DeploymentService } from "../deployment/deployment.service"; -type IDocWithMeta = { id: string; APP_META?: Record }; - @Injectable({ providedIn: "root" }) /** * The dynamic data service handles the process of loading data from sheets, storing @@ -54,8 +51,8 @@ export class DynamicDataService extends AsyncServiceBase { super("Dynamic Data"); this.registerInitFunction(this.initialise); // register action handlers - const { set_data, reset_data } = new ActionsFactory(this); - this.templateActionRegistry.register({ set_data, reset_data }); + const { set_data, reset_data, add_data, remove_data } = new ActionsFactory(this); + this.templateActionRegistry.register({ set_data, reset_data, add_data, remove_data }); // HACK - Legacy `set_item` action still managed here (will be removed in #2454) this.registerLegacyItemsActions(); } @@ -102,7 +99,7 @@ export class DynamicDataService extends AsyncServiceBase { } /** Watch for changes to a specific flow */ - public async query$( + public async query$( flow_type: FlowTypes.FlowType, flow_name: string, queryObj?: MangoQuery @@ -115,21 +112,14 @@ export class DynamicDataService extends AsyncServiceBase { // use a live query to return all documents in the collection, converting // from reactive documents to json data instead let query = this.db.query(collectionName, queryObj); - return query.pipe( - map((v) => { - const docs = v as RxDocument[]; - return docs.map((doc) => { - // we need mutable json so that we can replace dynamic references as required - const data = doc.toMutableJSON(); - // ensure any previously extracted metadata fields are repopulated - return this.populateMeta(data) as T; - }); - }) - ); + return query; } /** Take a snapshot of the current state of a table */ - public async snapshot(flow_type: FlowTypes.FlowType, flow_name: string) { + public async snapshot( + flow_type: FlowTypes.FlowType, + flow_name: string + ) { const obs = await this.query$(flow_type, flow_name); return firstValueFrom(obs); } @@ -158,6 +148,30 @@ export class DynamicDataService extends AsyncServiceBase { } } + public async insert( + flow_type: FlowTypes.FlowType, + flow_name: string, + data: Partial + ) { + const { collectionName } = await this.ensureCollection(flow_type, flow_name); + const { id } = data; + // TODO - remove metadata + const res = await this.db.bulkInsert(collectionName, [data]); + // TODO - generating row_id and row_index + // - possibly just use 0 index for now and can re-populate later after retrieval (?) + // TODO - track whether user generated/owned? + // TODO - check what happens after data_list updated (user entries retained?) + this.writeCache.update({ flow_type, flow_name, id, data }); + // TODO - what if id already exists? + } + + public async remove(flow_type: FlowTypes.FlowType, flow_name: string, ids: string[]) { + const { collectionName } = await this.ensureCollection(flow_type, flow_name); + const collection = this.db.getCollection(collectionName); + await collection.bulkRemove(ids); + this.writeCache.delete(flow_type, flow_name, ids); + } + /** Remove user writes on a flow to return it to its original state */ public async resetFlow(flow_type: FlowTypes.FlowType, flow_name: string) { const collectionName = this.normaliseCollectionName(flow_type, flow_name); @@ -194,6 +208,11 @@ export class DynamicDataService extends AsyncServiceBase { return this.db.getCollection(collectionName)?.schema; } + public getCount(flow_type: FlowTypes.FlowType, flow_name: string) { + const collectionName = this.normaliseCollectionName(flow_type, flow_name); + return this.db.getCollection(collectionName)?.count().exec(); + } + /** Ensure a collection exists, creating if not and populating with corresponding list data */ private async ensureCollection(flow_type: FlowTypes.FlowType, flow_name: string) { const collectionName = this.normaliseCollectionName(flow_type, flow_name); @@ -228,21 +247,19 @@ export class DynamicDataService extends AsyncServiceBase { * compatible in case of schema changes * */ private async prepareInitialData(flow_type: FlowTypes.FlowType, flow_name: string) { - const flowData = await this.appDataService.getSheet(flow_type, flow_name); + const flowData = await this.appDataService.getSheet(flow_type, flow_name); if (!flowData || flowData.rows.length === 0) { throw new Error(`No data exists for collection [${flow_name}], cannot initialise`); } // Infer schema from flow. Specific data types will be included within flow._metadata, // and all other fields considered string - const schema = this.inferSchema(flowData.rows[0]); + const schema = this.inferSchema(flowData.rows[0], flowData._metadata); // Cached data will automatically be cast to correct data type from schema, // with any additional fields ignored const mergedData = this.mergeWriteCacheData(flow_type, flow_name, flowData.rows); - // HACK - rxdb can't write any fields prefixed with `_` so extract all to top-level APP_META key - const cleaned = mergedData.map((el) => this.extractMeta(el)); // add index property to each element before insert, for sorting queried data by original order - const data = cleaned.map((el) => ({ ...el, row_index: cleaned.indexOf(el) })); + const data = mergedData.map((el, i) => ({ ...el, row_index: i })); return { data, schema }; } @@ -252,11 +269,12 @@ export class DynamicDataService extends AsyncServiceBase { initialData: any[] ) { const writeData = this.writeCache.get(flow_type, flow_name) || {}; - const writeDataArray: IDocWithMeta[] = Object.entries(writeData).map(([id, v]) => ({ + const writeDataArray: FlowTypes.Data_listRow[] = Object.entries(writeData).map(([id, v]) => ({ ...v, id, })); const mergedData = this.mergeData(initialData, writeDataArray); + // TODO - how to preserve order when including user-generated writes (should just work...) return mergedData; } @@ -265,7 +283,7 @@ export class DynamicDataService extends AsyncServiceBase { return `${flow_type}${flow_name}`.toLowerCase().replace(/[^a-z0-9]/g, ""); } - private mergeData(flowData: T[] = [], dbData: T[] = []) { + private mergeData(flowData: T[] = [], dbData: T[] = []) { const flowHashmap = arrayToHashmap(flowData, "id"); const dbDataHashmap = arrayToHashmap(dbData, "id"); const merged = deepMergeObjects(flowHashmap, dbDataHashmap); @@ -288,8 +306,9 @@ export class DynamicDataService extends AsyncServiceBase { } const schema = REACTIVE_SCHEMA_BASE; for (const key of Object.keys(fields)) { - if (!schema.properties[key]) { - // assign any provided metadata, with fallback 'string' type if not specified + // assign any provided metadata, with fallback 'string' type if not specified + // ignore any `_` fields as these will be merged into APP_META (do not satisfy rxdb regex) + if (!schema.properties[key] && !key.startsWith("_")) { const type = metadata[key]?.type || "string"; schema.properties[key] = { ...metadata[key], type }; } @@ -329,29 +348,6 @@ export class DynamicDataService extends AsyncServiceBase { console.warn(`[SET ITEM] - No item ${_id ? "with ID " + _id : "at index " + _index}`); } } - - /** - * Iterate over a document's key-value pairs and populate any properties starting with - * an underscore to a single top-level APP_META property - */ - private extractMeta(doc: IDocWithMeta) { - const APP_META: Record = {}; - for (const [key, value] of Object.entries(doc)) { - if (key.startsWith("_")) { - APP_META[key] = value; - delete doc[key]; - } - } - if (Object.keys(APP_META).length > 0) { - doc.APP_META = APP_META; - } - return doc; - } - /** Populate any previously extracted APP_META properties back to document */ - private populateMeta(doc: IDocWithMeta) { - const { APP_META, ...data } = doc; - return { ...data, ...APP_META }; - } } /** the context for evaluating the target item to be updated, provided by the data-items component */ From c08c20c591bc991277066c5b60ad747b7ec64485 Mon Sep 17 00:00:00 2001 From: chrismclarke Date: Sat, 14 Dec 2024 12:11:26 -0800 Subject: [PATCH 07/10] refactor: set_data action code standalone --- .../components/data-items/data-items.utils.ts | 2 +- .../dynamic-data/actions/add_data.action.ts | 28 +++ .../services/dynamic-data/actions/index.ts | 97 +++++++++ .../set_data.action.spec.ts} | 98 +++------ .../dynamic-data/actions/set_data.action.ts | 101 +++++++++ .../dynamic-data/dynamic-data.actions.ts | 204 ------------------ .../dynamic-data/dynamic-data.service.ts | 5 +- 7 files changed, 260 insertions(+), 275 deletions(-) create mode 100644 src/app/shared/services/dynamic-data/actions/add_data.action.ts create mode 100644 src/app/shared/services/dynamic-data/actions/index.ts rename src/app/shared/services/dynamic-data/{dynamic-data.actions.spec.ts => actions/set_data.action.spec.ts} (68%) create mode 100644 src/app/shared/services/dynamic-data/actions/set_data.action.ts delete mode 100644 src/app/shared/services/dynamic-data/dynamic-data.actions.ts diff --git a/src/app/shared/components/template/components/data-items/data-items.utils.ts b/src/app/shared/components/template/components/data-items/data-items.utils.ts index 09d27bbf5d..44b5345a55 100644 --- a/src/app/shared/components/template/components/data-items/data-items.utils.ts +++ b/src/app/shared/components/template/components/data-items/data-items.utils.ts @@ -1,5 +1,5 @@ import { FlowTypes } from "packages/data-models"; -import { IActionRemoveDataParams } from "src/app/shared/services/dynamic-data/dynamic-data.actions"; +import { IActionRemoveDataParams } from "src/app/shared/services/dynamic-data/actions"; import { ISetItemContext } from "src/app/shared/services/dynamic-data/dynamic-data.service"; /** diff --git a/src/app/shared/services/dynamic-data/actions/add_data.action.ts b/src/app/shared/services/dynamic-data/actions/add_data.action.ts new file mode 100644 index 0000000000..489713029a --- /dev/null +++ b/src/app/shared/services/dynamic-data/actions/add_data.action.ts @@ -0,0 +1,28 @@ +import { generateUUID } from "src/app/shared/utils"; +import { coerceDataUpdateTypes, evaluateDynamicDataUpdate } from "../dynamic-data.utils"; +import { DynamicDataService } from "../dynamic-data.service"; + +export type IActionAddDataParams = { + _list_id: string; +} & Record; + +export default async (service: DynamicDataService, params: IActionAddDataParams) => { + const { _list_id, ...data } = await parseParams(params); + // assign a row_index to push user_generated docs to bottom of list + data.row_index = await service.getCount("data_list", _list_id); + // generate a unique id for the entry + data.id = generateUUID(); + // add metadata to track user created + data._user_created = true; + // HACK - use the same dynamic data evaluator as set_data action + // This requires passing an item list, so just create an ad-hoc list with a single item + // TODO - add support for evaluating @list reference to parent list + const [evaluated] = evaluateDynamicDataUpdate([{ id: data.id }], data); + const schema = service.getSchema("data_list", _list_id); + const [coerced] = coerceDataUpdateTypes(schema?.jsonSchema?.properties, [evaluated]); + return service.insert("data_list", _list_id, coerced); +}; + +async function parseParams(params) { + return params; +} diff --git a/src/app/shared/services/dynamic-data/actions/index.ts b/src/app/shared/services/dynamic-data/actions/index.ts new file mode 100644 index 0000000000..069059ee64 --- /dev/null +++ b/src/app/shared/services/dynamic-data/actions/index.ts @@ -0,0 +1,97 @@ +import { isObjectLiteral } from "packages/shared/src/utils/object-utils"; +import { + IActionHandler, + TemplateActionRegistry, +} from "../../../components/template/services/instance/template-action.registry"; +import type { DynamicDataService } from "../dynamic-data.service"; + +import addDataAction, { IActionAddDataParams } from "./add_data.action"; +import setDataAction, { IActionSetDataParams } from "./set_data.action"; + +interface IActionBaseParams { + _list_id?: string; + _index?: string | number; +} + +export type IActionRemoveDataParams = { + _list_id: string; + _id: string; +}; + +type IActionResetDataParams = { + _list_id: string; +}; + +export const registerDynamicDataActions = ( + service: DynamicDataService, + templateActionRegistry: TemplateActionRegistry +) => { + const { add_data, remove_data, reset_data, set_data } = new DynamicDataActionFactory(service); + return templateActionRegistry.register({ add_data, remove_data, reset_data, set_data }); +}; + +// Use action factory to make it easier to apply common logic to action params and pass service +class DynamicDataActionFactory { + constructor(private service: DynamicDataService) {} + + public set_data: IActionHandler = async ({ params }: { params?: IActionSetDataParams }) => { + const parsedParams = parseActionParams(params); + return setDataAction(this.service, parsedParams); + }; + + public reset_data: IActionHandler = async ({ params }: { params?: IActionResetDataParams }) => { + const { _list_id } = parseActionParams(params); + return this.service.resetFlow("data_list", _list_id); + }; + + public remove_data: IActionHandler = async ({ params }: { params?: IActionRemoveDataParams }) => { + const { _id, _list_id } = params; + return this.service.remove("data_list", _list_id, [_id]); + }; + + public add_data: IActionHandler = async ({ params }: { params?: IActionAddDataParams }) => { + const parsedParams = parseActionParams(params); + return addDataAction(this.service, parsedParams); + }; +} + +/** Parse action parameters to generate a list of updates to apply */ +function parseActionParams(params: T) { + if (isObjectLiteral(params)) { + const parsed = hackParseTemplatedParams(params); + if (!parsed._list_id) { + console.error(params); + throw new Error("[Data Actions] could not parse list id"); + } + return parsed; + } + + // throw error if args not parsed correctly + console.error(params); + throw new Error("[set_data] could not parse params"); +} + +/** + * Any params provided by templating system will already be partially parsed to replace dynamic references + * such as @local or @field. This also converts @item references to `this.item` + * Revert the item changes and also convert string parameter values to number where required + * + * TODO - this method should be removed and `items` made to update their own `this` context + **/ +function hackParseTemplatedParams(params: T) { + const parsed = {} as T; + // HACK - un-parse @item references that the templating system converts to `this.item` + for (const [key, value] of Object.entries(params)) { + if (typeof value === "string") { + parsed[key] = value.replace(/this\.item/g, "@item"); + } else { + parsed[key] = value; + } + } + // convert _index param which may be passed as string from template if defined inline + // NOTE - RXDB will automatically cast all other string values to correct type due to inferred schema + if (typeof params._index === "string") { + parsed._index = Number(params._index); + } + return parsed; +} diff --git a/src/app/shared/services/dynamic-data/dynamic-data.actions.spec.ts b/src/app/shared/services/dynamic-data/actions/set_data.action.spec.ts similarity index 68% rename from src/app/shared/services/dynamic-data/dynamic-data.actions.spec.ts rename to src/app/shared/services/dynamic-data/actions/set_data.action.spec.ts index fd87793be0..811fbe5c74 100644 --- a/src/app/shared/services/dynamic-data/dynamic-data.actions.spec.ts +++ b/src/app/shared/services/dynamic-data/actions/set_data.action.spec.ts @@ -1,17 +1,16 @@ import { TestBed } from "@angular/core/testing"; import { HttpClientTestingModule } from "@angular/common/http/testing"; -import { MockAppDataService } from "../data/app-data.service.mock.spec"; -import { AppDataService } from "../data/app-data.service"; +import { MockAppDataService } from "../../data/app-data.service.mock.spec"; +import { AppDataService } from "../../data/app-data.service"; -import { IActionSetDataParams } from "./dynamic-data.actions"; -import { DynamicDataService } from "./dynamic-data.service"; -import ActionFactory from "./dynamic-data.actions"; +import setDataAction, { IActionSetDataParams } from "./set_data.action"; +import { DynamicDataService } from "../dynamic-data.service"; import { firstValueFrom } from "rxjs"; import { FlowTypes } from "packages/data-models"; -import { DeploymentService } from "../deployment/deployment.service"; -import { MockDeploymentService } from "../deployment/deployment.service.spec"; -import { TemplateActionRegistry } from "../../components/template/services/instance/template-action.registry"; +import { DeploymentService } from "../../deployment/deployment.service"; +import { MockDeploymentService } from "../../deployment/deployment.service.spec"; +import { TemplateActionRegistry } from "../../../components/template/services/instance/template-action.registry"; type ITestRow = { id: string; number: number; string: string; _meta_field?: any }; @@ -37,38 +36,13 @@ const TEST_DATA_LIST = (): FlowTypes.Data_list => ({ * Test Utilities *******************************************************************************/ -/** Generate a rows to trigger set_data action with included params */ -function getTestActionRow( - params: IActionSetDataParams, - action_id: FlowTypes.TemplateRowAction["action_id"] -) { - params._list_id = "test_flow"; - const actionRow: FlowTypes.TemplateRowAction = { - action_id, - trigger: "click", - args: [], - params, - }; - return actionRow; -} - /** * Trigger the set_data action with included params and return first update * to corresponding data_list * * */ async function triggerTestSetDataAction(service: DynamicDataService, params: IActionSetDataParams) { - const actionRow = getTestActionRow(params, "set_data"); - const actions = new ActionFactory(service); - await actions.set_data(actionRow); - const obs = await service.query$("data_list", "test_flow"); - const data = await firstValueFrom(obs); - return data; -} - -async function triggerAddDataAction(service: DynamicDataService, params: IActionSetDataParams) { - const actionRow = getTestActionRow(params, "add_data"); - const actions = new ActionFactory(service); - await actions.add_data(actionRow); + params._list_id = "test_flow"; + await setDataAction(service, params); const obs = await service.query$("data_list", "test_flow"); const data = await firstValueFrom(obs); return data; @@ -76,11 +50,11 @@ async function triggerAddDataAction(service: DynamicDataService, params: IAction /******************************************************************************** * Tests - * yarn ng test --include src/app/shared/services/dynamic-data/dynamic-data.actions.spec.ts + * yarn ng test --include src/app/shared/services/dynamic-data/actions/set_data.action.spec.ts *******************************************************************************/ -describe("DynamicDataService Actions", () => { +describe("set_data Action", () => { let service: DynamicDataService; - let actions: ActionFactory; + let serviceUpdateSpy: jasmine.Spy; beforeEach(async () => { TestBed.configureTestingModule({ @@ -110,10 +84,12 @@ describe("DynamicDataService Actions", () => { window.global = window; service = TestBed.inject(DynamicDataService); + + serviceUpdateSpy = spyOn(service, "update").and.callThrough(); + await service.ready(); // Ensure any data previously persisted is cleared await service.resetFlow("data_list", "test_flow"); - actions = new ActionFactory(service); }); /************************************************************* @@ -156,8 +132,12 @@ describe("DynamicDataService Actions", () => { it("set_data ignores updates for unchanged data", async () => { const params: IActionSetDataParams = { _list_id: "test_flow", number: 1 }; - const updates = await actions["generateUpdateList"](params); - expect(updates).toEqual([{ id: "id_0", number: 1 }]); + await triggerTestSetDataAction(service, params); + // expect only 1 row to be updated (skip id_1 which has same number) + console.log("spy", serviceUpdateSpy.calls.all()); + expect(serviceUpdateSpy).toHaveBeenCalledOnceWith("data_list", "test_flow", "id_0", { + number: 1, + }); }); it("set_data prevents update to metadata fields", async () => { @@ -175,16 +155,6 @@ describe("DynamicDataService Actions", () => { expect(data[1].number).toEqual(1); }); - it("reset_data action restores data to initial", async () => { - const updatedData = await triggerTestSetDataAction(service, { string: "updated string" }); - expect(updatedData[0].string).toEqual("updated string"); - const resetActionBase = getTestActionRow({}, "reset_data"); - await actions.reset_data({ ...resetActionBase, action_id: "reset_data" }); - const obs = await service.query$("data_list", "test_flow"); - const resetData = await firstValueFrom(obs); - expect(resetData[0].string).toEqual("hello"); - }); - /************************************************************* * Misc ************************************************************/ @@ -201,27 +171,21 @@ describe("DynamicDataService Actions", () => { ************************************************************/ it("throws error if provided _id does not exist", async () => { - const params = getTestActionRow( - { - _id: "missing_id", - string: "sets an item correctly a given id", - }, - "set_data" - ); - await expectAsync(actions.set_data(params)).toBeRejectedWithError( + const params = { + _id: "missing_id", + string: "sets an item correctly a given id", + }; + await expectAsync(triggerTestSetDataAction(service, params)).toBeRejectedWithError( `[Update Fail] no doc exists\ndata_list: test_flow\n_id: missing_id` ); }); it("throws error if provided _index does not exist", async () => { - const params = getTestActionRow( - { - _index: 10, - string: "sets an item correctly a given id", - }, - "set_data" - ); - await expectAsync(actions.set_data(params)).toBeRejectedWithError( + const params = { + _index: 10, + string: "sets an item correctly a given id", + }; + await expectAsync(triggerTestSetDataAction(service, params)).toBeRejectedWithError( `[Update Fail] no doc exists\ndata_list: test_flow\n_index: 10` ); }); diff --git a/src/app/shared/services/dynamic-data/actions/set_data.action.ts b/src/app/shared/services/dynamic-data/actions/set_data.action.ts new file mode 100644 index 0000000000..931ec52f1d --- /dev/null +++ b/src/app/shared/services/dynamic-data/actions/set_data.action.ts @@ -0,0 +1,101 @@ +import { MangoQuery, RxSchema } from "rxdb"; +import { DynamicDataService } from "../dynamic-data.service"; +import { firstValueFrom } from "rxjs"; +import { FlowTypes } from "packages/data-models"; +import { + coerceDataUpdateTypes, + evaluateDynamicDataUpdate, + isItemChanged, +} from "../dynamic-data.utils"; + +interface IActionSetDataOperatorParams { + // TODO - ideally use same itemPipe operators + // (although filter will need to be updated to include dynamic context) + _filter?: string; + _limit?: number; + _reverse?: boolean; + _sort?: string; +} + +/** Metadata passed to set_data action to specify data for update **/ +interface IActionSetDataParamsMeta extends IActionSetDataOperatorParams { + /** Reference to source data_list id. All rows in list will be updated */ + _list_id?: string; + + /** ID of data_list row for single update */ + _id?: string; + + /** Index of data_list row for single update */ + _index?: number; + + /** List of compiled data updates if computed outside of action (e.g. data_items) */ + _updates?: FlowTypes.Data_listRow[]; +} + +/** Key-value pairs to update. These support reference to self `@item` context */ +export type IActionSetDataParams = IActionSetDataParamsMeta & Record; + +export default async (service: DynamicDataService, params: IActionSetDataParams) => { + // if called from set_item will already include list of updates to apply, if not generate + if (!params._updates) { + params._updates = await generateUpdateList(service, params); + } + const { _list_id, _updates } = params; + // Hack, no current method for bulk update so make successive (changes debounced in component) + for (const { id, ...writeableProps } of _updates) { + await service.update("data_list", _list_id, id, writeableProps); + } +}; + +async function generateUpdateList(service: DynamicDataService, params: IActionSetDataParams) { + // remove metadata from rest of update + const { _id, _index, _list_id, _updates, ...update } = params; + const query: MangoQuery = {}; + if (_id) { + query.selector = { id: _id }; + } + let ref = await service.query$("data_list", _list_id, query); + let items = await firstValueFrom(ref); + if (items.length === 0) { + const msg = `[Update Fail] no doc exists\ndata_list: ${_list_id}\n_id: ${_id}`; + throw new Error(msg); + } + // NOTE - RXDB doesn't support querying by index or pagination so still retrieve all and then reduce + if (typeof _index === "number") { + const targetItem = items[_index]; + if (!targetItem) { + const msg = `[Update Fail] no doc exists\ndata_list: ${_list_id}\n_index: ${_index}`; + throw new Error(msg); + } + items = [items[_index]]; + } + // Coerce updates to correct data types (inline parameter_list values parsed as strings) + const schema = service.getSchema("data_list", _list_id); + + return parseUpdateData(schema, update, items, _list_id); +} + +function parseUpdateData( + schema: RxSchema, + updateData: Record, + items: FlowTypes.Data_listRow[], + _list_id: string +) { + const cleanedUpdate = removeUpdateMetadata(updateData); + + // Evaluate item updates for any `@item` self-references + const evaluated = evaluateDynamicDataUpdate(items, cleanedUpdate); + + // Coerce updates to correct data types (inline parameter_list values parsed as strings) + const coerced = coerceDataUpdateTypes(schema?.jsonSchema?.properties, evaluated); + + // Filter to only include updates that will change original item + return coerced.filter((el, i) => isItemChanged(items[i], el)); +} + +function removeUpdateMetadata(update: Record) { + for (const key of Object.keys(update)) { + if (key.startsWith("_")) delete update[key]; + } + return update; +} diff --git a/src/app/shared/services/dynamic-data/dynamic-data.actions.ts b/src/app/shared/services/dynamic-data/dynamic-data.actions.ts deleted file mode 100644 index c88004a0d6..0000000000 --- a/src/app/shared/services/dynamic-data/dynamic-data.actions.ts +++ /dev/null @@ -1,204 +0,0 @@ -import { IActionHandler } from "../../components/template/services/instance/template-action.registry"; -import type { DynamicDataService } from "./dynamic-data.service"; -import { firstValueFrom } from "rxjs"; -import { isObjectLiteral } from "packages/shared/src/utils/object-utils"; -import { FlowTypes } from "packages/data-models"; -import { MangoQuery } from "rxdb"; -import { - coerceDataUpdateTypes, - evaluateDynamicDataUpdate, - isItemChanged, -} from "./dynamic-data.utils"; -import { generateUUID } from "../../utils"; - -interface IActionSetDataOperatorParams { - // TODO - ideally use same itemPipe operators - // (although filter will need to be updated to include dynamic context) - _filter?: string; - _limit?: number; - _reverse?: boolean; - _sort?: string; -} - -/** Metadata passed to set_data action to specify data for update **/ -interface IActionSetDataParamsMeta extends IActionSetDataOperatorParams { - /** Reference to source data_list id. All rows in list will be updated */ - _list_id?: string; - - /** ID of data_list row for single update */ - _id?: string; - - /** Index of data_list row for single update */ - _index?: number; - - /** List of compiled data updates if computed outside of action (e.g. data_items) */ - _updates?: FlowTypes.Data_listRow[]; -} - -/** Key-value pairs to update. These support reference to self `@item` context */ -export type IActionSetDataParams = IActionSetDataParamsMeta & Record; - -export type IActionRemoveDataParams = { - _list_id: string; - _id: string; -}; - -class DynamicDataActionFactory { - constructor(private service: DynamicDataService) {} - - /** - * Similar syntax can be used for multiple items. - * - * or from outside a loop can specify - * click | set_data | _list: @data.example_list, completed:true; - */ - public set_data: IActionHandler = async ({ params }: { params?: IActionSetDataParams }) => { - const parsed = await this.parseParams(params); - // if called from set_item will already include list of updates to apply, if not generate - if (!parsed._updates) { - parsed._updates = await this.generateUpdateList(parsed); - } - const { _list_id, _updates } = parsed; - // Hack, no current method for bulk update so make successive (changes debounced in component) - for (const { id, ...writeableProps } of _updates) { - await this.service.update("data_list", _list_id, id, writeableProps); - } - }; - - public reset_data: IActionHandler = async ({ params }: { params?: IActionSetDataParams }) => { - const { _list_id } = await this.parseParams(params); - return this.service.resetFlow("data_list", _list_id); - }; - - public remove_data: IActionHandler = async ({ params }: { params?: IActionRemoveDataParams }) => { - const { _id, _list_id } = params; - return this.service.remove("data_list", _list_id, [_id]); - }; - - public add_data: IActionHandler = async ({ params }: { params?: IActionSetDataParams }) => { - const { _list_id, ...data } = await this.parseParams(params); - // assign a row_index to push user_generated docs to bottom of list - let row_index = await this.service.getCount("data_list", _list_id); - data.id = generateUUID(); - // add metadata to track user created - data._user_created = true; - data.row_index = row_index; - // HACK - use the same dynamic data evaluator as set_data action - // This requires passing an item list, so just create an ad-hoc list with a single item - const [evaluated] = evaluateDynamicDataUpdate([{ id: data.id }], data); - // TODO - add support for evaluating @list statements - const schema = this.service.getSchema("data_list", _list_id); - const [coerced] = coerceDataUpdateTypes(schema?.jsonSchema?.properties, [evaluated]); - return this.service.insert("data_list", _list_id, coerced); - }; - - /** Parse action parameters to generate a list of updates to apply */ - private async parseParams(params: IActionSetDataParams) { - if (isObjectLiteral(params)) { - const parsed = this.hackParseTemplatedParams(params); - if (!parsed._list_id) { - console.error(params); - throw new Error("[Data Actions] could not parse list id"); - } - return parsed; - } - - // throw error if args not parsed correctly - console.error(params); - throw new Error("[set_data] could not parse params"); - } - - private async generateUpdateList(params: IActionSetDataParams) { - // remove metadata from rest of update - const { _id, _index, _list_id, _updates, ...update } = params; - const query: MangoQuery = {}; - if (_id) { - query.selector = { id: _id }; - } - let ref = await this.service.query$("data_list", _list_id, query); - let items = await firstValueFrom(ref); - if (items.length === 0) { - const msg = `[Update Fail] no doc exists\ndata_list: ${_list_id}\n_id: ${_id}`; - throw new Error(msg); - } - // NOTE - RXDB doesn't support querying by index or pagination so still retrieve all and then reduce - if (typeof _index === "number") { - const targetItem = items[_index]; - if (!targetItem) { - const msg = `[Update Fail] no doc exists\ndata_list: ${_list_id}\n_index: ${_index}`; - throw new Error(msg); - } - items = [items[_index]]; - } - - return this.parseUpdateData(update, items, _list_id); - } - - private parseUpdateData( - updateData: Record, - items: FlowTypes.Data_listRow[], - _list_id: string - ) { - const cleanedUpdate = this.removeUpdateMetadata(updateData); - - // Evaluate item updates for any `@item` self-references - const evaluated = evaluateDynamicDataUpdate(items, cleanedUpdate); - - // Coerce updates to correct data types (inline parameter_list values parsed as strings) - const schema = this.service.getSchema("data_list", _list_id); - const coerced = coerceDataUpdateTypes(schema?.jsonSchema?.properties, evaluated); - - // Filter to only include updates that will change original item - return coerced.filter((el, i) => isItemChanged(items[i], el)); - } - - private applyUpdateOperations(params: IActionSetDataParams) { - // TODO - decide when to evaluate... e.g. filter: @item.id > @local.value - const { _filter, _reverse, _sort, _limit } = params; - const parsedOps: IActionSetDataOperatorParams = {}; - for (const [operator, arg] of Object.entries({ _filter, _reverse, _sort, _limit })) { - if (arg !== undefined) { - let parsedArg = ""; - // TODO - understand if dynamic or numeric string... - // TODO - better to convert these params in parser not at runtime!!! (same for _index previously) - // using some sort of parseNumericalString or similar regex lookup... same for boolean strings and other similar - if (operator === "_limit" && typeof parsedArg === "string") { - } - } - } - } - - private removeUpdateMetadata(update: Record) { - for (const key of Object.keys(update)) { - if (key.startsWith("_")) delete update[key]; - } - return update; - } - - /** - * Any params provided by templating system will already be partially parsed to replace dynamic references - * such as @local or @field. This also converts @item references to `this.item` - * Revert the item changes and also convert string parameter values to number where required - * - * TODO - this method should be removed and `items` made to update their own `this` context - **/ - private hackParseTemplatedParams(params: IActionSetDataParams) { - const parsed: IActionSetDataParams = {}; - // HACK - un-parse @item references that the templating system converts to `this.item` - for (const [key, value] of Object.entries(params)) { - if (typeof value === "string") { - parsed[key] = value.replace(/this\.item/g, "@item"); - } else { - parsed[key] = value; - } - } - // convert _index param which may be passed as string from template if defined inline - // NOTE - RXDB will automatically cast all other string values to correct type due to inferred schema - if (typeof params._index === "string") { - parsed._index = Number(params._index); - } - return parsed; - } -} - -export default DynamicDataActionFactory; diff --git a/src/app/shared/services/dynamic-data/dynamic-data.service.ts b/src/app/shared/services/dynamic-data/dynamic-data.service.ts index 537c6a0fcb..4ddbc89ae1 100644 --- a/src/app/shared/services/dynamic-data/dynamic-data.service.ts +++ b/src/app/shared/services/dynamic-data/dynamic-data.service.ts @@ -10,7 +10,7 @@ import { arrayToHashmap, deepMergeObjects } from "../../utils"; import { PersistedMemoryAdapter } from "./adapters/persistedMemory"; import { ReactiveMemoryAdapter, REACTIVE_SCHEMA_BASE } from "./adapters/reactiveMemory"; import { TemplateActionRegistry } from "../../components/template/services/instance/template-action.registry"; -import ActionsFactory from "./dynamic-data.actions"; +import { registerDynamicDataActions } from "./actions"; import { DeploymentService } from "../deployment/deployment.service"; @Injectable({ providedIn: "root" }) @@ -51,8 +51,7 @@ export class DynamicDataService extends AsyncServiceBase { super("Dynamic Data"); this.registerInitFunction(this.initialise); // register action handlers - const { set_data, reset_data, add_data, remove_data } = new ActionsFactory(this); - this.templateActionRegistry.register({ set_data, reset_data, add_data, remove_data }); + registerDynamicDataActions(this, templateActionRegistry); // HACK - Legacy `set_item` action still managed here (will be removed in #2454) this.registerLegacyItemsActions(); } From d46019a41690a8c53cfc199bf436f1cf233185af Mon Sep 17 00:00:00 2001 From: chrismclarke Date: Sat, 14 Dec 2024 12:56:32 -0800 Subject: [PATCH 08/10] chore: code tidying --- .../shared/services/dynamic-data/actions/index.ts | 15 ++------------- .../services/dynamic-data/dynamic-data.service.ts | 5 +++-- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/src/app/shared/services/dynamic-data/actions/index.ts b/src/app/shared/services/dynamic-data/actions/index.ts index 069059ee64..fab5169438 100644 --- a/src/app/shared/services/dynamic-data/actions/index.ts +++ b/src/app/shared/services/dynamic-data/actions/index.ts @@ -1,8 +1,5 @@ import { isObjectLiteral } from "packages/shared/src/utils/object-utils"; -import { - IActionHandler, - TemplateActionRegistry, -} from "../../../components/template/services/instance/template-action.registry"; +import { IActionHandler } from "../../../components/template/services/instance/template-action.registry"; import type { DynamicDataService } from "../dynamic-data.service"; import addDataAction, { IActionAddDataParams } from "./add_data.action"; @@ -22,16 +19,8 @@ type IActionResetDataParams = { _list_id: string; }; -export const registerDynamicDataActions = ( - service: DynamicDataService, - templateActionRegistry: TemplateActionRegistry -) => { - const { add_data, remove_data, reset_data, set_data } = new DynamicDataActionFactory(service); - return templateActionRegistry.register({ add_data, remove_data, reset_data, set_data }); -}; - // Use action factory to make it easier to apply common logic to action params and pass service -class DynamicDataActionFactory { +export class DynamicDataActionFactory { constructor(private service: DynamicDataService) {} public set_data: IActionHandler = async ({ params }: { params?: IActionSetDataParams }) => { diff --git a/src/app/shared/services/dynamic-data/dynamic-data.service.ts b/src/app/shared/services/dynamic-data/dynamic-data.service.ts index 4ddbc89ae1..078943a381 100644 --- a/src/app/shared/services/dynamic-data/dynamic-data.service.ts +++ b/src/app/shared/services/dynamic-data/dynamic-data.service.ts @@ -10,7 +10,7 @@ import { arrayToHashmap, deepMergeObjects } from "../../utils"; import { PersistedMemoryAdapter } from "./adapters/persistedMemory"; import { ReactiveMemoryAdapter, REACTIVE_SCHEMA_BASE } from "./adapters/reactiveMemory"; import { TemplateActionRegistry } from "../../components/template/services/instance/template-action.registry"; -import { registerDynamicDataActions } from "./actions"; +import { DynamicDataActionFactory } from "./actions"; import { DeploymentService } from "../deployment/deployment.service"; @Injectable({ providedIn: "root" }) @@ -51,7 +51,8 @@ export class DynamicDataService extends AsyncServiceBase { super("Dynamic Data"); this.registerInitFunction(this.initialise); // register action handlers - registerDynamicDataActions(this, templateActionRegistry); + const { add_data, remove_data, reset_data, set_data } = new DynamicDataActionFactory(this); + this.templateActionRegistry.register({ add_data, remove_data, reset_data, set_data }); // HACK - Legacy `set_item` action still managed here (will be removed in #2454) this.registerLegacyItemsActions(); } From 47eb15a0dfffacd4c6e438dcd47792accfe3d7e5 Mon Sep 17 00:00:00 2001 From: chrismclarke Date: Sat, 14 Dec 2024 13:00:38 -0800 Subject: [PATCH 09/10] chore: update set_data tests --- .../dynamic-data/actions/set_data.action.spec.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/app/shared/services/dynamic-data/actions/set_data.action.spec.ts b/src/app/shared/services/dynamic-data/actions/set_data.action.spec.ts index 811fbe5c74..a314362086 100644 --- a/src/app/shared/services/dynamic-data/actions/set_data.action.spec.ts +++ b/src/app/shared/services/dynamic-data/actions/set_data.action.spec.ts @@ -11,6 +11,7 @@ import { FlowTypes } from "packages/data-models"; import { DeploymentService } from "../../deployment/deployment.service"; import { MockDeploymentService } from "../../deployment/deployment.service.spec"; import { TemplateActionRegistry } from "../../../components/template/services/instance/template-action.registry"; +import { DynamicDataActionFactory } from "./index"; type ITestRow = { id: string; number: number; string: string; _meta_field?: any }; @@ -155,6 +156,21 @@ describe("set_data Action", () => { expect(data[1].number).toEqual(1); }); + it("reset_data action restores data to initial", async () => { + const updatedData = await triggerTestSetDataAction(service, { string: "updated string" }); + expect(updatedData[0].string).toEqual("updated string"); + const { reset_data } = new DynamicDataActionFactory(service); + await reset_data({ + action_id: "reset_data", + args: [], + trigger: "click", + params: { _list_id: "test_flow" }, + }); + const obs = await service.query$("data_list", "test_flow"); + const resetData = await firstValueFrom(obs); + expect(resetData[0].string).toEqual("hello"); + }); + /************************************************************* * Misc ************************************************************/ From fc113ee628f0570aa82e7bf206787e3ac926d524 Mon Sep 17 00:00:00 2001 From: chrismclarke Date: Sat, 14 Dec 2024 17:12:49 -0800 Subject: [PATCH 10/10] test: add_data action --- .../actions/add_data.action.spec.ts | 138 ++++++++++++++++++ .../dynamic-data/actions/add_data.action.ts | 5 +- .../services/dynamic-data/actions/index.ts | 32 ++-- .../dynamic-data/actions/set_data.action.ts | 2 +- .../dynamic-data/dynamic-data.service.ts | 14 +- 5 files changed, 164 insertions(+), 27 deletions(-) create mode 100644 src/app/shared/services/dynamic-data/actions/add_data.action.spec.ts diff --git a/src/app/shared/services/dynamic-data/actions/add_data.action.spec.ts b/src/app/shared/services/dynamic-data/actions/add_data.action.spec.ts new file mode 100644 index 0000000000..f0498b191b --- /dev/null +++ b/src/app/shared/services/dynamic-data/actions/add_data.action.spec.ts @@ -0,0 +1,138 @@ +import { TestBed } from "@angular/core/testing"; + +import { HttpClientTestingModule } from "@angular/common/http/testing"; +import { MockAppDataService } from "../../data/app-data.service.mock.spec"; +import { AppDataService } from "../../data/app-data.service"; + +import addDataAction, { IActionAddDataParams } from "./add_data.action"; +import { DynamicDataService } from "../dynamic-data.service"; +import { firstValueFrom } from "rxjs"; +import { FlowTypes } from "packages/data-models"; +import { DeploymentService } from "../../deployment/deployment.service"; +import { MockDeploymentService } from "../../deployment/deployment.service.spec"; +import { TemplateActionRegistry } from "../../../components/template/services/instance/template-action.registry"; +import { DynamicDataActionFactory, IActionRemoveDataParams } from "./index"; + +type ITestRow = { id: string; number: number; string: string; _meta_field?: any }; + +const TEST_DATA_ROWS = (): FlowTypes.Data_listRow[] => [ + { id: "id_0", number: 0, string: "hello", _meta_field: "original" }, + { id: "id_1", number: 1, string: "hello", _meta_field: "original" }, +]; + +const TEST_DATA_LIST = (): FlowTypes.Data_list => ({ + flow_name: "test_flow", + flow_type: "data_list", + // Make deep clone of data to avoid data overwrite issues + rows: TEST_DATA_ROWS(), + // Metadata would be extracted from parser based on data or defined schema + _metadata: { + boolean: { type: "boolean" }, + number: { type: "number" }, + _meta_field: { type: "object" }, + }, +}); + +/******************************************************************************** + * Test Utilities + *******************************************************************************/ + +/** + * Trigger the set_data action with included params and return first update + * to corresponding data_list + * * */ +async function triggerTestAddDataAction( + service: DynamicDataService, + params: Partial +) { + await addDataAction(service, { ...params, _list_id: "test_flow" }); + const obs = await service.query$("data_list", "test_flow"); + const data = await firstValueFrom(obs); + return data; +} + +async function triggerRemoveDataAction( + service: DynamicDataService, + params: Partial +) { + const { remove_data } = new DynamicDataActionFactory(service); + await remove_data({ + action_id: "remove_data", + args: [], + trigger: "click", + params: { ...params, _list_id: "test_flow" }, + }); + const obs = await service.query$("data_list", "test_flow"); + const data = await firstValueFrom(obs); + return data; +} + +/******************************************************************************** + * Tests + * yarn ng test --include src/app/shared/services/dynamic-data/actions/add_data.action.spec.ts + *******************************************************************************/ +describe("set_data Action", () => { + let service: DynamicDataService; + + beforeEach(async () => { + TestBed.configureTestingModule({ + imports: [HttpClientTestingModule], + providers: [ + DynamicDataService, + { + provide: AppDataService, + useValue: new MockAppDataService({ + data_list: { + test_flow: TEST_DATA_LIST(), + }, + }), + }, + { + provide: DeploymentService, + useValue: new MockDeploymentService({ name: "test" }), + }, + { + provide: TemplateActionRegistry, + useValue: { register: () => null }, + }, + ], + }); + + // HACK - polyfill not loaded for rxdb dev plugin so manually fill global before running tests + window.global = window; + service = TestBed.inject(DynamicDataService); + await service.ready(); + // ensure data cleared between flows + await service.resetFlow("data_list", "test_flow"); + }); + + /************************************************************* + * Main Tests + ************************************************************/ + it("add_data", async () => { + const data = await triggerTestAddDataAction(service, { number: 2 }); + expect(data.length).toEqual(3); + const addedData = data[2]; + expect(addedData.number).toEqual(2); + // populated metadata + expect(addedData._user_created).toEqual(true); + expect(addedData.row_index).toEqual(2); + expect(addedData.id).toBeTruthy(); + }); + + it("add_data type coercion", async () => { + const data = await triggerTestAddDataAction(service, { number: "2" }); + const addedData = data[2]; + expect(addedData.number).toEqual(2); + }); + + it("remove_data", async () => { + const data = await triggerTestAddDataAction(service, { number: 2 }); + const addedData = data[2]; + expect(addedData.number).toEqual(2); + const updatedData = await triggerRemoveDataAction(service, { _id: addedData.id }); + expect(updatedData.length).toEqual(2); + }); + + // TODO - @list evaluation (when supported) +}); diff --git a/src/app/shared/services/dynamic-data/actions/add_data.action.ts b/src/app/shared/services/dynamic-data/actions/add_data.action.ts index 489713029a..cc6f7adf88 100644 --- a/src/app/shared/services/dynamic-data/actions/add_data.action.ts +++ b/src/app/shared/services/dynamic-data/actions/add_data.action.ts @@ -8,6 +8,9 @@ export type IActionAddDataParams = { export default async (service: DynamicDataService, params: IActionAddDataParams) => { const { _list_id, ...data } = await parseParams(params); + + const schema = await service.getSchema("data_list", _list_id); + // assign a row_index to push user_generated docs to bottom of list data.row_index = await service.getCount("data_list", _list_id); // generate a unique id for the entry @@ -18,7 +21,7 @@ export default async (service: DynamicDataService, params: IActionAddDataParams) // This requires passing an item list, so just create an ad-hoc list with a single item // TODO - add support for evaluating @list reference to parent list const [evaluated] = evaluateDynamicDataUpdate([{ id: data.id }], data); - const schema = service.getSchema("data_list", _list_id); + const [coerced] = coerceDataUpdateTypes(schema?.jsonSchema?.properties, [evaluated]); return service.insert("data_list", _list_id, coerced); }; diff --git a/src/app/shared/services/dynamic-data/actions/index.ts b/src/app/shared/services/dynamic-data/actions/index.ts index fab5169438..432ecdc23e 100644 --- a/src/app/shared/services/dynamic-data/actions/index.ts +++ b/src/app/shared/services/dynamic-data/actions/index.ts @@ -24,12 +24,12 @@ export class DynamicDataActionFactory { constructor(private service: DynamicDataService) {} public set_data: IActionHandler = async ({ params }: { params?: IActionSetDataParams }) => { - const parsedParams = parseActionParams(params); + const parsedParams = this.parseActionParams(params); return setDataAction(this.service, parsedParams); }; public reset_data: IActionHandler = async ({ params }: { params?: IActionResetDataParams }) => { - const { _list_id } = parseActionParams(params); + const { _list_id } = this.parseActionParams(params); return this.service.resetFlow("data_list", _list_id); }; @@ -39,25 +39,25 @@ export class DynamicDataActionFactory { }; public add_data: IActionHandler = async ({ params }: { params?: IActionAddDataParams }) => { - const parsedParams = parseActionParams(params); + const parsedParams = this.parseActionParams(params); return addDataAction(this.service, parsedParams); }; -} -/** Parse action parameters to generate a list of updates to apply */ -function parseActionParams(params: T) { - if (isObjectLiteral(params)) { - const parsed = hackParseTemplatedParams(params); - if (!parsed._list_id) { - console.error(params); - throw new Error("[Data Actions] could not parse list id"); + /** Parse action parameters to ensure list id provided and */ + private parseActionParams(params: T) { + if (isObjectLiteral(params)) { + const parsed = hackParseTemplatedParams(params); + if (!parsed._list_id) { + console.error(params); + throw new Error("[Data Actions] could not parse list id"); + } + return parsed; } - return parsed; - } - // throw error if args not parsed correctly - console.error(params); - throw new Error("[set_data] could not parse params"); + // throw error if args not parsed correctly + console.error(params); + throw new Error("[set_data] could not parse params"); + } } /** diff --git a/src/app/shared/services/dynamic-data/actions/set_data.action.ts b/src/app/shared/services/dynamic-data/actions/set_data.action.ts index 931ec52f1d..bf1ec9cb15 100644 --- a/src/app/shared/services/dynamic-data/actions/set_data.action.ts +++ b/src/app/shared/services/dynamic-data/actions/set_data.action.ts @@ -70,7 +70,7 @@ async function generateUpdateList(service: DynamicDataService, params: IActionSe items = [items[_index]]; } // Coerce updates to correct data types (inline parameter_list values parsed as strings) - const schema = service.getSchema("data_list", _list_id); + const schema = await service.getSchema("data_list", _list_id); return parseUpdateData(schema, update, items, _list_id); } diff --git a/src/app/shared/services/dynamic-data/dynamic-data.service.ts b/src/app/shared/services/dynamic-data/dynamic-data.service.ts index 078943a381..78933b234c 100644 --- a/src/app/shared/services/dynamic-data/dynamic-data.service.ts +++ b/src/app/shared/services/dynamic-data/dynamic-data.service.ts @@ -155,16 +155,11 @@ export class DynamicDataService extends AsyncServiceBase { ) { const { collectionName } = await this.ensureCollection(flow_type, flow_name); const { id } = data; - // TODO - remove metadata - const res = await this.db.bulkInsert(collectionName, [data]); - // TODO - generating row_id and row_index - // - possibly just use 0 index for now and can re-populate later after retrieval (?) - // TODO - track whether user generated/owned? - // TODO - check what happens after data_list updated (user entries retained?) + await this.db.bulkInsert(collectionName, [data]); this.writeCache.update({ flow_type, flow_name, id, data }); - // TODO - what if id already exists? } + /** Remove user_generated data row */ public async remove(flow_type: FlowTypes.FlowType, flow_name: string, ids: string[]) { const { collectionName } = await this.ensureCollection(flow_type, flow_name); const collection = this.db.getCollection(collectionName); @@ -203,8 +198,9 @@ export class DynamicDataService extends AsyncServiceBase { return this.writeCache.state; } - public getSchema(flow_type: FlowTypes.FlowType, flow_name: string) { - const collectionName = this.normaliseCollectionName(flow_type, flow_name); + public async getSchema(flow_type: FlowTypes.FlowType, flow_name: string) { + // ensure collection has been created when accessing schema + const { collectionName } = await this.ensureCollection(flow_type, flow_name); return this.db.getCollection(collectionName)?.schema; }