From 13edbb7d80144dc25de834c08cf66edc30d02384 Mon Sep 17 00:00:00 2001 From: Geoffrey Hendrey Date: Sat, 2 Nov 2024 16:15:32 -0700 Subject: [PATCH] refactor so that takes options --- README.md | 95 +++++++++++++++-- example/generate.json | 2 +- example/myGenerator.mjs | 3 +- example/myGenerator3.yaml | 8 ++ example/myGenerator4.yaml | 3 + example/myGeneratorVerbose.json | 3 + src/TemplateProcessor.ts | 13 ++- src/test/TemplateProcessor.test.js | 33 +++++- src/utils/GeneratorManager.ts | 157 +++++++++++++++++------------ 9 files changed, 233 insertions(+), 84 deletions(-) create mode 100644 example/myGenerator3.yaml create mode 100644 example/myGenerator4.yaml create mode 100644 example/myGeneratorVerbose.json diff --git a/README.md b/README.md index 25e06cf2..1524e8ec 100644 --- a/README.md +++ b/README.md @@ -1763,14 +1763,35 @@ keep in mind that all the values were sequentially pushed into the `generated` f "generated": 10 } ``` +If you are familiar with generators in JS, you know that generated values take a verbose form in which +a `{value, done}` object is [returned](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/AsyncGenerator/next#return_value). +If you wish to use the more verbose JS style of yielded/returned object, you can pass your generator +to `$generate` and disable `valuesOnly`, as follows. Notice how the yielded values now contain the JS style +`{value, done}` +```json +> .init -f example/myGeneratorVerbose.json --xf example/myGenerator.mjs +{ + "generated": "${$myGenerator()~>$generate({'valueOnly':false})}" +} +> .out +{ + "generated": { + "value": 10, + "done": true, + "return": "{function:}" + } +} +``` + A slight variation on the example accumulates every value yielded by the generator: ```json -> .init -f example/myGenerator2.json --xf example/myGenerator.mjs +> .init -f example/myGenerator2.json --xf example/myGenerator.mjs { - "generated": "${$myGenerator()}", - "onGenerated": "${$set('/accumulator/-', $$.generated)}", - "accumulator": [] + "generated": "${$myGenerator()}", + "onGenerated": "${$set('/accumulator/-', $$.generated)}", + "accumulator": [] } + ``` ```json ["data=[1,2,3,4,5,6,7,8,9,10]"] > .init -f example/myGenerator2.json --xf example/myGenerator.mjs --tail "/accumulator until $=[1,2,3,4,5,6,7,8,9,10]" @@ -1789,14 +1810,15 @@ Started tailing... Press Ctrl+C to stop. ] ``` -Or, you can use the built in `$generate` method, which takes an optional delay in ms, and turns the array -into an AsyncGenerator. In the example below the values 1 to 10 are pumped into the `generated` field -with 10 ms temporal separation. +You already saw hoe the built-in `$generate` function can accept a JS AsyncGeneraotr, and options. But $generate +can also be used to convert ordinary arrays or functions into async generators. When provided, the `interval` option, +causes the provided array to yield its elements periodically. When a function is provided, as opposed to an array, the +function is called periodically. ```json > .init -f example/generate.json { "delayMs": 250, - "generated":"${[1..10]~>$generate(delayMs)}" + "generated":"${[1..10]~>$generate({'interval':delayMs})}" } ``` ```json ["data.generated=10"] @@ -1808,6 +1830,63 @@ Started tailing... Press Ctrl+C to stop. } ``` +This `example/myGenerator3.yaml` shows how you can call `return` and stop a generator. +```yaml +generated: ${$generate($random, {'interval':10, 'valueOnly':false})} +onGenerated: | + ${ + $count(accumulator)<3 + ? $set('/accumulator/-', $$.generated.value) + : generated.return() /* shut off the generator when the accumulator has 10 items */ + } +accumulator: [] +``` +```json ["$count(data.accumulator)=3"] +> .init -f example/myGenerator3.yaml --tail "/ until $count(accumulator)=3" +Started tailing... Press Ctrl+C to stop. +{ + "generated": { + "value": 0.23433826655570145, + "done": false, + "return": "{function:}" + }, + "onGenerated": { + "value": null, + "done": true + }, + "accumulator": [ + 0.23433826655570145, + 0.23433826655570145, + 0.23433826655570145 + ] +} + + +``` +The `maxYield` parameter can also be used to stop a generator: +```json ["$count(data.accumulator)=0", "$count(data.accumulator)=5"] +> .init -f example/myGenerator4.yaml +{ + "generated": "${$generate($random, {'interval':10, 'maxYield':5})}", + "onGenerated": "${$set('/accumulator/-', $$.generated)}", + "accumulator": [] +} +> .init -f example/myGenerator4.yaml --tail "/ until $count(accumulator)=5" +{ + "generated": 0.5289126250886866, + "onGenerated": [ + "/accumulator/-" + ], + "accumulator": [ + 0.3260049204634301, + 0.4477190160739559, + 0.9414436597923774, + 0.8593436891141426, + 0.5289126250886866 + ] +} +``` + ### $setTimeout `$setTimeout` is the JavaScript `setTimeout` function. It receives a function and an timeout diff --git a/example/generate.json b/example/generate.json index 0862c6d0..f93f2dc3 100644 --- a/example/generate.json +++ b/example/generate.json @@ -1,4 +1,4 @@ { "delayMs": 250, - "generated":"${[1..10]~>$generate(delayMs)}" + "generated":"${[1..10]~>$generate({'interval':delayMs})}" } \ No newline at end of file diff --git a/example/myGenerator.mjs b/example/myGenerator.mjs index ab507f2f..b652b36e 100644 --- a/example/myGenerator.mjs +++ b/example/myGenerator.mjs @@ -1,5 +1,6 @@ export async function* myGenerator() { - for (let i = 1; i <= 10; i++) { + for (let i = 1; i < 10; i++) { yield i; } + return 10; // Last value with `done: true` } \ No newline at end of file diff --git a/example/myGenerator3.yaml b/example/myGenerator3.yaml new file mode 100644 index 00000000..6d1b73e3 --- /dev/null +++ b/example/myGenerator3.yaml @@ -0,0 +1,8 @@ +generated: ${$generate($random, {'interval':10, 'valueOnly':false})} +onGenerated: | + ${ + $count(accumulator)<3 + ? $set('/accumulator/-', $$.generated.value) + : generated.return() /* shut off the generator when the accumulator has 10 items */ + } +accumulator: [] diff --git a/example/myGenerator4.yaml b/example/myGenerator4.yaml new file mode 100644 index 00000000..e79ae1c7 --- /dev/null +++ b/example/myGenerator4.yaml @@ -0,0 +1,3 @@ +generated: ${$generate($random, {'interval':10, 'maxYield':5})} +onGenerated: ${$set('/accumulator/-', $$.generated)} +accumulator: [] diff --git a/example/myGeneratorVerbose.json b/example/myGeneratorVerbose.json new file mode 100644 index 00000000..94ac2107 --- /dev/null +++ b/example/myGeneratorVerbose.json @@ -0,0 +1,3 @@ +{ + "generated": "${$myGenerator()~>$generate({'valueOnly':false})}" +} \ No newline at end of file diff --git a/src/TemplateProcessor.ts b/src/TemplateProcessor.ts index 6e652803..a2181818 100644 --- a/src/TemplateProcessor.ts +++ b/src/TemplateProcessor.ts @@ -1633,7 +1633,7 @@ export default class TemplateProcessor { context ); if (evaluated?._jsonata_lambda) { - evaluated = this.wrapInOrdinaryFunction(evaluated); + evaluated = TemplateProcessor.wrapInOrdinaryFunction(evaluated); metaInfo.isFunction__ = true; } } catch (error: any) { @@ -1647,9 +1647,9 @@ export default class TemplateProcessor { _error.name = "JSONata evaluation exception"; throw _error; } - if (GeneratorManager.isGenerator(evaluated)) { - //returns the first item, and begins pumping remaining items into execution queue - evaluated = this.generatorManager.pumpItems(evaluated as AsyncGenerator, metaInfo, this); + if (GeneratorManager.isAsyncGenerator(evaluated)) { + //awaits and returns the first item. And pumpItems begins pumping remaining items into execution queue asynchronously + evaluated = await this.generatorManager.pumpItems(evaluated as AsyncGenerator, metaInfo, this); } return evaluated @@ -1869,7 +1869,6 @@ export default class TemplateProcessor { if (callbacks) { const promises = Array.from(callbacks).map(cbFn => Promise.resolve().then(() => { - //to do ... return here so asyn functions are actually awaited by Promise.all cbFn(data, jsonPointer as JsonPointerString, removed, op); }) //works with cbFn that is either sync or async by wrapping in promise ); @@ -1934,7 +1933,7 @@ export default class TemplateProcessor { } } - private wrapInOrdinaryFunction(jsonataLambda:any) { + public static wrapInOrdinaryFunction(jsonataLambda:any) { const wrappedFunction = (...args:any[])=> { // Call the 'apply' method of jsonataLambda with the captured arguments return jsonataLambda.apply(jsonataLambda, args); @@ -2062,7 +2061,7 @@ export default class TemplateProcessor { forkId: TemplateProcessor.simpleUniqueId() }; //do not await setData...$forked runs async - this.setDataForked (mvccSnapshotPlanStep); + void this.setDataForked (mvccSnapshotPlanStep); } } diff --git a/src/test/TemplateProcessor.test.js b/src/test/TemplateProcessor.test.js index 9a6a7df2..6b550486 100644 --- a/src/test/TemplateProcessor.test.js +++ b/src/test/TemplateProcessor.test.js @@ -3112,8 +3112,8 @@ test("test close", async () => { test("test generate array", async () => { const o = { - "delayMs": 10, - "a":"${[1..10]~>$generate(delayMs)}", + "options": {"interval":10, "valueOnly":true}, + "a":"${[1..10]~>$generate(options)}", "b": "${a}" }; @@ -3197,6 +3197,35 @@ test("test generate function result", async () => { } }); +test("test generate verbose function result", async () => { + const o = { + "a":"${$generate(function(){10}, {'valueOnly':false})}", + "b": "${a}" + }; + + let resolvePromise; + const allCallsMade = new Promise((resolve) => { + resolvePromise = resolve; + }); + + const changeHandler = jest.fn((data, ptr) => { + expect(ptr).toBe("/b"); // Ensure correct pointer + resolvePromise(); // Resolve the promise when callCount is reached + }); + + const tp = new TemplateProcessor(o); + tp.setDataChangeCallback('/b', changeHandler); + try { + await tp.initialize(); + await allCallsMade; + expect(changeHandler).toHaveBeenCalledTimes(1); + expect(tp.output.b).toMatchObject({value:10, done:true}); + expect(tp.output.b.return).toBeDefined(); //make sure 'return' function is provided + } finally { + await tp.close(); + } +}); + test("test lifecycle manager", async () => { const o = { diff --git a/src/utils/GeneratorManager.ts b/src/utils/GeneratorManager.ts index 181e40a5..1b36d87c 100644 --- a/src/utils/GeneratorManager.ts +++ b/src/utils/GeneratorManager.ts @@ -9,68 +9,84 @@ export class GeneratorManager{ } /** - * Generates an asynchronous generator that yields items from the provided generateMe, + * Generates an asynchronous generator that yields items from the provided input, * separated by the specified timeout, and automatically registers the generator. * - * @param generateMe The item or array of items to yield. If generate me is a function, the function is called to + * @param input The item or array of items to yield. If generate me is a function, the function is called to * get a result, and recursively passed to generate() - * @param timeout The delay in milliseconds between yields. Defaults to 0. + * @param options {valueOnly:boolean, interval?:number}={valueOnly:true, interval:-1} by default the generator will + * yield/return only the value and not the verbose {value, done} object that is yielded from a JS AsyncGenerator. + * The interval parameter is used to temporally space the yielding of array values when input is an array. When input + * is a function .or simple value, the interval is used to repeatedly call the function or yield the value * @returns The registered asynchronous generator. */ - public generate = (generateMe: any[]|any| (() => any), timeout: number = 0): AsyncGenerator => { + public generate = (input: AsyncGenerator|any[]|any| (() => any), options:{valueOnly:boolean, interval?:number, maxYield?:number}={valueOnly:true, interval:-1, maxYield:-1}): AsyncGenerator => { if (this.templateProcessor.isClosed) { throw new Error("generate() cannot be called on a closed TemplateProcessor"); } + const {interval=-1, valueOnly=true, maxYield=-1} = options; + if(maxYield === 0){ + throw new Error('maxYield must be greater than zero'); + } + if(GeneratorManager.isAsyncGenerator(input)){ //wrapping an existing async generator + input["valueOnly"] = options.valueOnly; //in effect, annotate the generator with the options, so that down the pike we can determine if we should pump just the value, or the entire shebang + return input + } const timerManager = this.templateProcessor.timerManager; + let g; //yield array items separated by timeout - if(Array.isArray(generateMe)) { - return async function* () { - for (let i = 0; i < generateMe.length; i++) { - yield generateMe[i]; - if (timeout > 0 && i < generateMe.length - 1) { - await new Promise((resolve) => timerManager.setTimeout(resolve, timeout)); + if(Array.isArray(input)) { + g = async function* () { + let max = input.length; + if(maxYield > 0){ + max = Math.min(max, maxYield); + } + let i; + for (i=0; i < max-1; i++) { + yield input[i]; + if (interval >= 0) { + await new Promise((resolve) => timerManager.setTimeout(resolve, interval)); } } + return input[i]; //last item is returned, not yielded, so don't is true with last item }(); } //call function and return result - if(typeof generateMe === 'function'){ - return async function*(){ - yield (generateMe as ()=>any)(); + else if(typeof input === 'function'){ + g = async function*(){ + if(interval < 0){ //no interval so call function once + return (input as ()=>any)();//return not yield, for done:true + }else{ //an interval is specified so we sit in a loop calling the function + let count = 0; + while(maxYield < 0 || count++ < maxYield-1){ + yield await (input as ()=>any)(); + await new Promise((resolve) => timerManager.setTimeout(resolve, interval)); + } + return await (input as ()=>any)(); + } + }(); + }else { + //yield individual item + g = async function* () { + if(interval < 0){ //no interval + return input; //return not yield, for done:true + }else{ + //interval is not supported for ordinary value as this will pump a duplicate same value over and over which will be deduped and ignored anyway + throw new Error("$generate cannot be used to repeat the same value on an 'interval' since Stated would simply dedup/ignore the repeated values."); + } }(); } - //yield individual item - return async function*(){ - yield generateMe; - }(); + (g as any)["valueOnly"] = valueOnly; + return g; }; - - - /** - * Extracts the first item from a generator, whether it is synchronous or asynchronous. - * Automatically registers the generator. - * - * @param gen The generator (sync or async) to extract the first item from. - * @returns A promise that resolves to the first item or `undefined` if empty. - */ - public async firstItem(gen: AsyncGenerator | Generator): Promise { - if (!GeneratorManager.isGenerator(gen)) { - throw new Error('The provided generator is not an AsynchronousGenerator, it is a `{typeof generator}`}.'); - } - - const asyncGen = gen as AsyncGenerator; - const result = await asyncGen.next(); - return result.done ? undefined : result.value; - } - /** * Checks if the provided object is a generator (synchronous or asynchronous). * * @param obj The object to check. * @returns `true` if the object is a generator; otherwise `false`. */ - public static isGenerator(obj: any): boolean { + public static isAsyncGenerator(obj: any): boolean { if (obj == null) return false; if (typeof obj.next === 'function' && typeof obj[Symbol.asyncIterator] === 'function') { return true; @@ -78,46 +94,66 @@ export class GeneratorManager{ return false; } + /** - * Pumps the remaining items from the generator into the TemplateProcessor. - * Automatically registers the generator and returns the first item. + * Pumps the remaining items (after the first item) from the generator into the TemplateProcessor. + * Automatically returns the first item. * * @param generator The generator to pump items from. * @param metaInfo The meta info for processing. * @param templateProcessor The TemplateProcessor to set data in. - * @returns The first generated item. + * @returns The first generated item wrapped as {value, done, return} */ public async pumpItems( - generator: AsyncGenerator | Generator, + generator: AsyncGenerator, metaInfo: any, templateProcessor: any ): Promise { - const first = await this.firstItem(generator); // Get the first item from the generator - // Check if the generator is asynchronous - if (this.isAsyncGenerator(generator)) { - // Handle asynchronous generator. Do not await, because we want items to be pumped into the template async - void this.handleAsyncGenerator(generator, metaInfo, templateProcessor); - } else { - // Handle synchronous generator - throw new Error('The provided generator is not an AsynchronousGenerator, it is a `{typeof generator}`}.'); + const {valueOnly=true} = generator as any; + const first = await generator.next(); // Get the first item from the generator + const {done} = first; + if(!done) { + if (GeneratorManager.isAsyncGenerator(generator)) { + // Handle asynchronous generator. Do not await, because we want items to be pumped into the template async. + // Also, fear not, pumpItems can only queue items which will queue the remaining items which won't be + //drained out of the queue until the item returned by this method has been processed + void this.pumpRemaining(generator, metaInfo, templateProcessor); + } else { + // Handle synchronous generator + throw new Error('The provided generator is not an AsynchronousGenerator, it is a `{typeof generator}`}.'); + } } - - return first; + return valueOnly?first.value: {...first, return: TemplateProcessor.wrapInOrdinaryFunction(generator.return.bind(generator))}; } /** - * Handles asynchronous generators, pumping values into the template processor. + * Handles asynchronous generators, pumping remaining values into the template processor. */ - private async handleAsyncGenerator( + private async pumpRemaining( generator: AsyncGenerator, metaInfo: any, templateProcessor: any ): Promise { - for await (const item of generator) { + while (true) { try { + const result = await generator.next(); + const {valueOnly=true} = generator as any; + const { value, done } = result; + + // Create an object that includes value, done, and the return function + const item = valueOnly?value:{ + value, + done, + return: TemplateProcessor.wrapInOrdinaryFunction(generator.return.bind(generator)) + }; + + // Pass the entire item object to setData await templateProcessor.setData(metaInfo.jsonPointer__ as string, item, "forceSetInternal"); - }catch(error:any){ - if(error.message === "Attempt to setData on a closed TemplateProcessor."){ + + // Break the loop if the generator is done + if (done) break; + } catch (error: any) { + if (error.message === "Attempt to setData on a closed TemplateProcessor.") { await generator.return(); break; } @@ -126,13 +162,4 @@ export class GeneratorManager{ } } - - /** - * Determines if a generator is asynchronous. - */ - private isAsyncGenerator(generator: any): generator is AsyncGenerator { - return typeof generator[Symbol.asyncIterator] === 'function'; - } - - }