refactor so that takes options

README.md

@@ -1763,14 +1763,35 @@ keep in mind that all the values were sequentially pushed into the `generated` f
   "generated": 10
 }
 ```
+If you are familiar with generators in JS, you know that generated values take a verbose form in which 
+a `{value, done}` object is [returned](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/AsyncGenerator/next#return_value).
+If you wish to use the more verbose JS style of yielded/returned object, you can pass your generator 
+to `$generate` and disable `valuesOnly`, as follows. Notice how the yielded values now contain the JS style 
+`{value, done}`
+```json
+> .init -f example/myGeneratorVerbose.json --xf example/myGenerator.mjs
+{
+  "generated": "${$myGenerator()~>$generate({'valueOnly':false})}"
+}
+> .out
+{
+  "generated": {
+    "value": 10,
+    "done": true,
+    "return": "{function:}"
+  }
+}
+```
+
 A slight variation on the example accumulates every value yielded by the generator:
 ```json
-> .init -f example/myGenerator2.json --xf example/myGenerator.mjs
+>  .init -f example/myGenerator2.json --xf example/myGenerator.mjs
 {
-   "generated": "${$myGenerator()}",
-   "onGenerated": "${$set('/accumulator/-', $$.generated)}",
-   "accumulator": []
+  "generated": "${$myGenerator()}",
+  "onGenerated": "${$set('/accumulator/-', $$.generated)}",
+  "accumulator": []
 }
+
 ```
 ```json ["data=[1,2,3,4,5,6,7,8,9,10]"]
 > .init -f example/myGenerator2.json --xf example/myGenerator.mjs --tail "/accumulator until $=[1,2,3,4,5,6,7,8,9,10]"
@@ -1789,14 +1810,15 @@ Started tailing... Press Ctrl+C to stop.
 ]
 
 ```
-Or, you can use the built in `$generate` method, which takes an optional delay in ms, and turns the array
-into an AsyncGenerator. In the example below the values 1 to 10 are pumped into the `generated` field
-with 10 ms temporal separation.
+You already saw hoe the built-in `$generate` function can accept a JS AsyncGeneraotr, and options. But $generate 
+can also be used to convert ordinary arrays or functions into async generators. When provided, the `interval` option, 
+causes the provided array to yield its elements periodically. When a function is provided, as opposed to an array, the
+function is called periodically.
 ```json
 > .init -f example/generate.json
 {
   "delayMs": 250,
-  "generated":"${[1..10]~>$generate(delayMs)}"
+  "generated":"${[1..10]~>$generate({'interval':delayMs})}"
 }
 ```
 ```json ["data.generated=10"]
@@ -1808,6 +1830,63 @@ Started tailing... Press Ctrl+C to stop.
 }
 
 ```
+This `example/myGenerator3.yaml` shows how you can call `return` and stop a generator.
+```yaml
+generated: ${$generate($random, {'interval':10, 'valueOnly':false})}
+onGenerated: |
+  ${
+    $count(accumulator)<3
+      ? $set('/accumulator/-', $$.generated.value)
+      : generated.return() /* shut off the generator when the accumulator has 10 items */
+  }
+accumulator: []
+```
+```json ["$count(data.accumulator)=3"]
+> .init -f example/myGenerator3.yaml --tail "/ until $count(accumulator)=3"
+Started tailing... Press Ctrl+C to stop.
+{
+  "generated": {
+    "value": 0.23433826655570145,
+    "done": false,
+    "return": "{function:}"
+  },
+  "onGenerated": {
+    "value": null,
+    "done": true
+  },
+  "accumulator": [
+    0.23433826655570145,
+    0.23433826655570145,
+    0.23433826655570145
+  ]
+}
+
+
+```
+The `maxYield` parameter can also be used to stop a generator:
+```json ["$count(data.accumulator)=0", "$count(data.accumulator)=5"]
+> .init -f example/myGenerator4.yaml
+{
+  "generated": "${$generate($random, {'interval':10, 'maxYield':5})}",
+  "onGenerated": "${$set('/accumulator/-', $$.generated)}",
+  "accumulator": []
+}
+> .init -f example/myGenerator4.yaml --tail "/ until $count(accumulator)=5"
+{
+  "generated": 0.5289126250886866,
+  "onGenerated": [
+    "/accumulator/-"
+  ],
+  "accumulator": [
+    0.3260049204634301,
+    0.4477190160739559,
+    0.9414436597923774,
+    0.8593436891141426,
+    0.5289126250886866
+  ]
+}
+```
+
 
 ### $setTimeout
 `$setTimeout` is the JavaScript `setTimeout` function. It receives a function and an timeout

example/generate.json

@@ -1,4 +1,4 @@
 {
   "delayMs": 250,
-  "generated":"${[1..10]~>$generate(delayMs)}"
+  "generated":"${[1..10]~>$generate({'interval':delayMs})}"
 }

example/myGenerator.mjs

@@ -1,5 +1,6 @@
 export async function* myGenerator() {
-    for (let i = 1; i <= 10; i++) {
+    for (let i = 1; i < 10; i++) {
         yield i;
     }
+    return 10; // Last value with `done: true`
 }

example/myGenerator3.yaml

@@ -0,0 +1,8 @@
+generated: ${$generate($random, {'interval':10, 'valueOnly':false})}
+onGenerated: |
+  ${
+    $count(accumulator)<3
+      ? $set('/accumulator/-', $$.generated.value)
+      : generated.return() /* shut off the generator when the accumulator has 10 items */
+  }
+accumulator: []

example/myGenerator4.yaml

@@ -0,0 +1,3 @@
+generated: ${$generate($random, {'interval':10, 'maxYield':5})}
+onGenerated:  ${$set('/accumulator/-', $$.generated)}
+accumulator: []

example/myGeneratorVerbose.json

@@ -0,0 +1,3 @@
+{
+  "generated": "${$myGenerator()~>$generate({'valueOnly':false})}"
+}

src/TemplateProcessor.ts

@@ -1633,7 +1633,7 @@ export default class TemplateProcessor {
                 context
             );
             if (evaluated?._jsonata_lambda) {
-                evaluated = this.wrapInOrdinaryFunction(evaluated);
+                evaluated = TemplateProcessor.wrapInOrdinaryFunction(evaluated);
                 metaInfo.isFunction__ = true;
             }
         } catch (error: any) {
@@ -1647,9 +1647,9 @@ export default class TemplateProcessor {
             _error.name = "JSONata evaluation exception";
             throw _error;
         }
-        if (GeneratorManager.isGenerator(evaluated)) {
-            //returns the first item, and begins pumping remaining items into execution queue
-            evaluated = this.generatorManager.pumpItems(evaluated as AsyncGenerator, metaInfo, this);
+        if (GeneratorManager.isAsyncGenerator(evaluated)) {
+            //awaits and returns the first item. And pumpItems begins pumping remaining items into execution queue asynchronously
+            evaluated = await this.generatorManager.pumpItems(evaluated as AsyncGenerator, metaInfo, this);
         }
         return evaluated
 
@@ -1869,7 +1869,6 @@ export default class TemplateProcessor {
         if (callbacks) {
             const promises = Array.from(callbacks).map(cbFn =>
                 Promise.resolve().then(() => {
-                    //to do ... return here so asyn functions are actually awaited by Promise.all
                     cbFn(data, jsonPointer as JsonPointerString, removed, op);
                 }) //works with cbFn that is either sync or async by wrapping in promise
             );
@@ -1934,7 +1933,7 @@ export default class TemplateProcessor {
         }
     }
 
-    private wrapInOrdinaryFunction(jsonataLambda:any) {
+    public static wrapInOrdinaryFunction(jsonataLambda:any) {
         const wrappedFunction = (...args:any[])=> {
             // Call the 'apply' method of jsonataLambda with the captured arguments
             return jsonataLambda.apply(jsonataLambda, args);
@@ -2062,7 +2061,7 @@ export default class TemplateProcessor {
                 forkId: TemplateProcessor.simpleUniqueId()
             };
             //do not await setData...$forked runs async
-            this.setDataForked (mvccSnapshotPlanStep);
+            void this.setDataForked (mvccSnapshotPlanStep);
         }
     }
 

src/test/TemplateProcessor.test.js

@@ -3112,8 +3112,8 @@ test("test close", async () => {
 
 test("test generate array", async () => {
     const o = {
-        "delayMs": 10,
-        "a":"${[1..10]~>$generate(delayMs)}",
+        "options": {"interval":10, "valueOnly":true},
+        "a":"${[1..10]~>$generate(options)}",
         "b": "${a}"
     };
 
@@ -3197,6 +3197,35 @@ test("test generate function result", async () => {
     }
 });
 
+test("test generate verbose function result", async () => {
+    const o = {
+        "a":"${$generate(function(){10}, {'valueOnly':false})}",
+        "b": "${a}"
+    };
+
+    let resolvePromise;
+    const allCallsMade = new Promise((resolve) => {
+        resolvePromise = resolve;
+    });
+
+    const changeHandler = jest.fn((data, ptr) => {
+        expect(ptr).toBe("/b"); // Ensure correct pointer
+        resolvePromise(); // Resolve the promise when callCount is reached
+    });
+
+    const tp = new TemplateProcessor(o);
+    tp.setDataChangeCallback('/b', changeHandler);
+    try {
+        await tp.initialize();
+        await allCallsMade;
+        expect(changeHandler).toHaveBeenCalledTimes(1);
+        expect(tp.output.b).toMatchObject({value:10, done:true});
+        expect(tp.output.b.return).toBeDefined(); //make sure 'return' function is provided
+    } finally {
+        await tp.close();
+    }
+});
+
 
 test("test lifecycle manager", async () => {
     const o = {