feat: add benchmark repetition (#27)

* feat: add benchmark repetitions as opts

* examples: add suite with repetition in create-uint32array

* test: add test to repetition opt

* chore: add package-lock.json in .gitignore

* lib,test,examples: rename opt from repetition to repeatSuite

* docs: update docs to add repeatSuite

.gitignore

@@ -1,3 +1,4 @@
 tags
 .idea
 node_modules/
+package-lock.json

README.md

@@ -93,7 +93,8 @@ const suite = new Suite({ reporter: false });
 * `options` {Object} Configuration options for the benchmark. Supported properties:
   * `minTime` {number} Minimum duration for the benchmark to run. **Default:** `0.05` seconds.
   * `maxTime` {number} Maximum duration for the benchmark to run. **Default:** `0.5` seconds.
-* `fn` {Function|AsyncFunction} The benchmark function. Can be synchronous or asynchronous.
+  * `repeatSuite` {number} Number of times to repeat benchmark to run. **Default:** `1` times.
+* `fn` {Function|AsyncFunction} The benchmark function. Can be synchronous or asynchronous. 
 * Returns: {Suite}
 
 Adds a benchmark function to the suite.

examples/create-uint32array/node.js

@@ -6,6 +6,9 @@ suite
   .add(`new Uint32Array(1024)`, function () {
     return new Uint32Array(1024);
   })
+  .add(`new Uint32Array(1024) with 10 repetitions`,  {repeatSuite: 10}, function () {
+    return new Uint32Array(1024);
+  })
   .add(`[Managed] new Uint32Array(1024)`, function (timer) {
     const assert = require('node:assert');
 

lib/index.js

@@ -21,13 +21,15 @@ class Benchmark {
   minTime;
   maxTime;
   plugins;
+  repeatSuite;
 
-  constructor(name, fn, minTime, maxTime, plugins) {
+  constructor(name, fn, minTime, maxTime, plugins, repeatSuite) {
     this.name = name;
     this.fn = fn;
     this.minTime = minTime;
     this.maxTime = maxTime;
     this.plugins = plugins;
+    this.repeatSuite = repeatSuite;
   }
 }
 
@@ -36,6 +38,8 @@ const defaultBenchOptions = {
   minTime: 0.05,
   // 0.5s - Arbitrary number used in some benchmark tools
   maxTime: 0.5,
+  // Number of times the benchmark will be repeated
+  repeatSuite: 1,
 };
 
 function throwIfNoNativesSyntax() {
@@ -81,6 +85,7 @@ class Suite {
       };
       validateNumber(options.minTime, 'options.minTime', timer.resolution * 1e3);
       validateNumber(options.maxTime, 'options.maxTime', options.minTime);
+      validateNumber(options.repeatSuite, 'options.repeatSuite', options.repeatSuite);
     }
     validateFunction(fn, 'fn');
 
@@ -90,6 +95,7 @@ class Suite {
       options.minTime,
       options.maxTime,
       this.#plugins,
+      options.repeatSuite,
     );
     this.#benchmarks.push(benchmark);
     return this;
@@ -111,8 +117,8 @@ class Suite {
       const benchmark = this.#benchmarks[i];
       // Warmup is calculated to reduce noise/bias on the results
       const initialIteration = await getInitialIterations(benchmark);
-      debugBench(`Starting ${ benchmark.name } with minTime=${ benchmark.minTime }, maxTime=${ benchmark.maxTime }`);
-      const result = await runBenchmark(benchmark, initialIteration);
+      debugBench(`Starting ${ benchmark.name } with minTime=${ benchmark.minTime }, maxTime=${ benchmark.maxTime }, repeatSuite=${ benchmark.repeatSuite }`);
+      const result = await runBenchmark(benchmark, initialIteration, benchmark.repeatSuite);
       results[i] = result;
     }
 

lib/lifecycle.js

@@ -64,12 +64,7 @@ async function runWarmup(bench, initialIterations, { minTime, maxTime }) {
   }
 }
 
-async function runBenchmark(bench, initialIterations) {
-  const histogram = new StatisticalHistogram();
-
-  const maxDuration = bench.maxTime * timer.scale;
-  const minSamples = 10;
-
+async function runBenchmarkOnce(bench, histogram, { initialIterations, maxDuration, minSamples }) {
   let iterations = 0;
   let timeSpent = 0;
 
@@ -88,13 +83,36 @@ async function runBenchmark(bench, initialIterations) {
     const minWindowTime = Math.max(0, Math.min((maxDuration - timeSpent) / timer.scale, bench.minTime));
     initialIterations = getItersForOpDuration(durationPerOp, minWindowTime);
   }
+
+  return { iterations, timeSpent };
+}
+
+async function runBenchmark(bench, initialIterations, repeatSuite) {
+  const histogram = new StatisticalHistogram();
+
+  const maxDuration = bench.maxTime * timer.scale;
+  const minSamples = 10;
+
+  let totalIterations = 0;
+  let totalTimeSpent = 0;
+
+  for (let i = 0; i < repeatSuite; ++i) {
+    const { iterations, timeSpent } = await runBenchmarkOnce(
+      bench,
+      histogram,
+      { initialIterations, maxDuration, minSamples }
+    );
+
+    totalTimeSpent += timeSpent;
+    totalIterations += iterations;
+  }
   histogram.finish()
 
-  const opsSec = iterations / (timeSpent / timer.scale);
+  const opsSec = totalIterations / (totalTimeSpent / timer.scale);
 
   return {
     opsSec,
-    iterations,
+    iterations: totalIterations,
     histogram,
     name: bench.name,
     plugins: parsePluginsResult(bench.plugins, bench.name),

test/basic.js

@@ -117,6 +117,16 @@ describe('API Interface', () => {
       // doesNotThrow
       bench.add('name', noop);
     });
+
+    it('repeatSuite should be a valid number', () => {
+      ['ds', {}, () => {}].forEach((r) => {
+        assert.throws(() => {
+          bench.add('name', { repeatSuite: r }, noop);
+        }, {
+          code: 'ERR_INVALID_ARG_TYPE',
+        });
+      });
+    });
   });
 });
 

test/plugin-api-doc.js

@@ -67,7 +67,7 @@ describe("plugin API", async () => {
       "getReport(string)",
       "getResult(string)",
       "isSupported()",
-      "onCompleteBenchmark([number, number, object], {fn, maxTime, minTime, name, plugins})",
+      "onCompleteBenchmark([number, number, object], {fn, maxTime, minTime, name, plugins, repeatSuite})",
       "toJSON(string)",
       "toString()",
     ]);