test,doc: add managed benchmark note & test

README.md

@@ -38,7 +38,8 @@ suite.run().then(results => {
 });
 ```
 
-This module uses V8 deoptimization to ensure that the code block is not optimized away, producing accurate benchmarks. See the [Writing JavaScript Microbenchmark Mistakes](#TODO) section for more details.
+This module uses V8 deoptimization to helps that the code block is not optimized away, producing accurate benchmarks -- But, not realistics.
+See the [Writing JavaScript Microbenchmark Mistakes](#TODO) section for more details.
 
 ```bash
 $ node --allow-natives-syntax my-benchmark.js
@@ -55,6 +56,7 @@ See the [examples folder](./examples/) for more common usage examples.
 2. [Plugins](#plugins)
 3. [Using Custom Reporter](#using-custom-reporter)
 4. [Setup and Teardown](#setup-and-teardown)
+    1. [Managed Benchmarks](#managd-benchmarks)
 
 ## Class: `Suite`
 
@@ -245,3 +247,51 @@ suite.run();
 > See: [Deleting Properties Example](./examples/deleting-properties/node.js).
 
 Ensure you call `.start()` and `.end()` methods when using the timer argument, or an `ERR_BENCHMARK_MISSING_OPERATION` error will be thrown.
+
+### Managed Benchmarks
+
+In regular benchmarks (when `timer` is not used), you run the benchmarked function in a loop,
+and the timing is managed implicitly.
+This means each iteration of the benchmarked function is measured directly.
+The downside is that optimizations like inlining or caching might affect the timing, especially for fast operations.
+
+Example:
+
+```cjs
+suite.add('Using includes', function () {
+  const text = 'text/html,...';
+  const r = text.includes('application/json');
+});
+```
+
+Here, `%DoNotOptimize` is being called inside the loop for regular benchmarks (assuming V8NeverOptimizePlugin is being used),
+ensuring that the operation is not overly optimized within each loop iteration.
+This prevents V8 from optimizing away the operation (e.g., skipping certain steps because the result is not used or the function is too trivial).
+
+Managed benchmarks explicitly handle timing through `start()` and `end()` calls around the benchmarked code.
+This encapsulates the entire set of iterations in one timed block,
+which can result in tighter measurement with less overhead.
+However, it can lead to over-optimistic results, especially if the timer’s start and stop calls are placed outside of the loop,
+allowing V8 to over-optimize the entire block.
+
+Example:
+
+```cjs
+suite.add('[Managed] Using includes', function (timer) {
+  timer.start();
+  for (let i = 0; i < timer.count; i++) {
+    const text = 'text/html,...';
+    const r = text.includes('application/json');
+    assert.ok(r);  // Ensure the result is used so it doesn't get V8 optimized away
+  }
+  timer.end(timer.count);
+});
+```
+
+In this case, `%DoNotOptimize` is being applied outside the loop, so it does not protect each iteration from
+excessive optimization. This can result in higher operation counts because V8 might optimize away repetitive tasks.
+That's why an `assert.ok(r)` has been used. To avoid V8 optimizing the entire block as the `r` var was not being used.
+
+> [!NOTE]
+> V8 assumptions can change any time soon. Therefore, it's crucial to investigate
+> results between versions of V8/Node.js.

test/env.js

@@ -1,6 +1,45 @@
 const { describe, it, before } = require('node:test');
 const assert = require('node:assert');
 const copyBench = require('./fixtures/copy');
+const { managedBench, managedOptBench } = require('./fixtures/opt-managed');
+
+function assertMinBenchmarkDifference(results, { percentageLimit, ciPercentageLimit }) {
+  assertBenchmarkDifference(results, { percentageLimit, ciPercentageLimit, greaterThan: true });
+}
+
+function assertMaxBenchmarkDifference(results, { percentageLimit, ciPercentageLimit }) {
+  assertBenchmarkDifference(results, { percentageLimit, ciPercentageLimit, greaterThan: false });
+}
+
+function assertBenchmarkDifference(results, { percentageLimit, ciPercentageLimit, greaterThan }) {
+  for (let i = 0; i < results.length; i++) {
+    for (let j = 0; j < results.length; j++) {
+      if (i !== j) {
+        const opsSec1 = results[i].opsSec;
+        const opsSec2 = results[j].opsSec;
+
+        // Calculate the percentage difference
+        const difference = Math.abs(opsSec1 - opsSec2);
+        const percentageDifference = (difference / Math.min(opsSec1, opsSec2)) * 100;
+
+        // Check if the percentage difference is less than or equal to 10%
+        if (process.env.CI) {
+          // CI runs in a shared-env so the percentage of difference
+          // must be greather there due to high variance of hardware
+          assert.ok(
+            greaterThan ? percentageLimit >= ciPercentageLimit : percentageDifference <= ciPercentageLimit,
+            `"${results[i].name}" too different from "${results[j].name}" - ${percentageDifference} != ${ciPercentageLimit}`
+          );
+        } else {
+          assert.ok(
+            greaterThan ? percentageLimit >= percentageLimit : percentageDifference <= percentageLimit,
+            `${results[i].name} too different from ${results[j].name} - ${percentageDifference} != ${percentageLimit}`
+          );
+        }
+      }
+    }
+  }
+}
 
 describe('Same benchmark function', () => {
   let results;
@@ -10,32 +49,23 @@ describe('Same benchmark function', () => {
   });
 
   it('must have a similar benchmark result', () => {
-    for (let i = 0; i < results.length; i++) {
-      for (let j = 0; j < results.length; j++) {
-        if (i !== j) {
-          const opsSec1 = results[i].opsSec;
-          const opsSec2 = results[j].opsSec;
-
-          // Calculate the percentage difference
-          const difference = Math.abs(opsSec1 - opsSec2);
-          const percentageDifference = (difference / Math.min(opsSec1, opsSec2)) * 100;
-
-          // Check if the percentage difference is less than or equal to 10%
-          if (process.env.CI) {
-            // CI runs in a shared-env so the percentage of difference
-            // must be greather there due to high variance of hardware
-            assert.ok(
-              percentageDifference <= 30,
-              `${opsSec1} too different from ${opsSec2} - ${results[i].name}`
-            );
-          } else {
-            assert.ok(
-              percentageDifference <= 10,
-              `${opsSec1} too different from ${opsSec2} - ${results[i].name}`
-            );
-          }
-        }
-      }
-    }
+    assertMaxBenchmarkDifference(results, { percentageLimit: 10, ciPercentageLimit: 30 });
   });
 });
+
+describe('Managed can be V8 optimized', () => {
+  let optResults, results;
+
+  before(async () => {
+    optResults = await managedOptBench.run();
+    results = await managedBench.run();
+  });
+
+  it('should be more than 50% different from unmanaged', () => {
+    assertMinBenchmarkDifference(optResults, { percentageLimit: 50, ciPercentageLimit: 30 });
+  });
+
+  // it('should be similar when avoiding V8 optimizatio', () => {
+  //   assertBenchmarkDifference(results, 50, 30);
+  // });
+});

test/fixtures/opt-managed.js

@@ -0,0 +1,45 @@
+const { Suite } = require('../../lib');
+const assert = require('node:assert');
+
+const suite = new Suite({ reporter: false });
+
+suite
+  .add('Using includes', function () {
+    const text = 'text/html,application/xhtml+xml,application/xml;application/json;q=0.9,image/avif,image/webp,*/*;q=0.8'
+    const r = text.includes('application/json')
+    assert.ok(r)
+  })
+  .add('[Managed] Using includes', function (timer) {
+    timer.start()
+    for (let i = 0; i < timer.count; i++) {
+      const text = 'text/html,application/xhtml+xml,application/xml;application/json;q=0.9,image/avif,image/webp,*/*;q=0.8'
+      const r = text.includes('application/json')
+      assert.ok(r)
+    }
+    timer.end(timer.count)
+  })
+
+suite.run();
+
+const optSuite = new Suite({ reporter: false });
+
+optSuite
+  .add('Using includes', function () {
+    const text = 'text/html,application/xhtml+xml,application/xml;application/json;q=0.9,image/avif,image/webp,*/*;q=0.8'
+    const r = text.includes('application/json')
+    // assert.ok(r)
+  })
+  .add('[Managed] Using includes', function (timer) {
+    timer.start()
+    for (let i = 0; i < timer.count; i++) {
+      const text = 'text/html,application/xhtml+xml,application/xml;application/json;q=0.9,image/avif,image/webp,*/*;q=0.8'
+      const r = text.includes('application/json')
+      // assert.ok(r)
+    }
+    timer.end(timer.count)
+  })
+
+module.exports = {
+  managedBench: suite,
+  managedOptBench: optSuite,
+};