From bc43a5685f14940a3c6b8ed0ebdf4b7c50df75c4 Mon Sep 17 00:00:00 2001 From: RafaelGSS Date: Tue, 15 Oct 2024 17:56:42 -0300 Subject: [PATCH 1/2] test: add scenario for optimized managed benchmark --- test/env.js | 84 ++++++++++++++++++++++++------------ test/fixtures/opt-managed.js | 45 +++++++++++++++++++ 2 files changed, 102 insertions(+), 27 deletions(-) create mode 100644 test/fixtures/opt-managed.js diff --git a/test/env.js b/test/env.js index 8361379..0883bf1 100644 --- a/test/env.js +++ b/test/env.js @@ -1,6 +1,45 @@ const { describe, it, before } = require('node:test'); const assert = require('node:assert'); const copyBench = require('./fixtures/copy'); +const { managedBench, managedOptBench } = require('./fixtures/opt-managed'); + +function assertMinBenchmarkDifference(results, { percentageLimit, ciPercentageLimit }) { + assertBenchmarkDifference(results, { percentageLimit, ciPercentageLimit, greaterThan: true }); +} + +function assertMaxBenchmarkDifference(results, { percentageLimit, ciPercentageLimit }) { + assertBenchmarkDifference(results, { percentageLimit, ciPercentageLimit, greaterThan: false }); +} + +function assertBenchmarkDifference(results, { percentageLimit, ciPercentageLimit, greaterThan }) { + for (let i = 0; i < results.length; i++) { + for (let j = 0; j < results.length; j++) { + if (i !== j) { + const opsSec1 = results[i].opsSec; + const opsSec2 = results[j].opsSec; + + // Calculate the percentage difference + const difference = Math.abs(opsSec1 - opsSec2); + const percentageDifference = (difference / Math.min(opsSec1, opsSec2)) * 100; + + // Check if the percentage difference is less than or equal to 10% + if (process.env.CI) { + // CI runs in a shared-env so the percentage of difference + // must be greather there due to high variance of hardware + assert.ok( + greaterThan ? percentageLimit >= ciPercentageLimit : percentageDifference <= ciPercentageLimit, + `"${results[i].name}" too different from "${results[j].name}" - ${percentageDifference} != ${ciPercentageLimit}` + ); + } else { + assert.ok( + greaterThan ? percentageLimit >= percentageLimit : percentageDifference <= percentageLimit, + `${results[i].name} too different from ${results[j].name} - ${percentageDifference} != ${percentageLimit}` + ); + } + } + } + } +} describe('Same benchmark function', () => { let results; @@ -10,32 +49,23 @@ describe('Same benchmark function', () => { }); it('must have a similar benchmark result', () => { - for (let i = 0; i < results.length; i++) { - for (let j = 0; j < results.length; j++) { - if (i !== j) { - const opsSec1 = results[i].opsSec; - const opsSec2 = results[j].opsSec; - - // Calculate the percentage difference - const difference = Math.abs(opsSec1 - opsSec2); - const percentageDifference = (difference / Math.min(opsSec1, opsSec2)) * 100; - - // Check if the percentage difference is less than or equal to 10% - if (process.env.CI) { - // CI runs in a shared-env so the percentage of difference - // must be greather there due to high variance of hardware - assert.ok( - percentageDifference <= 30, - `${opsSec1} too different from ${opsSec2} - ${results[i].name}` - ); - } else { - assert.ok( - percentageDifference <= 10, - `${opsSec1} too different from ${opsSec2} - ${results[i].name}` - ); - } - } - } - } + assertMaxBenchmarkDifference(results, { percentageLimit: 10, ciPercentageLimit: 30 }); }); }); + +describe('Managed can be V8 optimized', () => { + let optResults, results; + + before(async () => { + optResults = await managedOptBench.run(); + results = await managedBench.run(); + }); + + it('should be more than 50% different from unmanaged', () => { + assertMinBenchmarkDifference(optResults, { percentageLimit: 50, ciPercentageLimit: 30 }); + }); + + // it('should be similar when avoiding V8 optimizatio', () => { + // assertBenchmarkDifference(results, 50, 30); + // }); +}); diff --git a/test/fixtures/opt-managed.js b/test/fixtures/opt-managed.js new file mode 100644 index 0000000..93bfcf3 --- /dev/null +++ b/test/fixtures/opt-managed.js @@ -0,0 +1,45 @@ +const { Suite } = require('../../lib'); +const assert = require('node:assert'); + +const suite = new Suite({ reporter: false }); + +suite + .add('Using includes', function () { + const text = 'text/html,application/xhtml+xml,application/xml;application/json;q=0.9,image/avif,image/webp,*/*;q=0.8' + const r = text.includes('application/json') + assert.ok(r) + }) + .add('[Managed] Using includes', function (timer) { + timer.start() + for (let i = 0; i < timer.count; i++) { + const text = 'text/html,application/xhtml+xml,application/xml;application/json;q=0.9,image/avif,image/webp,*/*;q=0.8' + const r = text.includes('application/json') + assert.ok(r) + } + timer.end(timer.count) + }) + +suite.run(); + +const optSuite = new Suite({ reporter: false }); + +optSuite + .add('Using includes', function () { + const text = 'text/html,application/xhtml+xml,application/xml;application/json;q=0.9,image/avif,image/webp,*/*;q=0.8' + const r = text.includes('application/json') + // assert.ok(r) + }) + .add('[Managed] Using includes', function (timer) { + timer.start() + for (let i = 0; i < timer.count; i++) { + const text = 'text/html,application/xhtml+xml,application/xml;application/json;q=0.9,image/avif,image/webp,*/*;q=0.8' + const r = text.includes('application/json') + // assert.ok(r) + } + timer.end(timer.count) + }) + +module.exports = { + managedBench: suite, + managedOptBench: optSuite, +}; From 7c0bce4455d5e5aa5f65d9dbd76a1910c8efbb42 Mon Sep 17 00:00:00 2001 From: RafaelGSS Date: Tue, 15 Oct 2024 17:57:00 -0300 Subject: [PATCH 2/2] doc: include note about using managed benchmarks --- README.md | 52 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3fc9cbf..5151992 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,8 @@ suite.run().then(results => { }); ``` -This module uses V8 deoptimization to ensure that the code block is not optimized away, producing accurate benchmarks. See the [Writing JavaScript Microbenchmark Mistakes](#TODO) section for more details. +This module uses V8 deoptimization to helps that the code block is not optimized away, producing accurate benchmarks -- But, not realistics. +See the [Writing JavaScript Microbenchmark Mistakes](#TODO) section for more details. ```bash $ node --allow-natives-syntax my-benchmark.js @@ -55,6 +56,7 @@ See the [examples folder](./examples/) for more common usage examples. 2. [Plugins](#plugins) 3. [Using Custom Reporter](#using-custom-reporter) 4. [Setup and Teardown](#setup-and-teardown) + 1. [Managed Benchmarks](#managd-benchmarks) ## Class: `Suite` @@ -245,3 +247,51 @@ suite.run(); > See: [Deleting Properties Example](./examples/deleting-properties/node.js). Ensure you call `.start()` and `.end()` methods when using the timer argument, or an `ERR_BENCHMARK_MISSING_OPERATION` error will be thrown. + +### Managed Benchmarks + +In regular benchmarks (when `timer` is not used), you run the benchmarked function in a loop, +and the timing is managed implicitly. +This means each iteration of the benchmarked function is measured directly. +The downside is that optimizations like inlining or caching might affect the timing, especially for fast operations. + +Example: + +```cjs +suite.add('Using includes', function () { + const text = 'text/html,...'; + const r = text.includes('application/json'); +}); +``` + +Here, `%DoNotOptimize` is being called inside the loop for regular benchmarks (assuming V8NeverOptimizePlugin is being used), +ensuring that the operation is not overly optimized within each loop iteration. +This prevents V8 from optimizing away the operation (e.g., skipping certain steps because the result is not used or the function is too trivial). + +Managed benchmarks explicitly handle timing through `start()` and `end()` calls around the benchmarked code. +This encapsulates the entire set of iterations in one timed block, +which can result in tighter measurement with less overhead. +However, it can lead to over-optimistic results, especially if the timer’s start and stop calls are placed outside of the loop, +allowing V8 to over-optimize the entire block. + +Example: + +```cjs +suite.add('[Managed] Using includes', function (timer) { + timer.start(); + for (let i = 0; i < timer.count; i++) { + const text = 'text/html,...'; + const r = text.includes('application/json'); + assert.ok(r); // Ensure the result is used so it doesn't get V8 optimized away + } + timer.end(timer.count); +}); +``` + +In this case, `%DoNotOptimize` is being applied outside the loop, so it does not protect each iteration from +excessive optimization. This can result in higher operation counts because V8 might optimize away repetitive tasks. +That's why an `assert.ok(r)` has been used. To avoid V8 optimizing the entire block as the `r` var was not being used. + +> [!NOTE] +> V8 assumptions can change any time soon. Therefore, it's crucial to investigate +> results between versions of V8/Node.js.