From c3d27252fce23648e8c86dc6f97d3879ca6443cd Mon Sep 17 00:00:00 2001 From: RafaelGSS Date: Mon, 5 Oct 2026 10:26:02 -0300 Subject: [PATCH] lib: default node:bench warmup to 10 invocations Signed-off-by: RafaelGSS --- doc/api/bench.md | 8 ++++- doc/api/cli.md | 2 ++ doc/node.1 | 2 ++ lib/internal/bench_runner/benchmark.js | 2 +- test/fixtures/bench-runner/diagnostic.cjs | 1 + test/fixtures/bench-runner/many-records.cjs | 2 +- .../fixtures/bench-runner/recorded-detail.cjs | 2 +- test/fixtures/bench-runner/serial.js | 2 +- test/fixtures/bench-runner/utf8-output.cjs | 2 +- test/parallel/test-bench-auto-run.js | 4 ++- test/parallel/test-bench-context-control.js | 4 ++- test/parallel/test-bench-create-runner.js | 4 +-- test/parallel/test-bench-default-warmup.js | 35 +++++++++++++++++++ .../test-bench-diagnostic-channels.js | 8 ++++- test/parallel/test-bench-diagnostics.js | 4 +++ test/parallel/test-bench-errors.js | 2 +- test/parallel/test-bench-filtering.js | 2 +- test/parallel/test-bench-run.js | 2 +- test/parallel/test-bench-stream.js | 20 +++++------ test/parallel/test-bench-validation.js | 2 +- .../test-bench-yield-between-samples.js | 2 +- 21 files changed, 86 insertions(+), 26 deletions(-) create mode 100644 test/parallel/test-bench-default-warmup.js diff --git a/doc/api/bench.md b/doc/api/bench.md index e06622ddf366..c04ae971523a 100644 --- a/doc/api/bench.md +++ b/doc/api/bench.md @@ -74,6 +74,12 @@ once to provide an externally measured sample. Setup before `start()` and cleanup after `end()` are outside the measured region. Promise-returning functions are awaited. +By default, ten unreported warmup invocations run before the measured samples +so that early samples are less likely to be taken before the measured code has +been optimized. A fixed warmup count does not guarantee a stable optimization +state; inspect raw `samples` and increase `warmup` when early samples are +consistently slower. Set `warmup` to `0` to measure from the first invocation. + By default, an event loop turn occurs between sample invocations. An embedded runner can disable this using `yieldBetweenSamples`. The runner executes benchmarks serially, but it does not provide process isolation. Other work in @@ -336,7 +342,7 @@ added: v26.9.0 * `timeout` {number} The number of milliseconds after which the benchmark fails. **Default:** `Infinity`. * `warmup` {number} The number of unreported callback invocations before - measured samples. Must be a 32-bit unsigned integer. **Default:** `0`. + measured samples. Must be a 32-bit unsigned integer. **Default:** `10`. * `fn` {Function|AsyncFunction} The benchmark function. It receives a {BenchContext}. * Returns: {Promise} Fulfilled with the benchmark result after a top-level diff --git a/doc/api/cli.md b/doc/api/cli.md index 4e17d81169db..4ba608a789f6 100644 --- a/doc/api/cli.md +++ b/doc/api/cli.md @@ -630,6 +630,8 @@ added: v26.9.0 Overrides the number of unreported warmup callback invocations for every selected benchmark. `count` must be an integer between `0` and `4294967295`. +Without this option, each benchmark uses its own `warmup` value, which defaults +to `10`. ### `--build-sea=config` diff --git a/doc/node.1 b/doc/node.1 index 70488dff26ca..683773b68d87 100644 --- a/doc/node.1 +++ b/doc/node.1 @@ -384,6 +384,8 @@ selected benchmark. A benchmark may finish earlier by calling .It Fl -bench-warmup Ns = Ns Ar count Overrides the number of unreported warmup callback invocations for every selected benchmark. \fBcount\fR must be an integer between \fB0\fR and \fB4294967295\fR. +Without this option, each benchmark uses its own \fBwarmup\fR value, which defaults +to \fB10\fR. . .It Fl -build-sea Ns = Ns Ar config Generates a single executable application from a JSON diff --git a/lib/internal/bench_runner/benchmark.js b/lib/internal/bench_runner/benchmark.js index ddcf54169a0d..0ab3da9f24d8 100644 --- a/lib/internal/bench_runner/benchmark.js +++ b/lib/internal/bench_runner/benchmark.js @@ -47,7 +47,7 @@ const { structuredClone } = require('internal/worker/js_transferable'); const { bigint: hrtime } = process.hrtime; const kDefaultSamples = 30; -const kDefaultWarmup = 0; +const kDefaultWarmup = 10; const kEmptyDiagnosticChannels = ObjectFreeze([]); const kEmptyNamePath = ObjectFreeze([]); const kEmptyParams = ObjectFreeze({ __proto__: null }); diff --git a/test/fixtures/bench-runner/diagnostic.cjs b/test/fixtures/bench-runner/diagnostic.cjs index 9e413e943dd7..2cb654aa4233 100644 --- a/test/fixtures/bench-runner/diagnostic.cjs +++ b/test/fixtures/bench-runner/diagnostic.cjs @@ -9,6 +9,7 @@ const diagnosticChannel = channel(channelName); bench('diagnostic relay', { diagnosticChannels: [channelName], samples: 1, + warmup: 0, }, (b) => { const message = { value: 42n }; diagnosticChannel.publish(message); diff --git a/test/fixtures/bench-runner/many-records.cjs b/test/fixtures/bench-runner/many-records.cjs index aacb17821035..c0923a70ea88 100644 --- a/test/fixtures/bench-runner/many-records.cjs +++ b/test/fixtures/bench-runner/many-records.cjs @@ -2,7 +2,7 @@ const { bench } = require('node:bench'); -bench('many records', { samples: 30 }, (b) => { +bench('many records', { samples: 30, warmup: 0 }, (b) => { process.stdout.write(`${b.index}\n`); b.record({ duration_ns: 1n, operations: 1 }); }); diff --git a/test/fixtures/bench-runner/recorded-detail.cjs b/test/fixtures/bench-runner/recorded-detail.cjs index 1643783225d0..0ad6f66ed9c7 100644 --- a/test/fixtures/bench-runner/recorded-detail.cjs +++ b/test/fixtures/bench-runner/recorded-detail.cjs @@ -2,7 +2,7 @@ const { bench } = require('node:bench'); -bench('recorded detail', { samples: 3 }, (b) => { +bench('recorded detail', { samples: 3, warmup: 0 }, (b) => { b.record({ __proto__: null, detail: { diff --git a/test/fixtures/bench-runner/serial.js b/test/fixtures/bench-runner/serial.js index 4e6f44c3b679..b177080425c7 100644 --- a/test/fixtures/bench-runner/serial.js +++ b/test/fixtures/bench-runner/serial.js @@ -9,7 +9,7 @@ module.exports = function register(name) { fs.writeFileSync(lock, `${process.pid}`, { flag: 'wx' }); process.on('exit', () => fs.unlinkSync(lock)); - bench(name, { samples: 1 }, async (b) => { + bench(name, { samples: 1, warmup: 0 }, async (b) => { b.start(); await setTimeout(25); b.end(1); diff --git a/test/fixtures/bench-runner/utf8-output.cjs b/test/fixtures/bench-runner/utf8-output.cjs index 7dab41cd7c22..6937432a4a80 100644 --- a/test/fixtures/bench-runner/utf8-output.cjs +++ b/test/fixtures/bench-runner/utf8-output.cjs @@ -6,7 +6,7 @@ const { bench } = require('node:bench'); const output = Buffer.from('split:\u20ac\n'); process.stdout.write(output.subarray(0, 7)); -bench('UTF-8 output', { samples: 1 }, async (b) => { +bench('UTF-8 output', { samples: 1, warmup: 0 }, async (b) => { b.start(); await setTimeout(20); process.stdout.write(output.subarray(7)); diff --git a/test/parallel/test-bench-auto-run.js b/test/parallel/test-bench-auto-run.js index 82e91f97b4a4..f8a500312bc3 100644 --- a/test/parallel/test-bench-auto-run.js +++ b/test/parallel/test-bench-auto-run.js @@ -14,7 +14,9 @@ const child = spawnSync(process.execPath, [ ]); assert.strictEqual(child.status, 1); -const completion = bench('automatic execution', common.mustCall((b) => { +const completion = bench('automatic execution', { + warmup: 0, +}, common.mustCall((b) => { b.record({ duration_ns: 1n, operations: 1 }); }, 30)); diff --git a/test/parallel/test-bench-context-control.js b/test/parallel/test-bench-context-control.js index 56c170211ac8..1657e0c169c1 100644 --- a/test/parallel/test-bench-context-control.js +++ b/test/parallel/test-bench-context-control.js @@ -36,7 +36,7 @@ const { createRunner } = require('node:bench'); }, 4)); const recordedCompletion = runner.bench( - 'recorded', { samples: 3 }, common.mustCall((b) => { + 'recorded', { samples: 3, warmup: 0 }, common.mustCall((b) => { const detail = { source: 'worker', value: 1n }; const sample = b.record({ __proto__: null, @@ -64,6 +64,7 @@ const { createRunner } = require('node:bench'); ]; const variableCompletion = runner.bench('variable batch', { samples: variableSamples.length, + warmup: 0, }, common.mustCall((b) => { b.record(variableSamples[b.index]); }, variableSamples.length)); @@ -116,6 +117,7 @@ const { createRunner } = require('node:bench'); [slowSample, slowSample, slowSample, fastSample, fastSample]; const slowCompletion = slowRunner.bench('sub-resolution rates', { samples: slowSamples.length, + warmup: 0, }, common.mustCall((b) => { b.record(slowSamples[b.index]); }, slowSamples.length)); diff --git a/test/parallel/test-bench-create-runner.js b/test/parallel/test-bench-create-runner.js index d12bc5db4b6e..c5cef7b09126 100644 --- a/test/parallel/test-bench-create-runner.js +++ b/test/parallel/test-bench-create-runner.js @@ -17,12 +17,12 @@ const { setImmediate } = require('timers/promises'); second.before(common.mustCall()); const firstCompletion = first.bench( - 'same name', { samples: 2 }, common.mustCall((b) => { + 'same name', { samples: 2, warmup: 0 }, common.mustCall((b) => { firstCalls++; completeSample(b); }, 2)); const secondCompletion = second.bench( - 'same name', { samples: 1 }, common.mustCall((b) => { + 'same name', { samples: 1, warmup: 0 }, common.mustCall((b) => { secondCalls++; completeSample(b); })); diff --git a/test/parallel/test-bench-default-warmup.js b/test/parallel/test-bench-default-warmup.js new file mode 100644 index 000000000000..5ba531bd5a8b --- /dev/null +++ b/test/parallel/test-bench-default-warmup.js @@ -0,0 +1,35 @@ +// Flags: --experimental-bench --no-warnings +'use strict'; + +const common = require('../common'); +const { completeSample } = require('../common/bench'); +const assert = require('assert'); +const { createRunner } = require('node:bench'); + +// Benchmarks that do not set `warmup` run ten unreported invocations before +// the measured samples. +(async () => { + const runner = createRunner({ yieldBetweenSamples: false }); + const phases = []; + + const completion = runner.bench('default warmup', { + samples: 2, + }, common.mustCall((b) => { + phases.push(b.phase); + completeSample(b); + }, 12)); + + const records = await runner.run().toArray(); + const result = await completion; + + const plan = records.find(({ type }) => type === 'bench:plan').data; + assert.strictEqual(plan.warmup, 10); + assert.deepStrictEqual(phases, [ + ...Array(10).fill('warmup'), + 'measurement', + 'measurement', + ]); + assert.strictEqual(result.samples.length, 2); + assert.strictEqual( + records.filter(({ type }) => type === 'bench:sample').length, 2); +})().then(common.mustCall()); diff --git a/test/parallel/test-bench-diagnostic-channels.js b/test/parallel/test-bench-diagnostic-channels.js index 1c6d3f60135b..f8afcd753b13 100644 --- a/test/parallel/test-bench-diagnostic-channels.js +++ b/test/parallel/test-bench-diagnostic-channels.js @@ -33,6 +33,7 @@ async function testCapture() { runner.bench('captured', { diagnosticChannels: [benchmarkName, nestedName], samples: 1, + warmup: 0, }, common.mustCall((context) => { const message = { value: 1 }; inheritedChannel.publish(message); @@ -45,7 +46,10 @@ async function testCapture() { })); })); })); - runner.bench('not captured', { samples: 1 }, common.mustCall((context) => { + runner.bench('not captured', { + samples: 1, + warmup: 0, + }, common.mustCall((context) => { inheritedChannel.publish({ value: 6 }); recordSample(context); })); @@ -79,6 +83,7 @@ async function testUncloneableMessage() { runner.bench('uncloneable', { diagnosticChannels: [name], samples: 1, + warmup: 0, }, common.mustCall((context) => { channel.publish(() => {}); recordSample(context); @@ -99,6 +104,7 @@ async function testAbortCleanup() { diagnosticChannels: [name], samples: 1, signal: controller.signal, + warmup: 0, }, common.mustCall((context) => { controller.abort(new Error('stop')); assert.strictEqual(channel.hasSubscribers, false); diff --git a/test/parallel/test-bench-diagnostics.js b/test/parallel/test-bench-diagnostics.js index b5383f2bdacc..1ba8cdcf01f4 100644 --- a/test/parallel/test-bench-diagnostics.js +++ b/test/parallel/test-bench-diagnostics.js @@ -19,6 +19,7 @@ async function testListenerFailure(callbackError) { const listenerError = new Error('diagnostic listener failed'); const completion = runner.bench('listener failure', { samples: 1, + warmup: 0, }, common.mustCall((b) => { b.diagnostic('listener failure'); if (callbackError !== undefined) throw callbackError; @@ -40,6 +41,7 @@ async function testAbortDuringDiagnosticDelivery() { const controller = new AbortController(); const completion = runner.bench('abort diagnostics', { samples: 1, + warmup: 0, }, common.mustCall((b) => { for (let i = 0; i < 32; i++) b.diagnostic(`diagnostic ${i}`); recordSample(b); @@ -82,6 +84,7 @@ async function testAfterEachFailurePrecedence() { })); const completion = runner.bench('afterEach precedence', { samples: 1, + warmup: 0, }, common.mustCall((b) => { b.diagnostic('before failures'); throw callbackError; @@ -112,6 +115,7 @@ async function testAfterEachFailurePrecedence() { const expectedError = new Error('benchmark failed'); const failed = runner.bench('failed diagnostic', { samples: 1, + warmup: 0, }, common.mustCall((b) => { b.diagnostic('before failure', { detail: { retained: true }, diff --git a/test/parallel/test-bench-errors.js b/test/parallel/test-bench-errors.js index 13efa51bf92e..5363efd6b14e 100644 --- a/test/parallel/test-bench-errors.js +++ b/test/parallel/test-bench-errors.js @@ -67,7 +67,7 @@ suiteCompletion.then(common.mustCall()); const completions = []; const sampleNames = []; let summary; -const stream = run(); +const stream = run({ warmup: 0 }); stream.on('bench:complete', (result) => completions.push(result)); stream.on('bench:sample', (sample) => sampleNames.push(sample.name)); stream.on('bench:summary', (result) => { summary = result; }); diff --git a/test/parallel/test-bench-filtering.js b/test/parallel/test-bench-filtering.js index b9a3124d8b7e..49954cfa6fbd 100644 --- a/test/parallel/test-bench-filtering.js +++ b/test/parallel/test-bench-filtering.js @@ -16,7 +16,7 @@ function complete(name) { } suite('selected', { only: true }, () => { - bench('included', { samples: 1 }, complete('included')); + bench('included', { samples: 1, warmup: 0 }, complete('included')); bench.skip('explicitly skipped', { samples: 1 }, common.mustNotCall()); bench('pattern filtered', { samples: 1 }, diff --git a/test/parallel/test-bench-run.js b/test/parallel/test-bench-run.js index 0107150262d7..abf31125500d 100644 --- a/test/parallel/test-bench-run.js +++ b/test/parallel/test-bench-run.js @@ -47,7 +47,7 @@ const suiteCompletion = suite('group', { tags: ['Group'] }, async () => { active = false; }, 3)); - bench('async', { samples: 2 }, common.mustCall(async (b) => { + bench('async', { samples: 2, warmup: 0 }, common.mustCall(async (b) => { assert.strictEqual(active, false); active = true; contexts.add(b); diff --git a/test/parallel/test-bench-stream.js b/test/parallel/test-bench-stream.js index dbb68f8be971..59d544400ddc 100644 --- a/test/parallel/test-bench-stream.js +++ b/test/parallel/test-bench-stream.js @@ -25,7 +25,7 @@ async function testReadableBackpressure() { calls++; recordSample(b); }); - const stream = runner.run(); + const stream = runner.run({ warmup: 0 }); const iterator = stream[Symbol.asyncIterator](); const first = await iterator.next(); @@ -59,7 +59,7 @@ async function testPlanBackpressure() { recordSample(b); })); } - const stream = runner.run(); + const stream = runner.run({ warmup: 0 }); const iterator = stream[Symbol.asyncIterator](); const first = await iterator.next(); @@ -90,7 +90,7 @@ async function testDestroyWhileBlocked() { samples: 1, }, recordSample)); } - const stream = runner.run(); + const stream = runner.run({ warmup: 0 }); const unblocked = stream.waitForDrain(); const iterator = stream[Symbol.asyncIterator](); await iterator.next(); @@ -120,7 +120,7 @@ async function testNamedEventsWithoutReading() { calls++; recordSample(b); }); - const stream = runner.run(); + const stream = runner.run({ warmup: 0 }); const summary = await new Promise((resolve) => { stream.once('bench:summary', resolve); }); @@ -140,7 +140,7 @@ async function testCancellationCompletesBenchmarks() { const second = runner.bench('continues headlessly', { samples: 1, }, recordSample); - const stream = runner.run(); + const stream = runner.run({ warmup: 0 }); const iterator = stream[Symbol.asyncIterator](); await iterator.next(); @@ -162,7 +162,7 @@ async function testDeliveryDoesNotConsumeTimeout() { samples: 32, timeout, }, recordSample); - const stream = runner.run(); + const stream = runner.run({ warmup: 0 }); const iterator = stream[Symbol.asyncIterator](); await iterator.next(); @@ -182,7 +182,7 @@ async function testReportingFailureSettlesBenchmarks() { const failure = new Error('record listener failed'); const first = runner.bench('reported', { samples: 1 }, recordSample); const second = runner.bench('settled', { samples: 1 }, recordSample); - const stream = runner.run(); + const stream = runner.run({ warmup: 0 }); stream.once('bench:complete', common.mustCall(() => { throw failure; })); @@ -200,7 +200,7 @@ async function testSummaryListenerFailure() { const completion = runner.bench('summary failure', { samples: 1, }, recordSample); - const stream = runner.run(); + const stream = runner.run({ warmup: 0 }); const diagnostics = []; stream.on('bench:diagnostic', (diagnostic) => { diagnostics.push(diagnostic); @@ -285,7 +285,7 @@ async function testRecordOwnership() { const afterTrap = runner.bench('after trapping error', { samples: 1, }, recordSample); - const stream = runner.run(); + const stream = runner.run({ warmup: 0 }); let eventSample; let eventComplete; let eventError; @@ -386,7 +386,7 @@ function testOperatorsWithoutStreamModule() { runner.bench('operators', { samples: 1 }, (b) => { b.record({ operations: 1, duration_ns: 1n }); }); - runner.run() + runner.run({ warmup: 0 }) .map((record) => record.type) .toArray() .then((types) => console.log(types.includes('bench:complete'))); diff --git a/test/parallel/test-bench-validation.js b/test/parallel/test-bench-validation.js index 2b508f18666f..e95a7ddf646e 100644 --- a/test/parallel/test-bench-validation.js +++ b/test/parallel/test-bench-validation.js @@ -69,7 +69,7 @@ bench('valid', { samples: 1 }, (b) => { completeSample(b); }); -const stream = run(); +const stream = run({ warmup: 0 }); stream.on('bench:start', common.mustCall(() => { assert.throws(() => bench('late', noop), { code: 'ERR_INVALID_STATE' }); }, 3)); diff --git a/test/parallel/test-bench-yield-between-samples.js b/test/parallel/test-bench-yield-between-samples.js index 92b51e9409a5..9a3409cb19bf 100644 --- a/test/parallel/test-bench-yield-between-samples.js +++ b/test/parallel/test-bench-yield-between-samples.js @@ -15,7 +15,7 @@ async function observe(factoryOptions, runOptions) { turnOccurred = true; }); - runner.bench('yielding', { samples: 2 }, (b) => { + runner.bench('yielding', { samples: 2, warmup: 0 }, (b) => { observed.push(turnOccurred); completeSample(b); });