lib: default node:bench warmup to 10 invocations

doc/api/bench.md

@@ -74,6 +74,12 @@ once to provide an externally measured sample. Setup before `start()` and
 cleanup after `end()` are outside the measured region. Promise-returning
 functions are awaited.
 
+By default, ten unreported warmup invocations run before the measured samples
+so that early samples are less likely to be taken before the measured code has
+been optimized. A fixed warmup count does not guarantee a stable optimization
+state; inspect raw `samples` and increase `warmup` when early samples are
+consistently slower. Set `warmup` to `0` to measure from the first invocation.
+
 By default, an event loop turn occurs between sample invocations. An embedded
 runner can disable this using `yieldBetweenSamples`. The runner executes
 benchmarks serially, but it does not provide process isolation. Other work in
@@ -336,7 +342,7 @@ added: v26.9.0
   * `timeout` {number} The number of milliseconds after which the benchmark
     fails. **Default:** `Infinity`.
   * `warmup` {number} The number of unreported callback invocations before
-    measured samples. Must be a 32-bit unsigned integer. **Default:** `0`.
+    measured samples. Must be a 32-bit unsigned integer. **Default:** `10`.
 * `fn` {Function|AsyncFunction} The benchmark function. It receives a
   {BenchContext}.
 * Returns: {Promise} Fulfilled with the benchmark result after a top-level

doc/api/cli.md

@@ -630,6 +630,8 @@ added: v26.9.0
 
 Overrides the number of unreported warmup callback invocations for every
 selected benchmark. `count` must be an integer between `0` and `4294967295`.
+Without this option, each benchmark uses its own `warmup` value, which defaults
+to `10`.
 
 ### `--build-sea=config`
 

doc/node.1

@@ -384,6 +384,8 @@ selected benchmark. A benchmark may finish earlier by calling
 .It Fl -bench-warmup Ns = Ns Ar count
 Overrides the number of unreported warmup callback invocations for every
 selected benchmark. \fBcount\fR must be an integer between \fB0\fR and \fB4294967295\fR.
+Without this option, each benchmark uses its own \fBwarmup\fR value, which defaults
+to \fB10\fR.
 .
 .It Fl -build-sea Ns = Ns Ar config
 Generates a single executable application from a JSON

lib/internal/bench_runner/benchmark.js

@@ -47,7 +47,7 @@ const { structuredClone } = require('internal/worker/js_transferable');
 
 const { bigint: hrtime } = process.hrtime;
 const kDefaultSamples = 30;
-const kDefaultWarmup = 0;
+const kDefaultWarmup = 10;
 const kEmptyDiagnosticChannels = ObjectFreeze([]);
 const kEmptyNamePath = ObjectFreeze([]);
 const kEmptyParams = ObjectFreeze({ __proto__: null });

test/fixtures/bench-runner/diagnostic.cjs

@@ -9,6 +9,7 @@ const diagnosticChannel = channel(channelName);
 bench('diagnostic relay', {
   diagnosticChannels: [channelName],
   samples: 1,
+  warmup: 0,
 }, (b) => {
   const message = { value: 42n };
   diagnosticChannel.publish(message);

test/fixtures/bench-runner/many-records.cjs

@@ -2,7 +2,7 @@
 
 const { bench } = require('node:bench');
 
-bench('many records', { samples: 30 }, (b) => {
+bench('many records', { samples: 30, warmup: 0 }, (b) => {
   process.stdout.write(`${b.index}\n`);
   b.record({ duration_ns: 1n, operations: 1 });
 });

test/fixtures/bench-runner/recorded-detail.cjs

@@ -2,7 +2,7 @@
 
 const { bench } = require('node:bench');
 
-bench('recorded detail', { samples: 3 }, (b) => {
+bench('recorded detail', { samples: 3, warmup: 0 }, (b) => {
   b.record({
     __proto__: null,
     detail: {

test/fixtures/bench-runner/serial.js

@@ -9,7 +9,7 @@ module.exports = function register(name) {
   fs.writeFileSync(lock, `${process.pid}`, { flag: 'wx' });
   process.on('exit', () => fs.unlinkSync(lock));
 
-  bench(name, { samples: 1 }, async (b) => {
+  bench(name, { samples: 1, warmup: 0 }, async (b) => {
     b.start();
     await setTimeout(25);
     b.end(1);

test/fixtures/bench-runner/utf8-output.cjs

@@ -6,7 +6,7 @@ const { bench } = require('node:bench');
 const output = Buffer.from('split:\u20ac\n');
 process.stdout.write(output.subarray(0, 7));
 
-bench('UTF-8 output', { samples: 1 }, async (b) => {
+bench('UTF-8 output', { samples: 1, warmup: 0 }, async (b) => {
   b.start();
   await setTimeout(20);
   process.stdout.write(output.subarray(7));

test/parallel/test-bench-auto-run.js

@@ -14,7 +14,9 @@ const child = spawnSync(process.execPath, [
 ]);
 assert.strictEqual(child.status, 1);
 
-const completion = bench('automatic execution', common.mustCall((b) => {
+const completion = bench('automatic execution', {
+  warmup: 0,
+}, common.mustCall((b) => {
   b.record({ duration_ns: 1n, operations: 1 });
 }, 30));
 

test/parallel/test-bench-context-control.js

@@ -36,7 +36,7 @@ const { createRunner } = require('node:bench');
   }, 4));
 
   const recordedCompletion = runner.bench(
-    'recorded', { samples: 3 }, common.mustCall((b) => {
+    'recorded', { samples: 3, warmup: 0 }, common.mustCall((b) => {
       const detail = { source: 'worker', value: 1n };
       const sample = b.record({
         __proto__: null,
@@ -64,6 +64,7 @@ const { createRunner } = require('node:bench');
   ];
   const variableCompletion = runner.bench('variable batch', {
     samples: variableSamples.length,
+    warmup: 0,
   }, common.mustCall((b) => {
     b.record(variableSamples[b.index]);
   }, variableSamples.length));
@@ -116,6 +117,7 @@ const { createRunner } = require('node:bench');
     [slowSample, slowSample, slowSample, fastSample, fastSample];
   const slowCompletion = slowRunner.bench('sub-resolution rates', {
     samples: slowSamples.length,
+    warmup: 0,
   }, common.mustCall((b) => {
     b.record(slowSamples[b.index]);
   }, slowSamples.length));

test/parallel/test-bench-create-runner.js

@@ -17,12 +17,12 @@ const { setImmediate } = require('timers/promises');
   second.before(common.mustCall());
 
   const firstCompletion = first.bench(
-    'same name', { samples: 2 }, common.mustCall((b) => {
+    'same name', { samples: 2, warmup: 0 }, common.mustCall((b) => {
       firstCalls++;
       completeSample(b);
     }, 2));
   const secondCompletion = second.bench(
-    'same name', { samples: 1 }, common.mustCall((b) => {
+    'same name', { samples: 1, warmup: 0 }, common.mustCall((b) => {
       secondCalls++;
       completeSample(b);
     }));

test/parallel/test-bench-default-warmup.js

@@ -0,0 +1,35 @@
+// Flags: --experimental-bench --no-warnings
+'use strict';
+
+const common = require('../common');
+const { completeSample } = require('../common/bench');
+const assert = require('assert');
+const { createRunner } = require('node:bench');
+
+// Benchmarks that do not set `warmup` run ten unreported invocations before
+// the measured samples.
+(async () => {
+  const runner = createRunner({ yieldBetweenSamples: false });
+  const phases = [];
+
+  const completion = runner.bench('default warmup', {
+    samples: 2,
+  }, common.mustCall((b) => {
+    phases.push(b.phase);
+    completeSample(b);
+  }, 12));
+
+  const records = await runner.run().toArray();
+  const result = await completion;
+
+  const plan = records.find(({ type }) => type === 'bench:plan').data;
+  assert.strictEqual(plan.warmup, 10);
+  assert.deepStrictEqual(phases, [
+    ...Array(10).fill('warmup'),
+    'measurement',
+    'measurement',
+  ]);
+  assert.strictEqual(result.samples.length, 2);
+  assert.strictEqual(
+    records.filter(({ type }) => type === 'bench:sample').length, 2);
+})().then(common.mustCall());

test/parallel/test-bench-diagnostic-channels.js

@@ -33,6 +33,7 @@ async function testCapture() {
       runner.bench('captured', {
         diagnosticChannels: [benchmarkName, nestedName],
         samples: 1,
+        warmup: 0,
       }, common.mustCall((context) => {
         const message = { value: 1 };
         inheritedChannel.publish(message);
@@ -45,7 +46,10 @@ async function testCapture() {
       }));
     }));
   }));
-  runner.bench('not captured', { samples: 1 }, common.mustCall((context) => {
+  runner.bench('not captured', {
+    samples: 1,
+    warmup: 0,
+  }, common.mustCall((context) => {
     inheritedChannel.publish({ value: 6 });
     recordSample(context);
   }));
@@ -79,6 +83,7 @@ async function testUncloneableMessage() {
   runner.bench('uncloneable', {
     diagnosticChannels: [name],
     samples: 1,
+    warmup: 0,
   }, common.mustCall((context) => {
     channel.publish(() => {});
     recordSample(context);
@@ -99,6 +104,7 @@ async function testAbortCleanup() {
     diagnosticChannels: [name],
     samples: 1,
     signal: controller.signal,
+    warmup: 0,
   }, common.mustCall((context) => {
     controller.abort(new Error('stop'));
     assert.strictEqual(channel.hasSubscribers, false);

test/parallel/test-bench-diagnostics.js

@@ -19,6 +19,7 @@ async function testListenerFailure(callbackError) {
   const listenerError = new Error('diagnostic listener failed');
   const completion = runner.bench('listener failure', {
     samples: 1,
+    warmup: 0,
   }, common.mustCall((b) => {
     b.diagnostic('listener failure');
     if (callbackError !== undefined) throw callbackError;
@@ -40,6 +41,7 @@ async function testAbortDuringDiagnosticDelivery() {
   const controller = new AbortController();
   const completion = runner.bench('abort diagnostics', {
     samples: 1,
+    warmup: 0,
   }, common.mustCall((b) => {
     for (let i = 0; i < 32; i++) b.diagnostic(`diagnostic ${i}`);
     recordSample(b);
@@ -82,6 +84,7 @@ async function testAfterEachFailurePrecedence() {
   }));
   const completion = runner.bench('afterEach precedence', {
     samples: 1,
+    warmup: 0,
   }, common.mustCall((b) => {
     b.diagnostic('before failures');
     throw callbackError;
@@ -112,6 +115,7 @@ async function testAfterEachFailurePrecedence() {
   const expectedError = new Error('benchmark failed');
   const failed = runner.bench('failed diagnostic', {
     samples: 1,
+    warmup: 0,
   }, common.mustCall((b) => {
     b.diagnostic('before failure', {
       detail: { retained: true },

test/parallel/test-bench-errors.js

@@ -67,7 +67,7 @@ suiteCompletion.then(common.mustCall());
 const completions = [];
 const sampleNames = [];
 let summary;
-const stream = run();
+const stream = run({ warmup: 0 });
 stream.on('bench:complete', (result) => completions.push(result));
 stream.on('bench:sample', (sample) => sampleNames.push(sample.name));
 stream.on('bench:summary', (result) => { summary = result; });

test/parallel/test-bench-filtering.js

@@ -16,7 +16,7 @@ function complete(name) {
 }
 
 suite('selected', { only: true }, () => {
-  bench('included', { samples: 1 }, complete('included'));
+  bench('included', { samples: 1, warmup: 0 }, complete('included'));
   bench.skip('explicitly skipped', { samples: 1 },
              common.mustNotCall());
   bench('pattern filtered', { samples: 1 },

test/parallel/test-bench-run.js

@@ -47,7 +47,7 @@ const suiteCompletion = suite('group', { tags: ['Group'] }, async () => {
     active = false;
   }, 3));
 
-  bench('async', { samples: 2 }, common.mustCall(async (b) => {
+  bench('async', { samples: 2, warmup: 0 }, common.mustCall(async (b) => {
     assert.strictEqual(active, false);
     active = true;
     contexts.add(b);

test/parallel/test-bench-stream.js

@@ -25,7 +25,7 @@ async function testReadableBackpressure() {
     calls++;
     recordSample(b);
   });
-  const stream = runner.run();
+  const stream = runner.run({ warmup: 0 });
   const iterator = stream[Symbol.asyncIterator]();
   const first = await iterator.next();
 
@@ -59,7 +59,7 @@ async function testPlanBackpressure() {
       recordSample(b);
     }));
   }
-  const stream = runner.run();
+  const stream = runner.run({ warmup: 0 });
   const iterator = stream[Symbol.asyncIterator]();
   const first = await iterator.next();
 
@@ -90,7 +90,7 @@ async function testDestroyWhileBlocked() {
       samples: 1,
     }, recordSample));
   }
-  const stream = runner.run();
+  const stream = runner.run({ warmup: 0 });
   const unblocked = stream.waitForDrain();
   const iterator = stream[Symbol.asyncIterator]();
   await iterator.next();
@@ -120,7 +120,7 @@ async function testNamedEventsWithoutReading() {
     calls++;
     recordSample(b);
   });
-  const stream = runner.run();
+  const stream = runner.run({ warmup: 0 });
   const summary = await new Promise((resolve) => {
     stream.once('bench:summary', resolve);
   });
@@ -140,7 +140,7 @@ async function testCancellationCompletesBenchmarks() {
   const second = runner.bench('continues headlessly', {
     samples: 1,
   }, recordSample);
-  const stream = runner.run();
+  const stream = runner.run({ warmup: 0 });
   const iterator = stream[Symbol.asyncIterator]();
 
   await iterator.next();
@@ -162,7 +162,7 @@ async function testDeliveryDoesNotConsumeTimeout() {
     samples: 32,
     timeout,
   }, recordSample);
-  const stream = runner.run();
+  const stream = runner.run({ warmup: 0 });
   const iterator = stream[Symbol.asyncIterator]();
 
   await iterator.next();
@@ -182,7 +182,7 @@ async function testReportingFailureSettlesBenchmarks() {
   const failure = new Error('record listener failed');
   const first = runner.bench('reported', { samples: 1 }, recordSample);
   const second = runner.bench('settled', { samples: 1 }, recordSample);
-  const stream = runner.run();
+  const stream = runner.run({ warmup: 0 });
   stream.once('bench:complete', common.mustCall(() => {
     throw failure;
   }));
@@ -200,7 +200,7 @@ async function testSummaryListenerFailure() {
   const completion = runner.bench('summary failure', {
     samples: 1,
   }, recordSample);
-  const stream = runner.run();
+  const stream = runner.run({ warmup: 0 });
   const diagnostics = [];
   stream.on('bench:diagnostic', (diagnostic) => {
     diagnostics.push(diagnostic);
@@ -285,7 +285,7 @@ async function testRecordOwnership() {
   const afterTrap = runner.bench('after trapping error', {
     samples: 1,
   }, recordSample);
-  const stream = runner.run();
+  const stream = runner.run({ warmup: 0 });
   let eventSample;
   let eventComplete;
   let eventError;
@@ -386,7 +386,7 @@ function testOperatorsWithoutStreamModule() {
     runner.bench('operators', { samples: 1 }, (b) => {
       b.record({ operations: 1, duration_ns: 1n });
     });
-    runner.run()
+    runner.run({ warmup: 0 })
       .map((record) => record.type)
       .toArray()
       .then((types) => console.log(types.includes('bench:complete')));

test/parallel/test-bench-validation.js

@@ -69,7 +69,7 @@ bench('valid', { samples: 1 }, (b) => {
   completeSample(b);
 });
 
-const stream = run();
+const stream = run({ warmup: 0 });
 stream.on('bench:start', common.mustCall(() => {
   assert.throws(() => bench('late', noop), { code: 'ERR_INVALID_STATE' });
 }, 3));

test/parallel/test-bench-yield-between-samples.js

@@ -15,7 +15,7 @@ async function observe(factoryOptions, runOptions) {
     turnOccurred = true;
   });
 
-  runner.bench('yielding', { samples: 2 }, (b) => {
+  runner.bench('yielding', { samples: 2, warmup: 0 }, (b) => {
     observed.push(turnOccurred);
     completeSample(b);
   });