diff --git a/benchmark/README.md b/benchmark/README.md index d2d1ac6a0f7c..78a55fba453f 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -80,9 +80,9 @@ Rscript benchmark/compare.R < compare-node-bench.csv Pass `--analyze` to run the same Welch analysis inline. `--max-regression N` implies `--analyze` and makes the command fail only when the Holm-Bonferroni -adjusted p-value is below 0.05 and the full 95% confidence interval is worse -than `-N%`. Requiring both conditions prevents a noisy point estimate from -failing a regression gate. +adjusted one-sided p-value against the `N%` threshold is below 0.05 and the full +95% confidence interval is worse than `-N%`. Requiring both conditions prevents +a noisy point estimate from failing a regression gate. ```console ./node benchmark/compare-node-bench.js \ diff --git a/benchmark/_node-bench-analysis.js b/benchmark/_node-bench-analysis.js index 5fa3c34d3976..bf0f7402b791 100644 --- a/benchmark/_node-bench-analysis.js +++ b/benchmark/_node-bench-analysis.js @@ -31,8 +31,19 @@ function holmAdjust(pValues) { return adjusted; } +function thresholdPValue(oldRates, newHistogram, scale, maxRegression) { + const factor = 1 - maxRegression / 100; + if (factor <= 0) return 1; + const thresholdHistogram = createRateHistogram( + oldRates.map((rate) => rate * factor), scale, 3); + const result = thresholdHistogram.welchTest(newHistogram); + if (Number.isNaN(result.pValue)) return 1; + return result.tStatistic > 0 ? + result.pValue / 2 : 1 - result.pValue / 2; +} + function isRegressionFailure(row, maxRegression) { - return row.pAdjusted < 0.05 && + return row.pThresholdAdjusted < 0.05 && row.improvement + row.ci95 < -maxRegression; } @@ -80,7 +91,7 @@ function analyzeCompare(samples, scale, maxRegression) { result.confidenceInterval.lower) / 2; return (half / (oldMean * scale)) * 100; }; - rows.push({ + const row = { ci95: ciPercent(w95), ci99: ciPercent(w99), ci999: ciPercent(w999), @@ -88,14 +99,24 @@ function analyzeCompare(samples, scale, maxRegression) { name, pValue: Number.isNaN(w95.pValue) ? 1 : w95.pValue, stars, - }); + }; + if (maxRegression !== undefined) { + row.pThreshold = thresholdPValue( + oldRates, newHistogram, scale, maxRegression); + } + rows.push(row); } const adjusted = holmAdjust(rows.map(({ pValue }) => pValue)); + const thresholdAdjusted = maxRegression === undefined ? null : + holmAdjust(rows.map(({ pThreshold }) => pThreshold)); let underpowered = 0; for (let index = 0; index < rows.length; index++) { const row = rows[index]; row.pAdjusted = adjusted[index]; + if (thresholdAdjusted !== null) { + row.pThresholdAdjusted = thresholdAdjusted[index]; + } row.inconclusive = maxRegression > 0 && row.stars.trim() === '' && row.ci95 > maxRegression; @@ -146,8 +167,13 @@ function analyzeCompare(samples, scale, maxRegression) { `After Holm-Bonferroni correction across ${rows.length} comparison` + `${rows.length === 1 ? '' : 's'}, ${significant} remain` + `${significant === 1 ? 's' : ''} significant at 5%.`, - '--max-regression uses the corrected values.', ); + if (maxRegression !== undefined) { + output.push( + `For --max-regression, one-sided p-values against the ` + + `${maxRegression}% threshold were corrected separately.`, + ); + } if (maxRegression > 0 && underpowered > 0) { output.push(''); @@ -159,21 +185,22 @@ function analyzeCompare(samples, scale, maxRegression) { ); } - const failures = maxRegression > 0 ? + const failures = maxRegression !== undefined ? rows.filter((row) => isRegressionFailure(row, maxRegression)) : []; if (failures.length > 0) { output.push(''); output.push( `FAIL: ${failures.length} benchmark${failures.length === 1 ? '' : 's'}` + ` regressed by more than ${maxRegression}% (the 95% interval excludes ` + - `the threshold and significance is family-wise corrected across ` + + `the threshold and its one-sided test is family-wise corrected across ` + `${rows.length} comparisons):`, ); for (const failure of failures) { output.push( ` ${failure.name} ${failure.improvement.toFixed(2)}% ` + `(95% CI up to ${(failure.improvement + failure.ci95).toFixed(2)}%, ` + - `adjusted p=${failure.pAdjusted.toExponential(2)})`, + `adjusted threshold p=` + + `${failure.pThresholdAdjusted.toExponential(2)})`, ); } } diff --git a/benchmark/compare-node-bench.js b/benchmark/compare-node-bench.js index d2bc6071bed7..295f15f0fdc1 100644 --- a/benchmark/compare-node-bench.js +++ b/benchmark/compare-node-bench.js @@ -36,9 +36,10 @@ async function main() { const runs = parseInteger(cli.optional.runs, 30, '--runs', 1); const warmup = parseInteger(cli.optional.warmup, 0, '--warmup', 0); const scale = parseInteger(cli.optional.scale, 1000, '--scale', 1); + const hasMaxRegression = cli.optional['max-regression'] !== undefined; const maxRegression = parseNumber( cli.optional['max-regression'], 0, '--max-regression', 0); - const analyze = !!cli.optional.analyze || maxRegression > 0; + const analyze = !!cli.optional.analyze || hasMaxRegression; const options = { namePattern: cli.optional['name-pattern'], nodeArgs: cli.optional['node-arg'], @@ -96,7 +97,8 @@ async function main() { } if (analyze) { - const result = analyzeCompare(rows, scale, maxRegression); + const result = analyzeCompare( + rows, scale, hasMaxRegression ? maxRegression : undefined); process.stdout.write(result.output); if (result.failed) process.exitCode = 1; return; diff --git a/doc/api/bench.md b/doc/api/bench.md index 7cb5720ffe28..b25503830f2c 100644 --- a/doc/api/bench.md +++ b/doc/api/bench.md @@ -11,7 +11,8 @@ added: REPLACEME The `node:bench` module supports defining and running JavaScript benchmarks in -the current process. To access it: +the current process, and running one benchmark file in a fresh child process. +To access it: ```mjs import { bench, suite } from 'node:bench'; @@ -38,12 +39,17 @@ suite('URL', () => { params: { input: 'short' }, }, (b) => { const operations = 10_000; + let totalLength = 0; b.start(); for (let i = 0; i < operations; i++) { - new URL(input); + totalLength += new URL(input).href.length; } b.end(operations); + + if (totalLength !== operations * input.length) { + throw new Error('Unexpected URL result'); + } }); }); ``` @@ -77,10 +83,70 @@ system load can all affect results. Keep raw samples when comparing results and investigate noisy or skewed distributions rather than treating a confidence interval as a pass/fail threshold. +### Measurement integrity + +A statistically consistent result does not prove that a benchmark measured the +intended work. An optimizing runtime can remove work whose result is unused or +specialize it more narrowly than the workload being modeled. Framework and loop +overhead can also dominate operations that are too short. To reduce these risks: + +* Make values produced by measured work observable outside the measured + interval, for example by validating an aggregate derived from every result. + Passing them only through unused local computations is insufficient. +* Perform enough operations in each sample to amortize fixed timer reads and + calls to `context.start()` and `context.end()`. If loop bookkeeping is material + relative to one operation, batch multiple operations per iteration and report + the total operation count. +* Inspect raw `samples` for trends that indicate insufficient warmup or + optimization tiering, pauses consistent with garbage collection, and + multimodal distributions. +* Validate surprising results with an independent benchmark shape that performs + the same intended work differently. + +`node:bench` does not force a particular optimization state or infer whether an +engine eliminated work. Such controls and diagnostics are runtime-specific and +heuristic, and do not replace validating the benchmark workload. + +### Dynamic sampling and variable batches + Calling `context.done()` during a measured sample completes the benchmark after that sample. This allows a higher-level tool to treat `samples` as a maximum and implement a dynamic sampling policy. +The number of operations can differ between samples. Summary statistics treat +each sample's `rate` as one equally weighted observation. In particular, +`summary.mean` is the arithmetic mean of the per-sample rates. It is not the +pooled throughput calculated as: + +```text +1_000_000_000 * sum(sample.operations) / sum(sample.duration_ns) +``` + +The two values can differ when sample durations vary because pooled throughput +weights each per-sample rate by its duration. A higher-level tool that varies +batch sizes should choose the aggregation that matches its analysis. It can +calculate pooled throughput from the raw `samples`; operation counts should be +summed as `bigint` values because their total can exceed +`Number.MAX_SAFE_INTEGER` even though each count cannot. + +### Comparing benchmark results + +`node:bench` does not designate a benchmark as a baseline or produce a pass/fail +comparison between runs. It exposes raw samples, stable benchmark identities, +parameters, and tags so that comparison policy can remain in higher-level +tools. A tool can use `benchId` to match the same declaration and parameters +across compatible source layouts, and use a tag or its own metadata to identify +a baseline. + +Comparison tools should retain the raw sample rates and verify that execution +plans and relevant environment details are comparable. The appropriate analysis +depends on the experimental design and distribution. For example, independent +samples might use Welch's t-test or a rank-based test, while observations that +were deliberately paired require paired analysis. Tools should also consider +effect sizes, uncertainty, and correction when testing multiple benchmarks. +The general-purpose {Histogram} statistics in `node:perf_hooks` can support such +analysis, but the runner does not select a method or significance threshold. + ## Reusable runners The module-level declaration functions use a shared runner and schedule it @@ -132,11 +198,32 @@ they do not corrupt reporter output. has lower startup overhead, but module, heap, and process state carry between files, and user writes share stdout and stderr with reporters. +Worker-thread isolation is not a CLI mode. Each newly constructed {Worker} has a +separate V8 isolate, JavaScript heap, and event loop, typically with lower +startup cost than a child process. Reusing a worker preserves its module and heap +state. Workers also share libuv's process-wide thread pool and can share +process-global native or addon state, so they do not provide the same boundary +as process isolation. + +Higher-level tools can experiment with worker isolation by loading benchmark +code inside a worker, measuring there, transferring structured sample data, and +passing it to [`context.record()`][]. The reported `duration_ns` can exclude +message transport when the worker captures both timestamps. Tools should +identify worker modules and workloads explicitly. They should not stringify +arbitrary functions or closures to move them between isolates, because closures +cannot be reconstructed with their original lexical environment. + Benchmark files passed to `--bench` should declare benchmarks but must not call `run()`. The CLI supports `--bench-name-pattern`, `--bench-samples`, `--bench-warmup`, `--bench-reporter`, and `--bench-reporter-destination`. See the [command-line options documentation][] for details. +Preload modules passed through `--require` or `--import` should not declare +benchmarks. Such declarations are not associated with an entry file and have +an `entryFile` value of `null`. Their `fileRunId` identifies the runner or child +execution in which they occurred. With process isolation, a preload is evaluated +and its declarations run once for every benchmark child process. + ## Benchmark reporters The built-in reporters are available from the scheme-only @@ -229,6 +316,9 @@ added: REPLACEME * `name` {string} The benchmark name. **Default:** The `name` property of `fn`, or `''` when `fn` has no name. * `options` {Object} + * `diagnosticChannels` {Array} String diagnostics channel names, deduplicated + and inherited from containing suites by union. Symbol values in the array + are silently ignored. **Default:** `[]`. * `only` {boolean} When any benchmark or containing suite has `only` set, benchmarks without `only` in their hierarchy are skipped. **Default:** `false`. @@ -259,12 +349,33 @@ samples, but their samples are discarded. An exception, rejection, timeout, abort, missing timing call, or duplicate timing call stops the current benchmark. Later benchmarks continue to run. +After a timeout or abort, the runner briefly waits for asynchronous benchmark +work to settle before continuing. If it remains pending, all later benchmarks +that were selected to run fail without running so that their measurements +cannot overlap with that work. + +For each warmup and measured callback, the runner subscribes to the configured +diagnostics channels. Each publication queues a context diagnostic whose +`message` is `{ name, message }`, containing the string channel name and the +published message. Subscriptions are removed when the callback settles or is +aborted. + A timeout or abort cannot interrupt synchronous JavaScript and does not forcibly cancel asynchronous work that ignores `context.signal`. -The stable `benchId` is based on the source file, hierarchical suite and -benchmark names, and canonicalized parameters. Declaring the same identity -more than once reports an error rather than merging the samples. +The `benchId` is based on the declaration source file, hierarchical suite and +benchmark names, and canonicalized parameters. It is stable for repeated runs +from the same source location, but the embedded source value is not normalized +across checkout roots, module formats, operating systems, or path casing. + +Execution scope is represented separately. A `runId` identifies one logical +run, while `fileRunId` identifies a file runner or child execution within that +run. The `entryFile` field records which entry-file import caused a declaration +and is `null` for declarations made by preload modules. +The same `benchId` can therefore occur under multiple `fileRunId` values when +entry files use a shared declaration helper. Declaring the same `benchId` more +than once within one file execution scope reports an error rather than merging +the samples. ### `bench.skip([name][, options], fn)` @@ -291,6 +402,9 @@ added: REPLACEME * `name` {string} The suite name. **Default:** The `name` property of `fn`, or `''` when `fn` has no name. * `options` {Object} + * `diagnosticChannels` {Array} String diagnostics channel names inherited by + nested suites and benchmarks. Symbol values in the array are silently + ignored. **Default:** `[]`. * `only` {boolean} Selects all benchmarks nested in this suite. **Default:** `false`. * `skip` {boolean|string} Skips all benchmarks nested in this suite. @@ -401,6 +515,55 @@ for await (const { type, data } of run()) { } ``` +## `runFile(path[, options])` + + + +* `path` {string|Buffer|URL} The path of one benchmark module. +* `options` {Object} + * `env` {Object} The child process environment. Property values must be + strings or `undefined`. This replaces, rather than extends, the parent + environment. **Default:** A snapshot of `process.env`. + * `execArgv` {string\[]} Node.js command-line options for the child process. + This replaces, rather than extends, inherited options. Benchmark runner + options, positional arguments, and options that select another execution + mode are not allowed. **Default:** Compatible options inherited from the + current process. + * `signal` {AbortSignal} Terminates the child process when aborted. +* Returns: {BenchmarksStream} + +Runs exactly one benchmark module in a fresh child process and returns its +object-mode event stream. A relative `path` is resolved from the current working +directory when `runFile()` is called. `path` is not interpreted as a glob. +Unless the signal is aborted or the stream is destroyed before startup, every +call uses a new child. Input discovery, ordering, concurrency, retries, and +multi-file scheduling remain the caller's responsibility. + +When the Permission Model is enabled, the caller must have file system read +access to `path` and permission to create child processes. + +Records use advanced child process serialization, preserving supported +structured values such as `bigint` and errors. Child writes to stdout and stderr +become `'bench:diagnostic'` records. A permission failure, module loading error, +abnormal child exit, or cancellation also emits an error diagnostic and produces +a terminal `'bench:summary'` whose `success` property is `false`; these execution +failures do not error the stream. If module evaluation fails after declaring +benchmarks, those declarations still run before the unsuccessful summary. + +`env`, effective inherited options, and an explicitly provided `execArgv` are +copied when `runFile()` is called. The runner removes `NODE_OPTIONS`, replaces +IPC-related environment variables, and sets its private child-context, run +identity, and file identity variables, overriding properties with those names +in `env`. Pass child Node.js options through `execArgv`, not `NODE_OPTIONS`. +Standard `child_process` environment propagation still applies, including +`NODE_V8_COVERAGE`, permission-model options, and required z/OS variables. +Aborting `signal` before the child starts produces an `AbortError` diagnostic +without spawning it. Aborting during execution sends `SIGTERM` to the child and +escalates to `SIGKILL` if it does not exit. Destroying the returned stream +follows the same termination procedure. + ## Class: `BenchContext` An instance of `BenchContext` is passed to every benchmark invocation. A new @@ -511,6 +674,34 @@ useful when a higher-level tool measures work in a worker and needs to exclude message transport from the duration. `record()` is mutually exclusive with `start()` and `end()` within one callback and must be called exactly once. +### `context.diagnostic(message[, options])` + + + +* `message` {any} A structured-cloneable diagnostic value. With CLI process + isolation, it must also be supported by advanced child process serialization. +* `options` {Object} + * `level` {string} Either `'info'` or `'warning'`. **Default:** `'info'`. + * `detail` {any} Additional structured-cloneable diagnostic data. With CLI + process isolation, it must also be supported by advanced child process + serialization. +* Returns: {undefined} + +Queues a diagnostic associated with the current benchmark, phase, and sample +index. Multiple diagnostics preserve call order. They are emitted after the +sample callback settles and before that sample's `'bench:sample'` event. Warmup +diagnostics are emitted even though warmup samples are not. Diagnostics queued +before a callback failure are emitted before the failed `'bench:complete'` +event and do not themselves cause the benchmark to fail. If a timeout or abort +wins before the callback settles, queued diagnostics might not be emitted. + +The message and detail are cloned synchronously. Options are also validated +synchronously. Calling `diagnostic()` between `context.start()` and +`context.end()` therefore includes that work in the measured duration. Invalid +arguments or an uncloneable message or detail violate the sample contract. + ### `context.done()`