From db66694cd75b05e26689db52bc6f5e591a72b7f0 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 13 Nov 2026 09:06:11 -0800 Subject: [PATCH 1/2] ci(mutation): exclude a LineSet mutant that allocates without bound Run 43437925608 on `main` at 3fd38e5d judged one mutant a timeout: `replace + with * in LineSet::last_word`. With `*` a set's last word is its offset times its length, `union_with` reserves that far, and every merge up a space tree multiplies the span again. The suite notices, with 551 failed tests in that log, but a nested fixture then asks for 168 GB and the test binary stalls past the timeout before it reports. Run 43278927651 judged the same mutant caught, so the verdict depends on the runner. The header's third kind now reads "non-terminating, or allocating without bound, under mutation". The configured list differs from the unconfigured one by exactly forty-seven mutants. Refs #1736 --- .cargo/mutants.toml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index 25a8ad1f..ad40b9f2 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -40,7 +40,7 @@ exclude_globs = ["**/*_tests.rs"] # - equivalent: identical on every reachable input, panics and side # effects included; # - unreachable: the code runs for no language; -# - non-terminating under mutation; +# - non-terminating, or allocating without bound, under mutation; # - same output, different work: no metric moves, and the entry names # the work the mutant skips or adds, which a test-support counter can # observe; @@ -233,6 +233,15 @@ exclude_re = [ # stale line stops excluding this mutant, and a line one off excludes # the live one. '^big-code-analysis-ast/src/checker\.rs:1105:9: replace \|\| with && in rust_attribute_run_member$', + # Allocates without bound under mutation. With `*` a set's last word + # is its offset times its length, so `union_with` reserves that far + # and every merge up a space tree multiplies the span again. The + # suite does notice: several hundred tests fail on the wrong rows. + # But a nested fixture then asks for a span of gigabytes, and on a + # hosted runner the test binary stalls past the timeout before it can + # report (run 43437925608: 168 GB requested, judged a timeout; run + # 43278927651 judged the same mutant caught). + '^src/metrics/loc/line_set\.rs:\d+:\d+: replace \+ with \* in LineSet::last_word$', # Non-terminating under mutation. The loop then breaks when # `goto_next_sibling` succeeds and repeats when it fails, so a list # with one child is visited forever. Its one caller walks the values From d42c3c170d25f595200a68d45009f995a383fe75 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic <490392+dekobon@users.noreply.github.com> Date: Fri, 13 Nov 2026 09:06:11 -0800 Subject: [PATCH 2/2] ci(mutation): split the run across twenty shards Run 43437925608 put eighteen shards of 291 mutants through in 108 to 237 minutes. The two slowest held no escape and no timeout, so the 2.2x spread is between runners doing the same work: 22 to 49 seconds per mutant. Two shards passed the 200 minutes the guide gives as the point to raise the count, and 237 leaves little under the 270-minute step timeout. Twenty holds about 261 of the list's 5227 mutants, roughly 213 minutes at the slowest rate seen. The guide now says to size for that rate and not the median. `make actionlint`, `make mutation-workflow-test` and `make mutation-report-test` pass. Refs #1736 --- .github/workflows/mutation-test.yml | 11 ++++++----- docs/development/mutation_testing.md | 15 +++++++++------ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/.github/workflows/mutation-test.yml b/.github/workflows/mutation-test.yml index 6191029a..453e11d4 100644 --- a/.github/workflows/mutation-test.yml +++ b/.github/workflows/mutation-test.yml @@ -46,10 +46,11 @@ jobs: # about 165 min at those slow-runner rates; the margin to 270 is for # the runner variance that made identical shards differ 1.6x. The # second complete run (43278927651, sixteen shards of 341 at - # 68b70b08) took 125 to 222 min, two shards past 200, so the count is - # eighteen: ~292 per shard of the 5249 the list holds once the test - # modules and the excluded mutants are out, about 190 min at that - # run's slowest rate of 39 s per mutant. + # 68b70b08) took 125 to 222 min, and the third (43437925608, eighteen + # shards of 291 at 3fd38e5d) 108 to 237: its two slowest shards held + # no escape, so the 2.2x spread is the runners', at 22 to 49 s per + # mutant. The count is twenty: ~261 per shard of the 5227 the list + # holds, about 213 min at that slowest rate. prepare: name: shard list runs-on: ubuntu-latest @@ -61,7 +62,7 @@ jobs: - id: define run: | set -euo pipefail - shards='[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]' + shards='[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]' echo "shards=${shards}" >> "$GITHUB_OUTPUT" echo "count=$(jq -n "${shards} | length")" >> "$GITHUB_OUTPUT" diff --git a/docs/development/mutation_testing.md b/docs/development/mutation_testing.md index f5d34332..490d42d0 100644 --- a/docs/development/mutation_testing.md +++ b/docs/development/mutation_testing.md @@ -128,12 +128,15 @@ artifact that records per-mutant `duration` — prices a root-crate mutant at ~28 seconds and an ast-crate one, which rebuilds both crates, at ~39, with a missed mutant about twice either since both suites then run to completion. The ast crate holds about 31 % of the -list since #1728 widened the selection. The current eighteen rests on -the second complete run (43278927651, 2026-11-12, at 68b70b08): its -sixteen shards of 341 mutants run for 125 to 222 minutes, two of them -past 200, and the slowest prices a mutant at 39 seconds. The list holds -5249 mutants with the test modules and the excluded mutants out, so -eighteen shards hold about 292 each, roughly 190 minutes at that rate. +list since #1728 widened the selection. The current twenty rests on +the third complete run (43437925608, 2026-11-13, at 3fd38e5d): its +eighteen shards of 291 mutants run for 108 to 237 minutes, two of them +past 200. Those two hold no escape, so the spread is the runners': 22 +to 49 seconds per mutant for the same work. The list holds 5227 +mutants with the test modules and the excluded mutants out, so twenty +shards hold about 261 each, roughly 213 minutes at the slowest rate +seen. Size for that rate, not the median: a shard that passes 270 is +cancelled and the run does not count. The laptop figure the count was first sized from, about ten seconds per mutant, is roughly half the hosted cost. Both halves of `utils/mutation-report.py` are covered by `make