From 55a7af23dad917c01794420c534e60238be35274 Mon Sep 17 00:00:00 2001 From: Binay <150876063+bkd-dotcom@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:44:06 -0400 Subject: [PATCH 1/4] fix(ci): run the publish gates even when the push fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both leaderboard.yml and benchmark.yml order their gate step last on purpose — the page publishes first, then the gate fails the run so a broken defense cannot be missed. That ordering was unguarded: when the push step failed, the job aborted before the gate ever ran. The scheduled leaderboard run on 2026-09-07 is the case in point. It regenerated the page, was rejected pushing to protected main, and exited without ever executing "every governed defense must hold" — so a genuine defense regression would have been indistinguishable from a git error. Guard both gates with `!cancelled() && steps.regen.outcome == 'success'` so publishing and judging are independent: the gate runs whenever there are numbers to judge, and is skipped only when regeneration produced none. Co-Authored-By: Claude Opus 5 --- .github/workflows/benchmark.yml | 4 ++++ .github/workflows/leaderboard.yml | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 068bf5f..59fd49a 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -55,6 +55,7 @@ jobs: exit 1 } - name: Regenerate the head-to-head table + id: regen run: | mkdir -p docs { @@ -82,6 +83,9 @@ jobs: git pull --rebase --autostash origin "${{ github.event.repository.default_branch || 'main' }}" git push origin "HEAD:${{ github.event.repository.default_branch || 'main' }}" - name: Gate on parity (Signetry must keep 100% recall, 0 false positives) + # Must run even when publishing failed — see the same guard in leaderboard.yml. + # Without it a rejected push aborts the job before the regression is judged. + if: ${{ !cancelled() && steps.regen.outcome == 'success' }} run: | # Last, deliberately: the regression is published above whatever this says, # then this step fails the run so it cannot be missed. diff --git a/.github/workflows/leaderboard.yml b/.github/workflows/leaderboard.yml index f61d4fd..68776f3 100644 --- a/.github/workflows/leaderboard.yml +++ b/.github/workflows/leaderboard.yml @@ -63,6 +63,7 @@ jobs: fi - name: Regenerate the leaderboard + id: regen run: | mkdir -p docs # Record WHICH signetry-core produced the live row. A leaderboard that says @@ -111,6 +112,12 @@ jobs: git push origin "HEAD:${{ github.event.repository.default_branch || 'main' }}" - name: Gate — every governed defense must hold + # Must run even when publishing failed. Ordering this step last is deliberate, + # but `if:` is what makes the ordering safe: a rejected push aborts the job + # before the gate, so a broken defense would surface as a git error and the + # defense itself would never be judged. Skipped only when regeneration did not + # produce numbers to judge in the first place. + if: ${{ !cancelled() && steps.regen.outcome == 'success' }} run: | # Last, deliberately: the page is published above whatever this says, then this # step fails the run so a broken defense is impossible to miss. From 63a670e1484e35718770579f7664478d91144e1c Mon Sep 17 00:00:00 2001 From: Binay <150876063+bkd-dotcom@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:44:40 -0400 Subject: [PATCH 2/4] docs: refresh the leaderboard for signetry-core 0.8.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pyproject pins signetry-core @ v0.8.0, but the committed page still labelled the live row 0.7.0 — the version the page last successfully published under. Every run since has regenerated the correct label and then failed to push it, so the page has been stale rather than wrong-by- generation. This is byte-for-byte what the leaderboard workflow produces: regenerated with `signetry-eval leaderboard --with-detection --version 0.8.0`, the same command and engine-detection branch CI takes, yielding the identical two-line diff (one in .md, one in .json) that the failed runs attempted. Co-Authored-By: Claude Opus 5 --- docs/LEADERBOARD.json | 2 +- docs/LEADERBOARD.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/LEADERBOARD.json b/docs/LEADERBOARD.json index eaddcb1..887f751 100644 --- a/docs/LEADERBOARD.json +++ b/docs/LEADERBOARD.json @@ -6,7 +6,7 @@ "name": "signetry-core", "status": "reproduced", "url": "https://github.com/Signetry/core", - "version": "0.7.0", + "version": "0.8.0", "attacks_run": 5, "asr_ungoverned": 1.0, "asr_governed": 0.0, diff --git a/docs/LEADERBOARD.md b/docs/LEADERBOARD.md index a1962e0..a31af25 100644 --- a/docs/LEADERBOARD.md +++ b/docs/LEADERBOARD.md @@ -25,7 +25,7 @@ by blocking everything is not a win. | System | Evidence | Attacks | ASR ungoverned | ASR governed | Reduction | Utility preserved | |---|:---:|:---:|:---:|:---:|:---:|:---:| -| [signetry-core `0.7.0`](https://github.com/Signetry/core) | reproduced here | 5 | 100% | 0% | 100% | 100% | +| [signetry-core `0.8.0`](https://github.com/Signetry/core) | reproduced here | 5 | 100% | 0% | 100% | 100% | | ungoverned agent (no checkpoint) | reproduced here | 5 | 100% | 100% | 0% | — | ### What the governed column does and does not claim From 2e4319b641735e2fa428b7bc6071060b4016556a Mon Sep 17 00:00:00 2001 From: Binay <150876063+bkd-dotcom@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:03:40 -0400 Subject: [PATCH 3/4] fix(ci): verify the published pages instead of pushing them to main MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `leaderboard.yml` and `benchmark.yml` regenerated their page and pushed the result straight to `main`. `main` requires a pull request, so every run that actually had something to publish was rejected by branch protection: remote: error: GH006: Protected branch update failed for refs/heads/main. remote: error: - Changes must be made through a pull request. The weekly refresh (run 34128111507, 2026-09-07) and the v0.3.0 release run (33526247640, 2026-09-01) both died there, so the published numbers went stale rather than current — LEADERBOARD.json still labelled the live row 0.7.0 after pyproject was pinned to signetry-core v0.8.0. benchmark.yml carried the same bug and only looked green: its push step exited 0 on "no change to publish". Both workflows now verify instead of publish. Each regenerates its page, diffs it against what is committed, and fails if the committed page is stale. `permissions:` drops from `contents: write` to `contents: read`, so neither can write to the repository at all, and both also run on `pull_request` so a stale page blocks the merge instead of being found a week later by the schedule. Refreshing is an ordinary pull request — a governance product whose own numbers arrive by a bot bypassing branch protection argues against its own thesis. Four further defects the failure was hiding: - The gates were masked. Both workflows ordered their gate (`signetry-eval run` for the governed defenses, corpus parity for detection) after the push, and a rejected push aborts the job before it. A broken defense or a detection regression would have read as a git error. 55a7af2 added the `!cancelled()` guard; tests/test_published_numbers.py now keeps it there. - CI and the page documented different commands. The workflow inlined its own copy of the generation commands while the page told readers to run `signetry-eval leaderboard --markdown`, which omits --with-detection and --version and so produces a different page. Under a regenerate-and-diff check that drift is a stale-page failure nobody can explain from the log. The new Makefile is the single recipe, for CI, for contributors, and in the instructions printed on the page itself. - pyproject.toml was not a trigger path, though the pinned signetry-core is the live row's version label — precisely the staleness that went unnoticed. - `git diff` says nothing about an untracked file, so a page deleted from the repo and regenerated would have passed the check while not being published at all. check-* asserts git tracks the page first. CONTRIBUTING.md documents the `make published-numbers` refresh path, and `make verify-published-numbers` reproduces the whole check locally. Co-Authored-By: Claude Opus 5 --- .github/workflows/benchmark.yml | 103 +++++++++++++---------- .github/workflows/leaderboard.yml | 131 +++++++++++++++--------------- CHANGELOG.md | 40 +++++++++ CONTRIBUTING.md | 41 ++++++++++ Makefile | 114 ++++++++++++++++++++++++++ docs/BENCHMARK.md | 2 +- docs/LEADERBOARD.md | 8 +- signetry_eval/leaderboard.py | 9 +- tests/test_leaderboard.py | 6 +- tests/test_published_numbers.py | 126 ++++++++++++++++++++++++++++ 10 files changed, 467 insertions(+), 113 deletions(-) create mode 100644 Makefile create mode 100644 tests/test_published_numbers.py diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 59fd49a..d696bcd 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,16 +1,25 @@ name: benchmark -# Regenerates docs/BENCHMARK.md — the detection head-to-head against the other -# scanners — and publishes it as a release artifact, so the published numbers always -# match the shipped engine. +# Checks that docs/BENCHMARK.{md,json} — the detection head-to-head against the other +# scanners — is current. # -# This workflow owns docs/BENCHMARK.{md,json} and nothing else; the governance -# leaderboard is owned by leaderboard.yml. Both push to main, so both sit in the -# `published-numbers` concurrency group. +# This workflow does not publish. It regenerates the table with `make benchmark` (the same +# recipe a contributor runs) and fails if the committed table differs. Refreshing is a +# pull request. +# +# It used to push the refreshed table straight to `main`, which branch protection rejects: +# every run that actually had a change to publish failed at the push, and the failure came +# before the parity gate below, so a detection regression would have read as a git error. +# Runs only looked green because the table happened to be unchanged and the push step took +# its "no change to publish" exit. Verifying instead of pushing removes both failure modes. # # It runs on a schedule as well as on releases: the detection corpus and the competing # scanners both change between tags, and a table only refreshed at release time can be # months out of date while still reading as current. +# +# This workflow owns docs/BENCHMARK.{md,json} and nothing else; the governance leaderboard +# is owned by leaderboard.yml. Nothing is pushed by either, so they no longer share a +# concurrency group. on: schedule: # Mondays 06:47 UTC. Off the hour on purpose (scheduled jobs queued at :00 are @@ -23,14 +32,30 @@ on: paths: - "signetry_eval/corpus/**" - "signetry_eval/detection.py" + # The table is produced by the pinned signetry-core's scanner, so the pin is an input + # to it: bumping it without refreshing is exactly the staleness caught here. + - "pyproject.toml" + - "Makefile" + # Verify on the pull request too, so a change to any input above cannot merge while + # leaving the committed table stale. Only safe to run on PRs because the token below is + # read-only and this job needs no secrets. + pull_request: + paths: + - "signetry_eval/corpus/**" + - "signetry_eval/detection.py" + - "pyproject.toml" + - "Makefile" workflow_dispatch: +# Read-only: nothing here writes to the repository any more. permissions: - contents: write + contents: read concurrency: - group: published-numbers - cancel-in-progress: false + # Only stops a superseded commit's verification from running alongside the commit that + # replaced it — there is no longer a push to serialise against leaderboard.yml. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: benchmark: @@ -38,32 +63,29 @@ jobs: steps: - uses: actions/checkout@v7 with: - # Never the incoming ref: on a release event that is the tag, which - # leaves us in detached HEAD and turns the push below into a tag write. - ref: ${{ github.event.repository.default_branch || 'main' }} + # On a pull request, check out the pull request (checkout's default for the + # event) — verifying the default branch would say nothing about the change under + # review. Otherwise pin the default branch: on a release event the incoming ref + # is the tag, which is not where the table is committed. + ref: ${{ github.event_name == 'pull_request' && '' || (github.event.repository.default_branch || 'main') }} - uses: actions/setup-python@v7 with: python-version: "3.12" - name: Install run: pip install -e ".[dev]" - - name: Require the signetry-core detection engine - run: | - # This benchmark has no meaning without the SAST engine, and an absent scanner - # must never be published as a zero. Fail rather than emit a table. - python -c "from signetry_core import scan_repository" || { - echo "::error::Installed signetry-core lacks scan_repository; pin a signetry-core with the detection engine." - exit 1 - } + - name: Regenerate the head-to-head table id: regen run: | - mkdir -p docs - { - echo "" - echo "" - signetry-eval corpus --markdown - } > docs/BENCHMARK.md - signetry-eval corpus --json > docs/BENCHMARK.json + # `make benchmark` is the single recipe, shared with contributors: the workflow + # no longer inlines the commands, so the generated header and the command that + # actually writes the file cannot drift apart. + # + # The recipe refuses to run without the signetry-core detection engine. That + # matters more here than anywhere else: this benchmark has no meaning without the + # scanner, and an absent scanner must never be published as a zero. + make benchmark + - uses: actions/upload-artifact@v7 with: name: detection-benchmark @@ -71,22 +93,21 @@ jobs: docs/BENCHMARK.md docs/BENCHMARK.json if-no-files-found: error - - name: Commit the refreshed benchmark + + - name: Are the committed numbers current? run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add docs/BENCHMARK.md docs/BENCHMARK.json - git diff --cached --quiet && { echo "no change to publish"; exit 0; } - git commit -m "docs: refresh the detection head-to-head benchmark" - # leaderboard.yml pushes to the same branch. The concurrency group serializes - # us; rebase anyway rather than force or swallow the failure. - git pull --rebase --autostash origin "${{ github.event.repository.default_branch || 'main' }}" - git push origin "HEAD:${{ github.event.repository.default_branch || 'main' }}" + # Diff-only: the regeneration above already wrote into the working tree. + make check-benchmark || { + echo "::error::docs/BENCHMARK.{md,json} are stale. Run 'make benchmark', commit the result, and open a pull request." + exit 1 + } + - name: Gate on parity (Signetry must keep 100% recall, 0 false positives) - # Must run even when publishing failed — see the same guard in leaderboard.yml. - # Without it a rejected push aborts the job before the regression is judged. + # Must run even when the staleness check failed — see the same guard in + # leaderboard.yml. Without it a stale table aborts the job before the regression is + # judged, and a real detection regression would surface as a docs error. if: ${{ !cancelled() && steps.regen.outcome == 'success' }} run: | - # Last, deliberately: the regression is published above whatever this says, - # then this step fails the run so it cannot be missed. + # Last, deliberately: staleness is reported above whatever this says, then this + # step fails the run so a regression cannot be missed. signetry-eval corpus --min-recall 1.0 --max-fp 0 diff --git a/.github/workflows/leaderboard.yml b/.github/workflows/leaderboard.yml index 68776f3..d784466 100644 --- a/.github/workflows/leaderboard.yml +++ b/.github/workflows/leaderboard.yml @@ -1,16 +1,26 @@ name: leaderboard -# Regenerates docs/LEADERBOARD.md — the two-axis governance leaderboard, including -# third-party submissions from leaderboard/entries/. +# Checks that docs/LEADERBOARD.{md,json} — the two-axis governance leaderboard, including +# third-party submissions from leaderboard/entries/ — is current. +# +# This workflow does not publish. It regenerates the page with `make leaderboard` (the +# same recipe a contributor runs and the one printed on the page itself) and fails if the +# committed page differs. Refreshing is a pull request. +# +# It used to push the refreshed page straight to `main`. `main` requires a pull request, +# so every run that actually had a change to publish was rejected by branch protection — +# the page went stale rather than current, and because the push failed before the gate +# below, a broken defense would have read as a git error. Verifying instead of pushing +# removes both failure modes and keeps the numbers arriving the way every other change +# does, which is the argument this repository exists to make. # # Why a schedule and not just releases: the governance axis takes submissions, so the # page changes when someone else's PR lands, not only when we cut a tag. A leaderboard # whose numbers are as old as the last release is a changelog. # # This workflow owns docs/LEADERBOARD.{md,json} and nothing else. The detection -# head-to-head (docs/BENCHMARK.md) is owned by benchmark.yml — one generator per file, -# so every "Generated by ..." header names the workflow that actually wrote it. Both -# share the `published-numbers` concurrency group because both push to main. +# head-to-head (docs/BENCHMARK.md) is owned by benchmark.yml — one generator per file, so +# every "Generated by ..." header names the recipe that actually wrote it. on: schedule: # Mondays 07:23 UTC. Off the hour on purpose: scheduled jobs queued at :00 are @@ -25,14 +35,33 @@ on: - "signetry_eval/leaderboard.py" - "signetry_eval/report.py" - "signetry_eval/scenarios/**" + # The live row is labelled with the pinned signetry-core, so the pin is an input to + # the page: bumping it without refreshing is exactly the staleness caught here. + - "pyproject.toml" + - "Makefile" + # Verify on the pull request too, so a change to any input above cannot merge while + # leaving the committed page stale. Only safe to run on PRs because the token below is + # read-only and this job needs no secrets. + pull_request: + paths: + - "leaderboard/entries/**" + - "signetry_eval/leaderboard.py" + - "signetry_eval/report.py" + - "signetry_eval/scenarios/**" + - "pyproject.toml" + - "Makefile" workflow_dispatch: +# Read-only: nothing here writes to the repository any more. permissions: - contents: write + contents: read concurrency: - group: published-numbers - cancel-in-progress: false + # Nothing is pushed any more, so there is no cross-workflow write to serialise against + # benchmark.yml; this only stops a superseded commit's verification from running + # alongside the commit that replaced it. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: leaderboard: @@ -40,56 +69,30 @@ jobs: steps: - uses: actions/checkout@v7 with: - # Never the incoming ref: on a release event that is the tag, which - # leaves us in detached HEAD and turns the push below into a tag write. - ref: ${{ github.event.repository.default_branch || 'main' }} + # On a pull request, check out the pull request (checkout's default for the + # event) — verifying the default branch would say nothing about the change under + # review. Otherwise pin the default branch: on a release event the incoming ref + # is the tag, which is not where the page is committed. + ref: ${{ github.event_name == 'pull_request' && '' || (github.event.repository.default_branch || 'main') }} - uses: actions/setup-python@v7 with: python-version: "3.12" - name: Install run: pip install -e ".[dev]" - - name: Does this build have the detection engine? - id: engine - run: | - # Axis 2 needs signetry-core's SAST engine. If it is absent we publish axis 1 - # alone and say so on the page. We do not print an empty table, and we do not - # score a scanner that did not run as zero. - if python -c "from signetry_core import scan_repository" 2>/dev/null; then - echo "available=true" >> "$GITHUB_OUTPUT" - else - echo "available=false" >> "$GITHUB_OUTPUT" - echo "::warning::signetry-core lacks scan_repository; publishing the governance axis only." - fi - - name: Regenerate the leaderboard id: regen run: | - mkdir -p docs - # Record WHICH signetry-core produced the live row. A leaderboard that says - # "measured live in CI" without naming the version measured cannot be - # reproduced by a reader, which is the one thing it exists to support. - # Read from the installed package, never hardcoded — a stale label would be - # worse than the null it replaces. Left unset if it cannot be read, so the - # row degrades to "not measured" rather than to a guess. - CORE_VERSION="$(python -c 'import signetry_core; print(signetry_core.__version__)' 2>/dev/null || true)" - if [ -n "$CORE_VERSION" ]; then - VERSION_ARG="--version $CORE_VERSION" - echo "live row labelled signetry-core $CORE_VERSION" - else - VERSION_ARG="" - echo "::warning::could not read signetry_core.__version__; the live row will be unlabelled." - fi - - # Runs the adversarial suite for the live signetry-core row. A failing defense - # still publishes — the row shows the failure. The gate below is what fails. - if [ "${{ steps.engine.outputs.available }}" = "true" ]; then - signetry-eval leaderboard --with-detection $VERSION_ARG > docs/LEADERBOARD.md - signetry-eval leaderboard --with-detection $VERSION_ARG --json > docs/LEADERBOARD.json - else - signetry-eval leaderboard $VERSION_ARG > docs/LEADERBOARD.md - signetry-eval leaderboard $VERSION_ARG --json > docs/LEADERBOARD.json - fi + # `make leaderboard` is the single recipe, shared with contributors. CI does not + # inline the commands: when it did, the workflow and the regeneration + # instructions printed on the page drifted, and the page ended up documenting a + # command that produced a different page. + # + # The recipe reads the version label from the installed signetry-core and fails + # outright if the SAST engine is absent, rather than regenerating an axis-1-only + # page that would diff against the committed two-axis page and read as merely + # stale when the real problem is the install. + make leaderboard - uses: actions/upload-artifact@v7 with: @@ -99,29 +102,25 @@ jobs: docs/LEADERBOARD.json if-no-files-found: error - - name: Commit the refreshed page + - name: Are the committed numbers current? run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add docs/LEADERBOARD.md docs/LEADERBOARD.json - git diff --cached --quiet && { echo "no change to publish"; exit 0; } - git commit -m "docs: refresh the governance leaderboard" - # benchmark.yml pushes to the same branch on release. The concurrency group - # serializes us, but rebase anyway rather than force or fail. - git pull --rebase --autostash origin "${{ github.event.repository.default_branch || 'main' }}" - git push origin "HEAD:${{ github.event.repository.default_branch || 'main' }}" + # Diff-only: the regeneration above already wrote into the working tree. + make check-leaderboard || { + echo "::error::docs/LEADERBOARD.{md,json} are stale. Run 'make leaderboard', commit the result, and open a pull request." + exit 1 + } - name: Gate — every governed defense must hold - # Must run even when publishing failed. Ordering this step last is deliberate, - # but `if:` is what makes the ordering safe: a rejected push aborts the job - # before the gate, so a broken defense would surface as a git error and the - # defense itself would never be judged. Skipped only when regeneration did not - # produce numbers to judge in the first place. + # Must run even when the staleness check failed. Ordering this step last is + # deliberate, but `if:` is what makes the ordering safe: a failing step above + # aborts the job before the gate, so a broken defense would surface as a stale-page + # error and the defense itself would never be judged. Skipped only when + # regeneration did not produce numbers to judge in the first place. if: ${{ !cancelled() && steps.regen.outcome == 'success' }} run: | - # Last, deliberately: the page is published above whatever this says, then this + # Last, deliberately: staleness is reported above whatever this says, then this # step fails the run so a broken defense is impossible to miss. signetry-eval run > /dev/null || { - echo "::error::An adversarial scenario's defense did not hold. The leaderboard was published with the failing row." + echo "::error::An adversarial scenario's defense did not hold." exit 1 } diff --git a/CHANGELOG.md b/CHANGELOG.md index 8dabc8d..d5605dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,46 @@ Follows [Keep a Changelog](https://keepachangelog.com/) / [SemVer](https://semver.org/). Until `1.0.0` the public API may change between minor versions. +## [Unreleased] + +### Fixed — the pages that publish the numbers could not publish + +`leaderboard.yml` and `benchmark.yml` regenerated their pages and pushed the result +straight to `main`. `main` requires a pull request, so every run that actually had +something to publish was rejected by branch protection (`GH006: Changes must be made +through a pull request`) — the weekly refresh on 2026-09-07 and the `v0.3.0` release run +on 2026-09-01 both failed at the push. The published numbers went stale rather than +current: `docs/LEADERBOARD.json` still labelled the live row `0.7.0` after +`pyproject.toml` had been pinned to `signetry-core v0.8.0`. + +Both workflows now **verify instead of publish**. Each regenerates its page, diffs it +against what is committed, and fails if the committed page is stale; `permissions:` +dropped from `contents: write` to `contents: read`, so neither can write to the repository +at all. Refreshing is an ordinary pull request — a governance product whose own numbers +arrive by a bot bypassing branch protection is arguing against its own thesis. + +- **Both gates were masked by the failure.** Each workflow ordered its gate — `signetry-eval + run` for the governed defenses, corpus parity for detection — *after* the push. A rejected + push aborts the job before the gate, so a broken defense or a detection regression would + have surfaced as a git error, or not at all. Both gates now carry + `if: ${{ !cancelled() && steps.regen.outcome == 'success' }}` and are judged whatever the + page check says. +- **`benchmark.yml` had the same bug and looked green.** Its push step exited 0 on `no + change to publish`, so the protected-branch failure stayed invisible there until the + table actually changed. +- **One recipe, in a `Makefile`.** CI inlined its own copy of the generation commands while + the page told readers to regenerate with `signetry-eval leaderboard --markdown` — which + omits `--with-detection` and `--version` and so produces a different page. Under a + regenerate-and-diff check, that drift is a stale-page failure nobody could explain from + the log. `make leaderboard` and `make benchmark` are now the single recipe: for CI, for + contributors, and in the instructions printed on the page itself. The generated headers + name that recipe. +- **`pyproject.toml` is now a trigger path for both.** The pinned `signetry-core` is the + live row's version label, so bumping the pin without refreshing was precisely the + staleness that went unnoticed. Both workflows also verify on `pull_request`, so a stale + page blocks the merge instead of being found a week later by the schedule. +- `make verify-published-numbers` reproduces the whole check locally. + ## [0.3.0] — 2026-09-01 ### Added — the Agent Governance Leaderboard diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d976523..809917f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -83,6 +83,42 @@ signetry-eval corpus --min-recall 1.0 --max-fp 0 CI runs that same gate whenever the installed `signetry-core` exposes the detection engine (`scan_repository`), and the release workflow enforces it unconditionally. +## The published pages are generated + +`docs/LEADERBOARD.{md,json}` and `docs/BENCHMARK.{md,json}` are build output that is +committed, so anyone can read the numbers without installing anything. Do not hand-edit +them — each one's first line names the recipe that writes it. + +There is exactly one recipe, and both you and CI run it: + +```bash +make leaderboard # docs/LEADERBOARD.{md,json} +make benchmark # docs/BENCHMARK.{md,json} +make published-numbers # both +make help # every target +``` + +If your change touches an input to either page — a scenario, a corpus case, the renderers, +or the pinned `signetry-core` in `pyproject.toml` — refresh the pages in the same pull +request: + +```bash +make published-numbers +git add docs/ && git commit -m "docs: refresh the published numbers" +``` + +`.github/workflows/leaderboard.yml` and `.github/workflows/benchmark.yml` regenerate the +pages themselves and **fail if what you committed differs**, so a stale page blocks the +merge instead of quietly misreporting. To see what CI will see before you push: + +```bash +make verify-published-numbers +``` + +CI has read-only access and never pushes: the numbers reach `main` through a pull request +like every other change. A governance product whose own numbers arrived by a bot bypassing +branch protection would be arguing against its own thesis. + ## Running the suite ```bash @@ -114,6 +150,8 @@ scenario, so it doubles as a CI regression guard. 4. Keep it **deterministic and offline**: a scripted adversary that models a non-compliant agent. No network, no API keys. 5. Add a test under `tests/`. +6. Run `make published-numbers` and commit the refreshed pages — a new scenario changes + the leaderboard's denominators, and CI fails the PR if the committed page is stale. No governance logic belongs in this repo — it is imported from `signetry-core`. This repo poses attacks and scores outcomes. @@ -134,6 +172,9 @@ repo poses attacks and scores outcomes. SAFE decoy**: any finding on it counts as a false positive. 4. Case ids must be unique across families (the package asserts this on import). 5. Run `signetry-eval corpus --min-recall 1.0 --max-fp 0` and add a test. +6. Run `make published-numbers` and commit the refreshed pages — a new case changes both + the head-to-head table and the leaderboard's detection axis, and CI fails the PR if + either committed page is stale. A SAFE decoy that exposes a real false positive is a *good* result — it is how `LANG-60` caught a bug in the engine's Go SSRF rule (see `CHANGELOG.md`). diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..26dce69 --- /dev/null +++ b/Makefile @@ -0,0 +1,114 @@ +# The one recipe for the published pages. +# +# CI does not reimplement these commands. It runs these targets and then checks that the +# working tree is unchanged, so the page you regenerate locally is byte-for-byte the page +# CI expects. If CI kept its own copy of the commands, any drift between the two would +# surface as a stale-page failure that nobody could explain from the log — which is +# exactly what happened when the workflow's inline commands and the regeneration +# instructions printed on the page itself had quietly diverged. +# +# Publishing is a pull request, not a CI push. `main` is protected, and a governance +# product whose own numbers arrive by a bot bypassing branch protection is arguing +# against its own thesis. So CI verifies, humans admit, and the numbers reach `main` the +# same way every other change does. + +PY ?= python +SIGNETRY_EVAL ?= signetry-eval + +# Label the live row with the signetry-core that is actually installed — never a +# hardcoded version. pyproject pins it, so this is reproducible; bumping that pin without +# refreshing the pages is precisely the staleness `verify-leaderboard` exists to catch. +CORE_VERSION := $(shell $(PY) -c 'import signetry_core; print(signetry_core.__version__)' 2>/dev/null) + +.PHONY: help leaderboard benchmark published-numbers \ + check-leaderboard check-benchmark \ + verify-leaderboard verify-benchmark verify-published-numbers test + +# The verify targets are regenerate-then-diff. Keep them ordered. +.NOTPARALLEL: + +help: + @echo "make leaderboard regenerate docs/LEADERBOARD.{md,json}" + @echo "make benchmark regenerate docs/BENCHMARK.{md,json}" + @echo "make published-numbers regenerate both" + @echo "make verify-leaderboard regenerate, then fail if the committed page is stale" + @echo "make verify-benchmark regenerate, then fail if the committed table is stale" + @echo "make verify-published-numbers fail if either is stale" + @echo "make test run the test suite" + +# Axis 2 needs signetry-core's SAST engine. The committed page includes axis 2, so +# regenerating without the engine would produce an axis-1-only page that diffs against +# the committed one and reads as "stale" when the real problem is the install. Fail with +# the actual reason instead. +leaderboard: + @test -n "$(CORE_VERSION)" || { \ + echo "cannot read signetry_core.__version__ — is signetry-core installed? (pip install -e '.[dev]')" >&2; \ + exit 1; \ + } + @$(PY) -c "from signetry_core import scan_repository" 2>/dev/null || { \ + echo "installed signetry-core $(CORE_VERSION) lacks scan_repository; the detection axis cannot be regenerated" >&2; \ + exit 1; \ + } + @mkdir -p docs + $(SIGNETRY_EVAL) leaderboard --with-detection --version "$(CORE_VERSION)" > docs/LEADERBOARD.md + $(SIGNETRY_EVAL) leaderboard --with-detection --version "$(CORE_VERSION)" --json > docs/LEADERBOARD.json + +benchmark: + @$(PY) -c "from signetry_core import scan_repository" 2>/dev/null || { \ + echo "installed signetry-core lacks scan_repository; refusing to emit a table for a scanner that did not run" >&2; \ + exit 1; \ + } + @mkdir -p docs + { \ + echo ''; \ + echo ""; \ + $(SIGNETRY_EVAL) corpus --markdown; \ + } > docs/BENCHMARK.md + $(SIGNETRY_EVAL) corpus --json > docs/BENCHMARK.json + +published-numbers: leaderboard benchmark + +# One target per owned file set, mirroring "one generator per file": the leaderboard +# workflow owns docs/LEADERBOARD.{md,json} and must not fail on a stale benchmark it does +# not generate. +# +# check-* is diff-only and assumes the regeneration already ran, so CI can regenerate +# once (and upload the artifact) and then check. verify-* is the one-shot for humans. +# +# Each checks that git tracks the page before diffing it: `git diff` says nothing about a +# file git is not tracking, so a page deleted from the repo and then regenerated would +# sail through the diff as "current" while not being published at all. +check-leaderboard: + @git ls-files --error-unmatch docs/LEADERBOARD.md docs/LEADERBOARD.json >/dev/null 2>&1 || { \ + echo "docs/LEADERBOARD.{md,json}: not tracked by git." >&2; \ + echo "A generated page git does not track cannot be verified — commit it." >&2; \ + exit 1; \ + } + @git diff --exit-code -- docs/LEADERBOARD.md docs/LEADERBOARD.json || { \ + echo "" >&2; \ + echo "docs/LEADERBOARD.{md,json} are stale — the diff above is what regeneration produced." >&2; \ + echo "Run 'make leaderboard', commit the result, and open a pull request." >&2; \ + exit 1; \ + } + +check-benchmark: + @git ls-files --error-unmatch docs/BENCHMARK.md docs/BENCHMARK.json >/dev/null 2>&1 || { \ + echo "docs/BENCHMARK.{md,json}: not tracked by git." >&2; \ + echo "A generated table git does not track cannot be verified — commit it." >&2; \ + exit 1; \ + } + @git diff --exit-code -- docs/BENCHMARK.md docs/BENCHMARK.json || { \ + echo "" >&2; \ + echo "docs/BENCHMARK.{md,json} are stale — the diff above is what regeneration produced." >&2; \ + echo "Run 'make benchmark', commit the result, and open a pull request." >&2; \ + exit 1; \ + } + +verify-leaderboard: leaderboard check-leaderboard + +verify-benchmark: benchmark check-benchmark + +verify-published-numbers: verify-leaderboard verify-benchmark + +test: + $(PY) -m pytest tests/ -q diff --git a/docs/BENCHMARK.md b/docs/BENCHMARK.md index 87c29b1..b367ea7 100644 --- a/docs/BENCHMARK.md +++ b/docs/BENCHMARK.md @@ -1,4 +1,4 @@ - + # Signetry detection benchmark — 60 public test cases diff --git a/docs/LEADERBOARD.md b/docs/LEADERBOARD.md index a31af25..3a4f012 100644 --- a/docs/LEADERBOARD.md +++ b/docs/LEADERBOARD.md @@ -1,4 +1,4 @@ - + # The Agent Governance Leaderboard @@ -89,6 +89,10 @@ Both paths are ordinary pull requests. See Regenerate this page locally with: ```bash -signetry-eval leaderboard --markdown +make leaderboard ``` +That is the same recipe CI runs to check this page is current, so the page you +regenerate is the page CI expects. Refreshing it is a pull request, not a bot +push: these numbers reach `main` the way every other change does. + diff --git a/signetry_eval/leaderboard.py b/signetry_eval/leaderboard.py index cd6171a..e77c2e9 100644 --- a/signetry_eval/leaderboard.py +++ b/signetry_eval/leaderboard.py @@ -252,7 +252,8 @@ def render_leaderboard(report: Any, *, detection_markdown: str | None = None, o = report.overall() lines = [ - "", + "", "", "# The Agent Governance Leaderboard", "", @@ -360,8 +361,12 @@ def render_leaderboard(report: Any, *, detection_markdown: str | None = None, "Regenerate this page locally with:", "", "```bash", - "signetry-eval leaderboard --markdown", + "make leaderboard", "```", + "", + "That is the same recipe CI runs to check this page is current, so the page you", + "regenerate is the page CI expects. Refreshing it is a pull request, not a bot", + "push: these numbers reach `main` the way every other change does.", ] return "\n".join(lines) + "\n" diff --git a/tests/test_leaderboard.py b/tests/test_leaderboard.py index 7dc3824..1b72185 100644 --- a/tests/test_leaderboard.py +++ b/tests/test_leaderboard.py @@ -190,9 +190,13 @@ def test_the_page_states_its_own_sample_size_in_prose(): assert "small corpus" in page, "the page must admit the corpus is small" -def test_the_generated_header_names_the_workflow_that_writes_it(): +def test_the_generated_header_names_the_recipe_and_the_verifying_workflow(): + # The page is generated by `make leaderboard` and only *checked* by the workflow. + # The header has to name both: the recipe, so a reader can reproduce the page, and + # the workflow, so they know what fails when they edit it by hand. page = render_leaderboard(run_all(None)) first = page.strip().split("\n")[0] + assert "make leaderboard" in first assert ".github/workflows/leaderboard.yml" in first assert "do not edit by hand" in first diff --git a/tests/test_published_numbers.py b/tests/test_published_numbers.py new file mode 100644 index 0000000..edeae47 --- /dev/null +++ b/tests/test_published_numbers.py @@ -0,0 +1,126 @@ +"""The published pages are verified, never pushed — enforced here, not by memory. + +`docs/LEADERBOARD.{md,json}` and `docs/BENCHMARK.{md,json}` are generated files that live +in the repository. Two workflows used to regenerate them and push the result straight to +`main`; `main` requires a pull request, so every run that actually had something to publish +was rejected by branch protection, and because the push came *before* each workflow's gate, +a broken defense would have surfaced as a git error instead of a failed gate. + +The fix has three parts, and each is easy to undo by accident, so each is asserted: + +1. Neither workflow can write to the repository — the pages arrive by pull request. +2. Both use the `Makefile` recipe rather than their own inlined copy of the commands, so + the page a contributor regenerates is byte-for-byte the page CI checks against. +3. Both gates carry `if: !cancelled() && ...`, so an earlier failure cannot mask them. + +These are text assertions on purpose: PyYAML is not a dependency of this suite, and adding +one to check four lines of workflow would cost more than it buys. +""" +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOWS = ROOT / ".github" / "workflows" + +# (workflow, make target, the page it owns) +OWNERSHIP = [ + ("leaderboard.yml", "leaderboard", "LEADERBOARD"), + ("benchmark.yml", "benchmark", "BENCHMARK"), +] + +pytestmark = pytest.mark.skipif( + not WORKFLOWS.is_dir(), + reason="not running from a checkout (the workflows are not installed with the package)", +) + + +def _code_lines(text: str) -> list[str]: + """Every line that is not a YAML comment. + + The workflows explain their own history in comments — including what they used to do — + so a check for `git push` has to look at what runs, not at what is described. + """ + return [ln for ln in text.split("\n") if not ln.strip().startswith("#")] + + +@pytest.mark.parametrize("workflow,target,page", OWNERSHIP) +def test_the_workflow_cannot_write_to_the_repository(workflow: str, target: str, page: str): + text = (WORKFLOWS / workflow).read_text() + block = re.search(r"^permissions:\n(?:[ \t]+\S.*\n)+", text, re.M) + assert block, f"{workflow} must declare permissions explicitly" + assert "contents: read" in block.group(0), f"{workflow} must take a read-only token" + assert "write" not in block.group(0), ( + f"{workflow} requests write access. It publishes nothing: the pages reach `main` " + "through a pull request, which is what branch protection requires and what this " + "repository argues for." + ) + + +@pytest.mark.parametrize("workflow,target,page", OWNERSHIP) +def test_the_workflow_does_not_push(workflow: str, target: str, page: str): + code = "\n".join(_code_lines((WORKFLOWS / workflow).read_text())) + for forbidden in ("git push", "git commit", "git config user"): + assert forbidden not in code, ( + f"{workflow} runs `{forbidden}`. Pushing to a protected branch is the bug this " + "design removed — regenerate, diff, and fail if the committed page is stale." + ) + + +@pytest.mark.parametrize("workflow,target,page", OWNERSHIP) +def test_the_workflow_uses_the_shared_recipe(workflow: str, target: str, page: str): + code = "\n".join(_code_lines((WORKFLOWS / workflow).read_text())) + assert f"make {target}" in code, ( + f"{workflow} must regenerate with `make {target}`, not its own copy of the " + "commands: a regenerate-and-diff check is only meaningful if CI and contributors " + "run the same recipe." + ) + assert f"make check-{target}" in code, ( + f"{workflow} must verify with `make check-{target}` — regenerating without diffing " + "checks nothing." + ) + + +@pytest.mark.parametrize("workflow,target,page", OWNERSHIP) +def test_the_gate_cannot_be_masked_by_an_earlier_failure(workflow: str, target: str, page: str): + text = (WORKFLOWS / workflow).read_text() + assert "id: regen" in text, f"{workflow} needs an id on the regeneration step for the gate to key off" + assert "!cancelled() && steps.regen.outcome == 'success'" in text, ( + f"{workflow}'s gate must run even when the staleness check failed. Without the " + "guard, a stale page aborts the job first and the defense is never judged — which " + "is exactly how a rejected push used to hide a possible regression." + ) + + +@pytest.mark.parametrize("workflow,target,page", OWNERSHIP) +def test_the_committed_page_names_the_recipe_that_writes_it(workflow: str, target: str, page: str): + md = ROOT / "docs" / f"{page}.md" + if not md.is_file(): + pytest.skip(f"docs/{page}.md is not present in this tree") + first = md.read_text().strip().split("\n")[0] + assert f"make {target}" in first, ( + f"docs/{page}.md must name the recipe that regenerates it, so a reader can " + f"reproduce the page instead of hand-editing it (got: {first!r})" + ) + assert workflow in first, f"docs/{page}.md must name the workflow that verifies it (got: {first!r})" + assert "do not edit by hand" in first + + +def test_the_benchmark_header_is_not_duplicated_out_of_sync(): + """The benchmark header is written by the Makefile; the leaderboard's by Python. + + Only the benchmark's lives in two places (the recipe that writes it and the committed + file), so only it can drift silently. Pin them together. + """ + makefile = ROOT / "Makefile" + md = ROOT / "docs" / "BENCHMARK.md" + if not (makefile.is_file() and md.is_file()): + pytest.skip("Makefile or docs/BENCHMARK.md is not present in this tree") + first = md.read_text().strip().split("\n")[0] + assert first in makefile.read_text(), ( + "docs/BENCHMARK.md's header is not the one `make benchmark` emits — regenerate it " + f"(committed header: {first!r})" + ) From 75fd873eb13e355d00ad04b5dd7855508b04863e Mon Sep 17 00:00:00 2001 From: Binay <150876063+bkd-dotcom@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:37:08 -0400 Subject: [PATCH 4/4] fix(ci): check out the PR, and refuse a tree without the recipe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs in the verify design, both found by running it on its own pull request (#39), where `leaderboard` and `benchmark` failed. The checkout ref was a falsy ternary: ref: ${{ github.event_name == 'pull_request' && '' || (... || 'main') }} Actions' `&&` and `||` yield operand values, not booleans, so a falsy true branch collapses straight through: `true && ''` is `''`, and `'' || 'main'` is `'main'`. Every event checked out the default branch, so the PR verified `main` instead of itself — the check said nothing about the change under review. Now the *release* case is the true branch (its ref is the tag, not where the page is committed) and everything else verifies `github.ref`, which on a pull request is `refs/pull/N/merge`: the merge result, which is what actually lands. Both branches are non-empty strings. The second bug is why that one was survivable rather than obvious, and it is the worse of the two. `main` carries no Makefile, so the regeneration step ran `make leaderboard` against a tree without the recipe — and make exited 0: make: Nothing to be done for 'leaderboard'. `leaderboard/` is a real directory in this repository, so with no Makefile to say otherwise make treats the target as already up to date. The step reported success, uploaded the untouched committed page as its artifact, and only the diff step failed — with "the page is stale", which was not the problem. That is the same shape as the push step that exited 0 on "no change to publish": a step that succeeds without doing its job. Both workflows now confirm the Makefile is present before trusting `make`, and say "wrong ref?" when it is not. tests/test_published_numbers.py grows three guards, each mutation-checked against the bug it describes: no ternary with an empty true branch, the ref expression must verify `github.ref` and special-case only `release`, and the regeneration step must assert the recipe exists. Co-Authored-By: Claude Opus 5 --- .github/workflows/benchmark.yml | 27 +++++++++++--- .github/workflows/leaderboard.yml | 27 +++++++++++--- CHANGELOG.md | 5 +++ tests/test_published_numbers.py | 59 +++++++++++++++++++++++++++++++ 4 files changed, 108 insertions(+), 10 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index d696bcd..64e3814 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -63,11 +63,18 @@ jobs: steps: - uses: actions/checkout@v7 with: - # On a pull request, check out the pull request (checkout's default for the - # event) — verifying the default branch would say nothing about the change under - # review. Otherwise pin the default branch: on a release event the incoming ref - # is the tag, which is not where the table is committed. - ref: ${{ github.event_name == 'pull_request' && '' || (github.event.repository.default_branch || 'main') }} + # Verify the ref that triggered us. On a pull request that is `refs/pull/N/merge` + # — the merge result, which is what actually lands — because verifying the default + # branch would say nothing about the change under review. The one exception is a + # release: its ref is the tag, which is not where the table is committed. + # + # The *release* case is the true branch on purpose. The obvious spelling, + # `event_name == 'pull_request' && '' || default_branch`, is a trap: Actions' `&&` + # and `||` yield operand values rather than booleans, so a falsy true branch ('') + # collapses to the false branch and *every* event checks out the default branch. + # This workflow shipped that bug for one run: PR #39 verified `main` instead of + # itself. Both branches here are non-empty strings. + ref: ${{ github.event_name == 'release' && (github.event.repository.default_branch || 'main') || github.ref }} - uses: actions/setup-python@v7 with: python-version: "3.12" @@ -84,6 +91,16 @@ jobs: # The recipe refuses to run without the signetry-core detection engine. That # matters more here than anywhere else: this benchmark has no meaning without the # scanner, and an absent scanner must never be published as a zero. + # The checked-out tree must be the one that carries the recipe. `make benchmark` + # without a Makefile does not necessarily fail: `benchmark` can name a real path in + # this repository, in which case make treats the target as already up to date and + # exits 0 — "Nothing to be done for 'benchmark'". A regeneration step that silently + # regenerates nothing is the same class of bug as the push step that silently + # exited 0 on "no change to publish", so check before trusting it. + test -f Makefile || { + echo "::error::No Makefile in the checked-out tree — regeneration would silently do nothing. Wrong ref?" + exit 1 + } make benchmark - uses: actions/upload-artifact@v7 diff --git a/.github/workflows/leaderboard.yml b/.github/workflows/leaderboard.yml index d784466..ccd5482 100644 --- a/.github/workflows/leaderboard.yml +++ b/.github/workflows/leaderboard.yml @@ -69,11 +69,18 @@ jobs: steps: - uses: actions/checkout@v7 with: - # On a pull request, check out the pull request (checkout's default for the - # event) — verifying the default branch would say nothing about the change under - # review. Otherwise pin the default branch: on a release event the incoming ref - # is the tag, which is not where the page is committed. - ref: ${{ github.event_name == 'pull_request' && '' || (github.event.repository.default_branch || 'main') }} + # Verify the ref that triggered us. On a pull request that is `refs/pull/N/merge` + # — the merge result, which is what actually lands — because verifying the default + # branch would say nothing about the change under review. The one exception is a + # release: its ref is the tag, which is not where the page is committed. + # + # The *release* case is the true branch on purpose. The obvious spelling, + # `event_name == 'pull_request' && '' || default_branch`, is a trap: Actions' `&&` + # and `||` yield operand values rather than booleans, so a falsy true branch ('') + # collapses to the false branch and *every* event checks out the default branch. + # This workflow shipped that bug for one run: PR #39 verified `main` instead of + # itself. Both branches here are non-empty strings. + ref: ${{ github.event_name == 'release' && (github.event.repository.default_branch || 'main') || github.ref }} - uses: actions/setup-python@v7 with: python-version: "3.12" @@ -92,6 +99,16 @@ jobs: # outright if the SAST engine is absent, rather than regenerating an axis-1-only # page that would diff against the committed two-axis page and read as merely # stale when the real problem is the install. + # The checked-out tree must be the one that carries the recipe. `make leaderboard` + # without a Makefile does not necessarily fail: `leaderboard` can name a real path in + # this repository, in which case make treats the target as already up to date and + # exits 0 — "Nothing to be done for 'leaderboard'". A regeneration step that silently + # regenerates nothing is the same class of bug as the push step that silently + # exited 0 on "no change to publish", so check before trusting it. + test -f Makefile || { + echo "::error::No Makefile in the checked-out tree — regeneration would silently do nothing. Wrong ref?" + exit 1 + } make leaderboard - uses: actions/upload-artifact@v7 diff --git a/CHANGELOG.md b/CHANGELOG.md index d5605dd..fa9870f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,11 @@ arrive by a bot bypassing branch protection is arguing against its own thesis. live row's version label, so bumping the pin without refreshing was precisely the staleness that went unnoticed. Both workflows also verify on `pull_request`, so a stale page blocks the merge instead of being found a week later by the schedule. +- **A regeneration that regenerates nothing now fails.** `make leaderboard` in a tree with + no Makefile does not error: `leaderboard/` is a real directory here, so make calls the + target up to date and exits 0 with "Nothing to be done". That is the same shape as the + push step that exited 0 on "no change to publish" — a step that reports success without + doing its job — so both workflows confirm the recipe is present before trusting it. - `make verify-published-numbers` reproduces the whole check locally. ## [0.3.0] — 2026-09-01 diff --git a/tests/test_published_numbers.py b/tests/test_published_numbers.py index edeae47..3b67fca 100644 --- a/tests/test_published_numbers.py +++ b/tests/test_published_numbers.py @@ -84,6 +84,65 @@ def test_the_workflow_uses_the_shared_recipe(workflow: str, target: str, page: s ) +@pytest.mark.parametrize("workflow,target,page", OWNERSHIP) +def test_the_checkout_ref_is_not_a_falsy_ternary(workflow: str, target: str, page: str): + """`A && '' || B` is always B, so the workflow would verify the wrong tree. + + Actions' `&&` and `||` yield operand values rather than booleans, which makes the + familiar ternary spelling a trap whenever the true branch is falsy: an empty string + collapses straight through to the false branch. Both of these workflows shipped exactly + that for one run, and PR #39 verified `main` instead of itself — the check passed and + proved nothing about the change under review. + """ + # Comments only, and the comment above the ref quotes the trap to explain it. + code = "\n".join(_code_lines((WORKFLOWS / workflow).read_text())) + for trap in ("&& '' ||", '&& "" ||', "&& ''||"): + assert trap not in code, ( + f"{workflow} uses a ternary with an empty true branch. Actions evaluates " + f"`cond && '' || other` to `other` for every event, so the ref this resolves to " + "is not the one it looks like. Give the true branch a non-empty value." + ) + + +@pytest.mark.parametrize("workflow,target,page", OWNERSHIP) +def test_the_pull_request_run_verifies_the_pull_request(workflow: str, target: str, page: str): + """A verification that checks out the default branch on a PR verifies nothing. + + `github.ref` is `refs/pull/N/merge` on a pull request — the merge result, which is what + actually lands. Only a release needs the override, because its ref is the tag rather + than the branch the page is committed on. + """ + text = (WORKFLOWS / workflow).read_text() + ref = re.search(r"^\s+ref: (.+)$", text, re.M) + assert ref, f"{workflow} must pin the checkout ref explicitly" + expr = ref.group(1) + assert "github.ref" in expr, ( + f"{workflow} must verify the triggering ref (so a PR verifies itself); got {expr!r}" + ) + assert "release" in expr, ( + f"{workflow} must special-case the release event, whose ref is the tag and not where " + f"the page is committed; got {expr!r}" + ) + + +@pytest.mark.parametrize("workflow,target,page", OWNERSHIP) +def test_the_workflow_refuses_a_tree_that_lacks_the_recipe(workflow: str, target: str, page: str): + """`make ` can exit 0 having done nothing at all. + + With no Makefile in the tree, make falls back to implicit rules — and `leaderboard` is a + real directory in this repository, so make calls the target up to date and prints + "Nothing to be done for 'leaderboard'" with status 0. That is how a run regenerated + nothing, uploaded the committed page as its artifact, and still reported the + regeneration step green. Same shape as the push step that exited 0 on "no change to + publish": a step that succeeds without doing its job. + """ + code = "\n".join(_code_lines((WORKFLOWS / workflow).read_text())) + assert "test -f Makefile" in code, ( + f"{workflow} must confirm the recipe is present before running it — otherwise a " + "wrong-tree checkout regenerates nothing and still passes." + ) + + @pytest.mark.parametrize("workflow,target,page", OWNERSHIP) def test_the_gate_cannot_be_masked_by_an_earlier_failure(workflow: str, target: str, page: str): text = (WORKFLOWS / workflow).read_text()