diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 068bf5f..64e3814 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,16 +1,25 @@ name: benchmark -# Regenerates docs/BENCHMARK.md — the detection head-to-head against the other -# scanners — and publishes it as a release artifact, so the published numbers always -# match the shipped engine. +# Checks that docs/BENCHMARK.{md,json} — the detection head-to-head against the other +# scanners — is current. # -# This workflow owns docs/BENCHMARK.{md,json} and nothing else; the governance -# leaderboard is owned by leaderboard.yml. Both push to main, so both sit in the -# `published-numbers` concurrency group. +# This workflow does not publish. It regenerates the table with `make benchmark` (the same +# recipe a contributor runs) and fails if the committed table differs. Refreshing is a +# pull request. +# +# It used to push the refreshed table straight to `main`, which branch protection rejects: +# every run that actually had a change to publish failed at the push, and the failure came +# before the parity gate below, so a detection regression would have read as a git error. +# Runs only looked green because the table happened to be unchanged and the push step took +# its "no change to publish" exit. Verifying instead of pushing removes both failure modes. # # It runs on a schedule as well as on releases: the detection corpus and the competing # scanners both change between tags, and a table only refreshed at release time can be # months out of date while still reading as current. +# +# This workflow owns docs/BENCHMARK.{md,json} and nothing else; the governance leaderboard +# is owned by leaderboard.yml. Nothing is pushed by either, so they no longer share a +# concurrency group. on: schedule: # Mondays 06:47 UTC. Off the hour on purpose (scheduled jobs queued at :00 are @@ -23,14 +32,30 @@ on: paths: - "signetry_eval/corpus/**" - "signetry_eval/detection.py" + # The table is produced by the pinned signetry-core's scanner, so the pin is an input + # to it: bumping it without refreshing is exactly the staleness caught here. + - "pyproject.toml" + - "Makefile" + # Verify on the pull request too, so a change to any input above cannot merge while + # leaving the committed table stale. Only safe to run on PRs because the token below is + # read-only and this job needs no secrets. + pull_request: + paths: + - "signetry_eval/corpus/**" + - "signetry_eval/detection.py" + - "pyproject.toml" + - "Makefile" workflow_dispatch: +# Read-only: nothing here writes to the repository any more. permissions: - contents: write + contents: read concurrency: - group: published-numbers - cancel-in-progress: false + # Only stops a superseded commit's verification from running alongside the commit that + # replaced it — there is no longer a push to serialise against leaderboard.yml. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: benchmark: @@ -38,31 +63,46 @@ jobs: steps: - uses: actions/checkout@v7 with: - # Never the incoming ref: on a release event that is the tag, which - # leaves us in detached HEAD and turns the push below into a tag write. - ref: ${{ github.event.repository.default_branch || 'main' }} + # Verify the ref that triggered us. On a pull request that is `refs/pull/N/merge` + # — the merge result, which is what actually lands — because verifying the default + # branch would say nothing about the change under review. The one exception is a + # release: its ref is the tag, which is not where the table is committed. + # + # The *release* case is the true branch on purpose. The obvious spelling, + # `event_name == 'pull_request' && '' || default_branch`, is a trap: Actions' `&&` + # and `||` yield operand values rather than booleans, so a falsy true branch ('') + # collapses to the false branch and *every* event checks out the default branch. + # This workflow shipped that bug for one run: PR #39 verified `main` instead of + # itself. Both branches here are non-empty strings. + ref: ${{ github.event_name == 'release' && (github.event.repository.default_branch || 'main') || github.ref }} - uses: actions/setup-python@v7 with: python-version: "3.12" - name: Install run: pip install -e ".[dev]" - - name: Require the signetry-core detection engine + + - name: Regenerate the head-to-head table + id: regen run: | - # This benchmark has no meaning without the SAST engine, and an absent scanner - # must never be published as a zero. Fail rather than emit a table. - python -c "from signetry_core import scan_repository" || { - echo "::error::Installed signetry-core lacks scan_repository; pin a signetry-core with the detection engine." + # `make benchmark` is the single recipe, shared with contributors: the workflow + # no longer inlines the commands, so the generated header and the command that + # actually writes the file cannot drift apart. + # + # The recipe refuses to run without the signetry-core detection engine. That + # matters more here than anywhere else: this benchmark has no meaning without the + # scanner, and an absent scanner must never be published as a zero. + # The checked-out tree must be the one that carries the recipe. `make benchmark` + # without a Makefile does not necessarily fail: `benchmark` can name a real path in + # this repository, in which case make treats the target as already up to date and + # exits 0 — "Nothing to be done for 'benchmark'". A regeneration step that silently + # regenerates nothing is the same class of bug as the push step that silently + # exited 0 on "no change to publish", so check before trusting it. + test -f Makefile || { + echo "::error::No Makefile in the checked-out tree — regeneration would silently do nothing. Wrong ref?" exit 1 } - - name: Regenerate the head-to-head table - run: | - mkdir -p docs - { - echo "" - echo "" - signetry-eval corpus --markdown - } > docs/BENCHMARK.md - signetry-eval corpus --json > docs/BENCHMARK.json + make benchmark + - uses: actions/upload-artifact@v7 with: name: detection-benchmark @@ -70,19 +110,21 @@ jobs: docs/BENCHMARK.md docs/BENCHMARK.json if-no-files-found: error - - name: Commit the refreshed benchmark + + - name: Are the committed numbers current? run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add docs/BENCHMARK.md docs/BENCHMARK.json - git diff --cached --quiet && { echo "no change to publish"; exit 0; } - git commit -m "docs: refresh the detection head-to-head benchmark" - # leaderboard.yml pushes to the same branch. The concurrency group serializes - # us; rebase anyway rather than force or swallow the failure. - git pull --rebase --autostash origin "${{ github.event.repository.default_branch || 'main' }}" - git push origin "HEAD:${{ github.event.repository.default_branch || 'main' }}" + # Diff-only: the regeneration above already wrote into the working tree. + make check-benchmark || { + echo "::error::docs/BENCHMARK.{md,json} are stale. Run 'make benchmark', commit the result, and open a pull request." + exit 1 + } + - name: Gate on parity (Signetry must keep 100% recall, 0 false positives) + # Must run even when the staleness check failed — see the same guard in + # leaderboard.yml. Without it a stale table aborts the job before the regression is + # judged, and a real detection regression would surface as a docs error. + if: ${{ !cancelled() && steps.regen.outcome == 'success' }} run: | - # Last, deliberately: the regression is published above whatever this says, - # then this step fails the run so it cannot be missed. + # Last, deliberately: staleness is reported above whatever this says, then this + # step fails the run so a regression cannot be missed. signetry-eval corpus --min-recall 1.0 --max-fp 0 diff --git a/.github/workflows/leaderboard.yml b/.github/workflows/leaderboard.yml index f61d4fd..ccd5482 100644 --- a/.github/workflows/leaderboard.yml +++ b/.github/workflows/leaderboard.yml @@ -1,16 +1,26 @@ name: leaderboard -# Regenerates docs/LEADERBOARD.md — the two-axis governance leaderboard, including -# third-party submissions from leaderboard/entries/. +# Checks that docs/LEADERBOARD.{md,json} — the two-axis governance leaderboard, including +# third-party submissions from leaderboard/entries/ — is current. +# +# This workflow does not publish. It regenerates the page with `make leaderboard` (the +# same recipe a contributor runs and the one printed on the page itself) and fails if the +# committed page differs. Refreshing is a pull request. +# +# It used to push the refreshed page straight to `main`. `main` requires a pull request, +# so every run that actually had a change to publish was rejected by branch protection — +# the page went stale rather than current, and because the push failed before the gate +# below, a broken defense would have read as a git error. Verifying instead of pushing +# removes both failure modes and keeps the numbers arriving the way every other change +# does, which is the argument this repository exists to make. # # Why a schedule and not just releases: the governance axis takes submissions, so the # page changes when someone else's PR lands, not only when we cut a tag. A leaderboard # whose numbers are as old as the last release is a changelog. # # This workflow owns docs/LEADERBOARD.{md,json} and nothing else. The detection -# head-to-head (docs/BENCHMARK.md) is owned by benchmark.yml — one generator per file, -# so every "Generated by ..." header names the workflow that actually wrote it. Both -# share the `published-numbers` concurrency group because both push to main. +# head-to-head (docs/BENCHMARK.md) is owned by benchmark.yml — one generator per file, so +# every "Generated by ..." header names the recipe that actually wrote it. on: schedule: # Mondays 07:23 UTC. Off the hour on purpose: scheduled jobs queued at :00 are @@ -25,14 +35,33 @@ on: - "signetry_eval/leaderboard.py" - "signetry_eval/report.py" - "signetry_eval/scenarios/**" + # The live row is labelled with the pinned signetry-core, so the pin is an input to + # the page: bumping it without refreshing is exactly the staleness caught here. + - "pyproject.toml" + - "Makefile" + # Verify on the pull request too, so a change to any input above cannot merge while + # leaving the committed page stale. Only safe to run on PRs because the token below is + # read-only and this job needs no secrets. + pull_request: + paths: + - "leaderboard/entries/**" + - "signetry_eval/leaderboard.py" + - "signetry_eval/report.py" + - "signetry_eval/scenarios/**" + - "pyproject.toml" + - "Makefile" workflow_dispatch: +# Read-only: nothing here writes to the repository any more. permissions: - contents: write + contents: read concurrency: - group: published-numbers - cancel-in-progress: false + # Nothing is pushed any more, so there is no cross-workflow write to serialise against + # benchmark.yml; this only stops a superseded commit's verification from running + # alongside the commit that replaced it. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: leaderboard: @@ -40,55 +69,47 @@ jobs: steps: - uses: actions/checkout@v7 with: - # Never the incoming ref: on a release event that is the tag, which - # leaves us in detached HEAD and turns the push below into a tag write. - ref: ${{ github.event.repository.default_branch || 'main' }} + # Verify the ref that triggered us. On a pull request that is `refs/pull/N/merge` + # — the merge result, which is what actually lands — because verifying the default + # branch would say nothing about the change under review. The one exception is a + # release: its ref is the tag, which is not where the page is committed. + # + # The *release* case is the true branch on purpose. The obvious spelling, + # `event_name == 'pull_request' && '' || default_branch`, is a trap: Actions' `&&` + # and `||` yield operand values rather than booleans, so a falsy true branch ('') + # collapses to the false branch and *every* event checks out the default branch. + # This workflow shipped that bug for one run: PR #39 verified `main` instead of + # itself. Both branches here are non-empty strings. + ref: ${{ github.event_name == 'release' && (github.event.repository.default_branch || 'main') || github.ref }} - uses: actions/setup-python@v7 with: python-version: "3.12" - name: Install run: pip install -e ".[dev]" - - name: Does this build have the detection engine? - id: engine - run: | - # Axis 2 needs signetry-core's SAST engine. If it is absent we publish axis 1 - # alone and say so on the page. We do not print an empty table, and we do not - # score a scanner that did not run as zero. - if python -c "from signetry_core import scan_repository" 2>/dev/null; then - echo "available=true" >> "$GITHUB_OUTPUT" - else - echo "available=false" >> "$GITHUB_OUTPUT" - echo "::warning::signetry-core lacks scan_repository; publishing the governance axis only." - fi - - name: Regenerate the leaderboard + id: regen run: | - mkdir -p docs - # Record WHICH signetry-core produced the live row. A leaderboard that says - # "measured live in CI" without naming the version measured cannot be - # reproduced by a reader, which is the one thing it exists to support. - # Read from the installed package, never hardcoded — a stale label would be - # worse than the null it replaces. Left unset if it cannot be read, so the - # row degrades to "not measured" rather than to a guess. - CORE_VERSION="$(python -c 'import signetry_core; print(signetry_core.__version__)' 2>/dev/null || true)" - if [ -n "$CORE_VERSION" ]; then - VERSION_ARG="--version $CORE_VERSION" - echo "live row labelled signetry-core $CORE_VERSION" - else - VERSION_ARG="" - echo "::warning::could not read signetry_core.__version__; the live row will be unlabelled." - fi - - # Runs the adversarial suite for the live signetry-core row. A failing defense - # still publishes — the row shows the failure. The gate below is what fails. - if [ "${{ steps.engine.outputs.available }}" = "true" ]; then - signetry-eval leaderboard --with-detection $VERSION_ARG > docs/LEADERBOARD.md - signetry-eval leaderboard --with-detection $VERSION_ARG --json > docs/LEADERBOARD.json - else - signetry-eval leaderboard $VERSION_ARG > docs/LEADERBOARD.md - signetry-eval leaderboard $VERSION_ARG --json > docs/LEADERBOARD.json - fi + # `make leaderboard` is the single recipe, shared with contributors. CI does not + # inline the commands: when it did, the workflow and the regeneration + # instructions printed on the page drifted, and the page ended up documenting a + # command that produced a different page. + # + # The recipe reads the version label from the installed signetry-core and fails + # outright if the SAST engine is absent, rather than regenerating an axis-1-only + # page that would diff against the committed two-axis page and read as merely + # stale when the real problem is the install. + # The checked-out tree must be the one that carries the recipe. `make leaderboard` + # without a Makefile does not necessarily fail: `leaderboard` can name a real path in + # this repository, in which case make treats the target as already up to date and + # exits 0 — "Nothing to be done for 'leaderboard'". A regeneration step that silently + # regenerates nothing is the same class of bug as the push step that silently + # exited 0 on "no change to publish", so check before trusting it. + test -f Makefile || { + echo "::error::No Makefile in the checked-out tree — regeneration would silently do nothing. Wrong ref?" + exit 1 + } + make leaderboard - uses: actions/upload-artifact@v7 with: @@ -98,23 +119,25 @@ jobs: docs/LEADERBOARD.json if-no-files-found: error - - name: Commit the refreshed page + - name: Are the committed numbers current? run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add docs/LEADERBOARD.md docs/LEADERBOARD.json - git diff --cached --quiet && { echo "no change to publish"; exit 0; } - git commit -m "docs: refresh the governance leaderboard" - # benchmark.yml pushes to the same branch on release. The concurrency group - # serializes us, but rebase anyway rather than force or fail. - git pull --rebase --autostash origin "${{ github.event.repository.default_branch || 'main' }}" - git push origin "HEAD:${{ github.event.repository.default_branch || 'main' }}" + # Diff-only: the regeneration above already wrote into the working tree. + make check-leaderboard || { + echo "::error::docs/LEADERBOARD.{md,json} are stale. Run 'make leaderboard', commit the result, and open a pull request." + exit 1 + } - name: Gate — every governed defense must hold + # Must run even when the staleness check failed. Ordering this step last is + # deliberate, but `if:` is what makes the ordering safe: a failing step above + # aborts the job before the gate, so a broken defense would surface as a stale-page + # error and the defense itself would never be judged. Skipped only when + # regeneration did not produce numbers to judge in the first place. + if: ${{ !cancelled() && steps.regen.outcome == 'success' }} run: | - # Last, deliberately: the page is published above whatever this says, then this + # Last, deliberately: staleness is reported above whatever this says, then this # step fails the run so a broken defense is impossible to miss. signetry-eval run > /dev/null || { - echo "::error::An adversarial scenario's defense did not hold. The leaderboard was published with the failing row." + echo "::error::An adversarial scenario's defense did not hold." exit 1 } diff --git a/CHANGELOG.md b/CHANGELOG.md index 8dabc8d..fa9870f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,51 @@ Follows [Keep a Changelog](https://keepachangelog.com/) / [SemVer](https://semver.org/). Until `1.0.0` the public API may change between minor versions. +## [Unreleased] + +### Fixed — the pages that publish the numbers could not publish + +`leaderboard.yml` and `benchmark.yml` regenerated their pages and pushed the result +straight to `main`. `main` requires a pull request, so every run that actually had +something to publish was rejected by branch protection (`GH006: Changes must be made +through a pull request`) — the weekly refresh on 2026-09-07 and the `v0.3.0` release run +on 2026-09-01 both failed at the push. The published numbers went stale rather than +current: `docs/LEADERBOARD.json` still labelled the live row `0.7.0` after +`pyproject.toml` had been pinned to `signetry-core v0.8.0`. + +Both workflows now **verify instead of publish**. Each regenerates its page, diffs it +against what is committed, and fails if the committed page is stale; `permissions:` +dropped from `contents: write` to `contents: read`, so neither can write to the repository +at all. Refreshing is an ordinary pull request — a governance product whose own numbers +arrive by a bot bypassing branch protection is arguing against its own thesis. + +- **Both gates were masked by the failure.** Each workflow ordered its gate — `signetry-eval + run` for the governed defenses, corpus parity for detection — *after* the push. A rejected + push aborts the job before the gate, so a broken defense or a detection regression would + have surfaced as a git error, or not at all. Both gates now carry + `if: ${{ !cancelled() && steps.regen.outcome == 'success' }}` and are judged whatever the + page check says. +- **`benchmark.yml` had the same bug and looked green.** Its push step exited 0 on `no + change to publish`, so the protected-branch failure stayed invisible there until the + table actually changed. +- **One recipe, in a `Makefile`.** CI inlined its own copy of the generation commands while + the page told readers to regenerate with `signetry-eval leaderboard --markdown` — which + omits `--with-detection` and `--version` and so produces a different page. Under a + regenerate-and-diff check, that drift is a stale-page failure nobody could explain from + the log. `make leaderboard` and `make benchmark` are now the single recipe: for CI, for + contributors, and in the instructions printed on the page itself. The generated headers + name that recipe. +- **`pyproject.toml` is now a trigger path for both.** The pinned `signetry-core` is the + live row's version label, so bumping the pin without refreshing was precisely the + staleness that went unnoticed. Both workflows also verify on `pull_request`, so a stale + page blocks the merge instead of being found a week later by the schedule. +- **A regeneration that regenerates nothing now fails.** `make leaderboard` in a tree with + no Makefile does not error: `leaderboard/` is a real directory here, so make calls the + target up to date and exits 0 with "Nothing to be done". That is the same shape as the + push step that exited 0 on "no change to publish" — a step that reports success without + doing its job — so both workflows confirm the recipe is present before trusting it. +- `make verify-published-numbers` reproduces the whole check locally. + ## [0.3.0] — 2026-09-01 ### Added — the Agent Governance Leaderboard diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d976523..809917f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -83,6 +83,42 @@ signetry-eval corpus --min-recall 1.0 --max-fp 0 CI runs that same gate whenever the installed `signetry-core` exposes the detection engine (`scan_repository`), and the release workflow enforces it unconditionally. +## The published pages are generated + +`docs/LEADERBOARD.{md,json}` and `docs/BENCHMARK.{md,json}` are build output that is +committed, so anyone can read the numbers without installing anything. Do not hand-edit +them — each one's first line names the recipe that writes it. + +There is exactly one recipe, and both you and CI run it: + +```bash +make leaderboard # docs/LEADERBOARD.{md,json} +make benchmark # docs/BENCHMARK.{md,json} +make published-numbers # both +make help # every target +``` + +If your change touches an input to either page — a scenario, a corpus case, the renderers, +or the pinned `signetry-core` in `pyproject.toml` — refresh the pages in the same pull +request: + +```bash +make published-numbers +git add docs/ && git commit -m "docs: refresh the published numbers" +``` + +`.github/workflows/leaderboard.yml` and `.github/workflows/benchmark.yml` regenerate the +pages themselves and **fail if what you committed differs**, so a stale page blocks the +merge instead of quietly misreporting. To see what CI will see before you push: + +```bash +make verify-published-numbers +``` + +CI has read-only access and never pushes: the numbers reach `main` through a pull request +like every other change. A governance product whose own numbers arrived by a bot bypassing +branch protection would be arguing against its own thesis. + ## Running the suite ```bash @@ -114,6 +150,8 @@ scenario, so it doubles as a CI regression guard. 4. Keep it **deterministic and offline**: a scripted adversary that models a non-compliant agent. No network, no API keys. 5. Add a test under `tests/`. +6. Run `make published-numbers` and commit the refreshed pages — a new scenario changes + the leaderboard's denominators, and CI fails the PR if the committed page is stale. No governance logic belongs in this repo — it is imported from `signetry-core`. This repo poses attacks and scores outcomes. @@ -134,6 +172,9 @@ repo poses attacks and scores outcomes. SAFE decoy**: any finding on it counts as a false positive. 4. Case ids must be unique across families (the package asserts this on import). 5. Run `signetry-eval corpus --min-recall 1.0 --max-fp 0` and add a test. +6. Run `make published-numbers` and commit the refreshed pages — a new case changes both + the head-to-head table and the leaderboard's detection axis, and CI fails the PR if + either committed page is stale. A SAFE decoy that exposes a real false positive is a *good* result — it is how `LANG-60` caught a bug in the engine's Go SSRF rule (see `CHANGELOG.md`). diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..26dce69 --- /dev/null +++ b/Makefile @@ -0,0 +1,114 @@ +# The one recipe for the published pages. +# +# CI does not reimplement these commands. It runs these targets and then checks that the +# working tree is unchanged, so the page you regenerate locally is byte-for-byte the page +# CI expects. If CI kept its own copy of the commands, any drift between the two would +# surface as a stale-page failure that nobody could explain from the log — which is +# exactly what happened when the workflow's inline commands and the regeneration +# instructions printed on the page itself had quietly diverged. +# +# Publishing is a pull request, not a CI push. `main` is protected, and a governance +# product whose own numbers arrive by a bot bypassing branch protection is arguing +# against its own thesis. So CI verifies, humans admit, and the numbers reach `main` the +# same way every other change does. + +PY ?= python +SIGNETRY_EVAL ?= signetry-eval + +# Label the live row with the signetry-core that is actually installed — never a +# hardcoded version. pyproject pins it, so this is reproducible; bumping that pin without +# refreshing the pages is precisely the staleness `verify-leaderboard` exists to catch. +CORE_VERSION := $(shell $(PY) -c 'import signetry_core; print(signetry_core.__version__)' 2>/dev/null) + +.PHONY: help leaderboard benchmark published-numbers \ + check-leaderboard check-benchmark \ + verify-leaderboard verify-benchmark verify-published-numbers test + +# The verify targets are regenerate-then-diff. Keep them ordered. +.NOTPARALLEL: + +help: + @echo "make leaderboard regenerate docs/LEADERBOARD.{md,json}" + @echo "make benchmark regenerate docs/BENCHMARK.{md,json}" + @echo "make published-numbers regenerate both" + @echo "make verify-leaderboard regenerate, then fail if the committed page is stale" + @echo "make verify-benchmark regenerate, then fail if the committed table is stale" + @echo "make verify-published-numbers fail if either is stale" + @echo "make test run the test suite" + +# Axis 2 needs signetry-core's SAST engine. The committed page includes axis 2, so +# regenerating without the engine would produce an axis-1-only page that diffs against +# the committed one and reads as "stale" when the real problem is the install. Fail with +# the actual reason instead. +leaderboard: + @test -n "$(CORE_VERSION)" || { \ + echo "cannot read signetry_core.__version__ — is signetry-core installed? (pip install -e '.[dev]')" >&2; \ + exit 1; \ + } + @$(PY) -c "from signetry_core import scan_repository" 2>/dev/null || { \ + echo "installed signetry-core $(CORE_VERSION) lacks scan_repository; the detection axis cannot be regenerated" >&2; \ + exit 1; \ + } + @mkdir -p docs + $(SIGNETRY_EVAL) leaderboard --with-detection --version "$(CORE_VERSION)" > docs/LEADERBOARD.md + $(SIGNETRY_EVAL) leaderboard --with-detection --version "$(CORE_VERSION)" --json > docs/LEADERBOARD.json + +benchmark: + @$(PY) -c "from signetry_core import scan_repository" 2>/dev/null || { \ + echo "installed signetry-core lacks scan_repository; refusing to emit a table for a scanner that did not run" >&2; \ + exit 1; \ + } + @mkdir -p docs + { \ + echo ''; \ + echo ""; \ + $(SIGNETRY_EVAL) corpus --markdown; \ + } > docs/BENCHMARK.md + $(SIGNETRY_EVAL) corpus --json > docs/BENCHMARK.json + +published-numbers: leaderboard benchmark + +# One target per owned file set, mirroring "one generator per file": the leaderboard +# workflow owns docs/LEADERBOARD.{md,json} and must not fail on a stale benchmark it does +# not generate. +# +# check-* is diff-only and assumes the regeneration already ran, so CI can regenerate +# once (and upload the artifact) and then check. verify-* is the one-shot for humans. +# +# Each checks that git tracks the page before diffing it: `git diff` says nothing about a +# file git is not tracking, so a page deleted from the repo and then regenerated would +# sail through the diff as "current" while not being published at all. +check-leaderboard: + @git ls-files --error-unmatch docs/LEADERBOARD.md docs/LEADERBOARD.json >/dev/null 2>&1 || { \ + echo "docs/LEADERBOARD.{md,json}: not tracked by git." >&2; \ + echo "A generated page git does not track cannot be verified — commit it." >&2; \ + exit 1; \ + } + @git diff --exit-code -- docs/LEADERBOARD.md docs/LEADERBOARD.json || { \ + echo "" >&2; \ + echo "docs/LEADERBOARD.{md,json} are stale — the diff above is what regeneration produced." >&2; \ + echo "Run 'make leaderboard', commit the result, and open a pull request." >&2; \ + exit 1; \ + } + +check-benchmark: + @git ls-files --error-unmatch docs/BENCHMARK.md docs/BENCHMARK.json >/dev/null 2>&1 || { \ + echo "docs/BENCHMARK.{md,json}: not tracked by git." >&2; \ + echo "A generated table git does not track cannot be verified — commit it." >&2; \ + exit 1; \ + } + @git diff --exit-code -- docs/BENCHMARK.md docs/BENCHMARK.json || { \ + echo "" >&2; \ + echo "docs/BENCHMARK.{md,json} are stale — the diff above is what regeneration produced." >&2; \ + echo "Run 'make benchmark', commit the result, and open a pull request." >&2; \ + exit 1; \ + } + +verify-leaderboard: leaderboard check-leaderboard + +verify-benchmark: benchmark check-benchmark + +verify-published-numbers: verify-leaderboard verify-benchmark + +test: + $(PY) -m pytest tests/ -q diff --git a/docs/BENCHMARK.md b/docs/BENCHMARK.md index 87c29b1..b367ea7 100644 --- a/docs/BENCHMARK.md +++ b/docs/BENCHMARK.md @@ -1,4 +1,4 @@ - + # Signetry detection benchmark — 60 public test cases diff --git a/docs/LEADERBOARD.json b/docs/LEADERBOARD.json index eaddcb1..887f751 100644 --- a/docs/LEADERBOARD.json +++ b/docs/LEADERBOARD.json @@ -6,7 +6,7 @@ "name": "signetry-core", "status": "reproduced", "url": "https://github.com/Signetry/core", - "version": "0.7.0", + "version": "0.8.0", "attacks_run": 5, "asr_ungoverned": 1.0, "asr_governed": 0.0, diff --git a/docs/LEADERBOARD.md b/docs/LEADERBOARD.md index a1962e0..3a4f012 100644 --- a/docs/LEADERBOARD.md +++ b/docs/LEADERBOARD.md @@ -1,4 +1,4 @@ - + # The Agent Governance Leaderboard @@ -25,7 +25,7 @@ by blocking everything is not a win. | System | Evidence | Attacks | ASR ungoverned | ASR governed | Reduction | Utility preserved | |---|:---:|:---:|:---:|:---:|:---:|:---:| -| [signetry-core `0.7.0`](https://github.com/Signetry/core) | reproduced here | 5 | 100% | 0% | 100% | 100% | +| [signetry-core `0.8.0`](https://github.com/Signetry/core) | reproduced here | 5 | 100% | 0% | 100% | 100% | | ungoverned agent (no checkpoint) | reproduced here | 5 | 100% | 100% | 0% | — | ### What the governed column does and does not claim @@ -89,6 +89,10 @@ Both paths are ordinary pull requests. See Regenerate this page locally with: ```bash -signetry-eval leaderboard --markdown +make leaderboard ``` +That is the same recipe CI runs to check this page is current, so the page you +regenerate is the page CI expects. Refreshing it is a pull request, not a bot +push: these numbers reach `main` the way every other change does. + diff --git a/signetry_eval/leaderboard.py b/signetry_eval/leaderboard.py index cd6171a..e77c2e9 100644 --- a/signetry_eval/leaderboard.py +++ b/signetry_eval/leaderboard.py @@ -252,7 +252,8 @@ def render_leaderboard(report: Any, *, detection_markdown: str | None = None, o = report.overall() lines = [ - "", + "", "", "# The Agent Governance Leaderboard", "", @@ -360,8 +361,12 @@ def render_leaderboard(report: Any, *, detection_markdown: str | None = None, "Regenerate this page locally with:", "", "```bash", - "signetry-eval leaderboard --markdown", + "make leaderboard", "```", + "", + "That is the same recipe CI runs to check this page is current, so the page you", + "regenerate is the page CI expects. Refreshing it is a pull request, not a bot", + "push: these numbers reach `main` the way every other change does.", ] return "\n".join(lines) + "\n" diff --git a/tests/test_leaderboard.py b/tests/test_leaderboard.py index 7dc3824..1b72185 100644 --- a/tests/test_leaderboard.py +++ b/tests/test_leaderboard.py @@ -190,9 +190,13 @@ def test_the_page_states_its_own_sample_size_in_prose(): assert "small corpus" in page, "the page must admit the corpus is small" -def test_the_generated_header_names_the_workflow_that_writes_it(): +def test_the_generated_header_names_the_recipe_and_the_verifying_workflow(): + # The page is generated by `make leaderboard` and only *checked* by the workflow. + # The header has to name both: the recipe, so a reader can reproduce the page, and + # the workflow, so they know what fails when they edit it by hand. page = render_leaderboard(run_all(None)) first = page.strip().split("\n")[0] + assert "make leaderboard" in first assert ".github/workflows/leaderboard.yml" in first assert "do not edit by hand" in first diff --git a/tests/test_published_numbers.py b/tests/test_published_numbers.py new file mode 100644 index 0000000..3b67fca --- /dev/null +++ b/tests/test_published_numbers.py @@ -0,0 +1,185 @@ +"""The published pages are verified, never pushed — enforced here, not by memory. + +`docs/LEADERBOARD.{md,json}` and `docs/BENCHMARK.{md,json}` are generated files that live +in the repository. Two workflows used to regenerate them and push the result straight to +`main`; `main` requires a pull request, so every run that actually had something to publish +was rejected by branch protection, and because the push came *before* each workflow's gate, +a broken defense would have surfaced as a git error instead of a failed gate. + +The fix has three parts, and each is easy to undo by accident, so each is asserted: + +1. Neither workflow can write to the repository — the pages arrive by pull request. +2. Both use the `Makefile` recipe rather than their own inlined copy of the commands, so + the page a contributor regenerates is byte-for-byte the page CI checks against. +3. Both gates carry `if: !cancelled() && ...`, so an earlier failure cannot mask them. + +These are text assertions on purpose: PyYAML is not a dependency of this suite, and adding +one to check four lines of workflow would cost more than it buys. +""" +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOWS = ROOT / ".github" / "workflows" + +# (workflow, make target, the page it owns) +OWNERSHIP = [ + ("leaderboard.yml", "leaderboard", "LEADERBOARD"), + ("benchmark.yml", "benchmark", "BENCHMARK"), +] + +pytestmark = pytest.mark.skipif( + not WORKFLOWS.is_dir(), + reason="not running from a checkout (the workflows are not installed with the package)", +) + + +def _code_lines(text: str) -> list[str]: + """Every line that is not a YAML comment. + + The workflows explain their own history in comments — including what they used to do — + so a check for `git push` has to look at what runs, not at what is described. + """ + return [ln for ln in text.split("\n") if not ln.strip().startswith("#")] + + +@pytest.mark.parametrize("workflow,target,page", OWNERSHIP) +def test_the_workflow_cannot_write_to_the_repository(workflow: str, target: str, page: str): + text = (WORKFLOWS / workflow).read_text() + block = re.search(r"^permissions:\n(?:[ \t]+\S.*\n)+", text, re.M) + assert block, f"{workflow} must declare permissions explicitly" + assert "contents: read" in block.group(0), f"{workflow} must take a read-only token" + assert "write" not in block.group(0), ( + f"{workflow} requests write access. It publishes nothing: the pages reach `main` " + "through a pull request, which is what branch protection requires and what this " + "repository argues for." + ) + + +@pytest.mark.parametrize("workflow,target,page", OWNERSHIP) +def test_the_workflow_does_not_push(workflow: str, target: str, page: str): + code = "\n".join(_code_lines((WORKFLOWS / workflow).read_text())) + for forbidden in ("git push", "git commit", "git config user"): + assert forbidden not in code, ( + f"{workflow} runs `{forbidden}`. Pushing to a protected branch is the bug this " + "design removed — regenerate, diff, and fail if the committed page is stale." + ) + + +@pytest.mark.parametrize("workflow,target,page", OWNERSHIP) +def test_the_workflow_uses_the_shared_recipe(workflow: str, target: str, page: str): + code = "\n".join(_code_lines((WORKFLOWS / workflow).read_text())) + assert f"make {target}" in code, ( + f"{workflow} must regenerate with `make {target}`, not its own copy of the " + "commands: a regenerate-and-diff check is only meaningful if CI and contributors " + "run the same recipe." + ) + assert f"make check-{target}" in code, ( + f"{workflow} must verify with `make check-{target}` — regenerating without diffing " + "checks nothing." + ) + + +@pytest.mark.parametrize("workflow,target,page", OWNERSHIP) +def test_the_checkout_ref_is_not_a_falsy_ternary(workflow: str, target: str, page: str): + """`A && '' || B` is always B, so the workflow would verify the wrong tree. + + Actions' `&&` and `||` yield operand values rather than booleans, which makes the + familiar ternary spelling a trap whenever the true branch is falsy: an empty string + collapses straight through to the false branch. Both of these workflows shipped exactly + that for one run, and PR #39 verified `main` instead of itself — the check passed and + proved nothing about the change under review. + """ + # Comments only, and the comment above the ref quotes the trap to explain it. + code = "\n".join(_code_lines((WORKFLOWS / workflow).read_text())) + for trap in ("&& '' ||", '&& "" ||', "&& ''||"): + assert trap not in code, ( + f"{workflow} uses a ternary with an empty true branch. Actions evaluates " + f"`cond && '' || other` to `other` for every event, so the ref this resolves to " + "is not the one it looks like. Give the true branch a non-empty value." + ) + + +@pytest.mark.parametrize("workflow,target,page", OWNERSHIP) +def test_the_pull_request_run_verifies_the_pull_request(workflow: str, target: str, page: str): + """A verification that checks out the default branch on a PR verifies nothing. + + `github.ref` is `refs/pull/N/merge` on a pull request — the merge result, which is what + actually lands. Only a release needs the override, because its ref is the tag rather + than the branch the page is committed on. + """ + text = (WORKFLOWS / workflow).read_text() + ref = re.search(r"^\s+ref: (.+)$", text, re.M) + assert ref, f"{workflow} must pin the checkout ref explicitly" + expr = ref.group(1) + assert "github.ref" in expr, ( + f"{workflow} must verify the triggering ref (so a PR verifies itself); got {expr!r}" + ) + assert "release" in expr, ( + f"{workflow} must special-case the release event, whose ref is the tag and not where " + f"the page is committed; got {expr!r}" + ) + + +@pytest.mark.parametrize("workflow,target,page", OWNERSHIP) +def test_the_workflow_refuses_a_tree_that_lacks_the_recipe(workflow: str, target: str, page: str): + """`make ` can exit 0 having done nothing at all. + + With no Makefile in the tree, make falls back to implicit rules — and `leaderboard` is a + real directory in this repository, so make calls the target up to date and prints + "Nothing to be done for 'leaderboard'" with status 0. That is how a run regenerated + nothing, uploaded the committed page as its artifact, and still reported the + regeneration step green. Same shape as the push step that exited 0 on "no change to + publish": a step that succeeds without doing its job. + """ + code = "\n".join(_code_lines((WORKFLOWS / workflow).read_text())) + assert "test -f Makefile" in code, ( + f"{workflow} must confirm the recipe is present before running it — otherwise a " + "wrong-tree checkout regenerates nothing and still passes." + ) + + +@pytest.mark.parametrize("workflow,target,page", OWNERSHIP) +def test_the_gate_cannot_be_masked_by_an_earlier_failure(workflow: str, target: str, page: str): + text = (WORKFLOWS / workflow).read_text() + assert "id: regen" in text, f"{workflow} needs an id on the regeneration step for the gate to key off" + assert "!cancelled() && steps.regen.outcome == 'success'" in text, ( + f"{workflow}'s gate must run even when the staleness check failed. Without the " + "guard, a stale page aborts the job first and the defense is never judged — which " + "is exactly how a rejected push used to hide a possible regression." + ) + + +@pytest.mark.parametrize("workflow,target,page", OWNERSHIP) +def test_the_committed_page_names_the_recipe_that_writes_it(workflow: str, target: str, page: str): + md = ROOT / "docs" / f"{page}.md" + if not md.is_file(): + pytest.skip(f"docs/{page}.md is not present in this tree") + first = md.read_text().strip().split("\n")[0] + assert f"make {target}" in first, ( + f"docs/{page}.md must name the recipe that regenerates it, so a reader can " + f"reproduce the page instead of hand-editing it (got: {first!r})" + ) + assert workflow in first, f"docs/{page}.md must name the workflow that verifies it (got: {first!r})" + assert "do not edit by hand" in first + + +def test_the_benchmark_header_is_not_duplicated_out_of_sync(): + """The benchmark header is written by the Makefile; the leaderboard's by Python. + + Only the benchmark's lives in two places (the recipe that writes it and the committed + file), so only it can drift silently. Pin them together. + """ + makefile = ROOT / "Makefile" + md = ROOT / "docs" / "BENCHMARK.md" + if not (makefile.is_file() and md.is_file()): + pytest.skip("Makefile or docs/BENCHMARK.md is not present in this tree") + first = md.read_text().strip().split("\n")[0] + assert first in makefile.read_text(), ( + "docs/BENCHMARK.md's header is not the one `make benchmark` emits — regenerate it " + f"(committed header: {first!r})" + )