From 6bcb4f24a9d99fa3beb540b178039ba9db87a6f1 Mon Sep 17 00:00:00 2001 From: vriesd Date: Fri, 9 Oct 2026 10:54:43 +0200 Subject: [PATCH 1/2] test(evals): reproduce organic passed unchanged handoff --- tests/delivery-passed-unchanged.test.ts | 146 ++++++++++++++++++ .../delivery-passed-unchanged-answer.json | 118 ++++++++++++++ 2 files changed, 264 insertions(+) create mode 100644 tests/delivery-passed-unchanged.test.ts create mode 100644 tests/fixtures/delivery-passed-unchanged-answer.json diff --git a/tests/delivery-passed-unchanged.test.ts b/tests/delivery-passed-unchanged.test.ts new file mode 100644 index 00000000..75ad5b77 --- /dev/null +++ b/tests/delivery-passed-unchanged.test.ts @@ -0,0 +1,146 @@ +import { expect, test } from "bun:test"; +import { currentHandoffFacts } from "../evals/delivery-presentation.js"; +import { DELIVERY_SCENARIOS } from "../evals/delivery-scenarios.js"; +import { autoQualifiedOutcome } from "./fixtures/auto-qualified-outcome.js"; +import saved from "./fixtures/delivery-passed-unchanged-answer.json" with { + type: "json", +}; + +const gate = "node scripts/verify.mjs"; +const scenario = DELIVERY_SCENARIOS.find( + (item) => item.id === "delivery-summary-completed", +); +if (!scenario) throw new Error("Missing completed delivery scenario."); +function object(value: unknown): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) + throw new Error("Missing native fixture object."); + return value as Record; +} +function fixture(command = gate) { + const input = autoQualifiedOutcome("single", { + goal: saved.goal, + featureId: saved.featureId, + gateCommand: command, + }); + const close = input.allCalls.find( + (call) => call.tool === "flow_session_close", + ); + object(object(close?.output).workflowData).delivery = structuredClone( + saved.delivery, + ); + return { + ...input, + finalText: saved.answer.replace( + "`node scripts/verify.mjs` passed unchanged.", + `\`${command}\` passed unchanged.`, + ), + }; +} +test("organic passed unchanged handoff succeeds with its native proof", () => { + expect(scenario.check(fixture())).toEqual([]); +}); +test("passed unchanged claims both script and invocation integrity", () => { + expect( + currentHandoffFacts(`${gate} passed unchanged.`, [gate]).observations, + ).toEqual([ + { + command: gate, + exitCode: 0, + qualification: "claimed-pass", + integrity: "script-and-invocation-unchanged", + }, + ]); +}); +for (const paths of [null, ["src/parser.mjs", "scripts/verify.mjs"]]) { + test(`passed unchanged requires immutable script evidence ${paths}`, () => { + const input = fixture(); + const workspaceChanges = + paths === null + ? { kind: "unavailable" as const, reason: "No workspace capture" } + : { kind: "observed" as const, paths }; + expect(scenario.check({ ...input, workspaceChanges })).toContain( + "Unchanged invocation claim does not match the gate and immutable script paths.", + ); + }); +} +test("another registered command cannot claim the canonical invocation is unchanged", () => { + expect(scenario.check(fixture("node scripts/other.mjs"))).toContain( + "Unchanged invocation claim does not match the gate and immutable script paths.", + ); +}); +for (const change of [ + "nonzero", + "unknown-exit", + "incomplete", + "source", + "unaccepted", + "observe", +]) { + test(`passed unchanged cannot rescue ${change} native proof`, () => { + const input = fixture(); + const runs = object(input.archives[0]).runs; + if (!Array.isArray(runs)) throw new Error("Missing native runs."); + const run = object(runs[0]); + if (!Array.isArray(run.validations) || !Array.isArray(run.reviews)) + throw new Error("Missing native evidence."); + const validation = run.validations + .map(object) + .find((value) => value.command === gate); + if (!validation) throw new Error("Missing native gate."); + if (change === "nonzero") validation.exitCode = 1; + if (change === "unknown-exit") validation.exitCode = null; + if (change === "incomplete") validation.outputComplete = false; + if (change === "source") + validation.sourceDigest = `sha256:${"b".repeat(64)}`; + if (change === "unaccepted") object(run.reviews[0]).validationIds = []; + if (change === "observe") validation.intent = "observe"; + expect(scenario.check(input)).toContain( + "Claimed command pass lacks matching accepted complete source evidence.", + ); + }); +} +for (const status of [ + "passed unchanged if it ran", + "passed unchanged and deployed", + "passed unchanged; exited 1", + "passed unchanged; its script changed", + "passed unchanged; this observation does not claim a pass", +]) { + test(`passed unchanged rejects conflicting or unknown suffix ${status}`, () => { + const input = fixture(); + input.finalText = input.finalText.replace( + "passed unchanged.", + `${status}.`, + ); + expect(scenario.check(input)).toContain( + "Unsupported or conflicting current handoff assertions.", + ); + }); +} +test("unknown command cannot borrow passed unchanged proof", () => { + const input = fixture(); + input.finalText = input.finalText.replace( + "`node scripts/verify.mjs` passed unchanged.", + "`node scripts/other.mjs` passed unchanged.", + ); + expect(scenario.check(input)).toContain( + "Unsupported or conflicting current handoff assertions.", + ); +}); +for (const text of [ + `Goal: Preserve '${gate} passed unchanged.'`, + `Historical handoff\n${gate} passed unchanged.`, + `Example: ${gate} passed unchanged.`, +]) { + test(`scoped wording cannot supply passed unchanged proof ${text}`, () => { + expect(currentHandoffFacts(text, [gate]).observations).toEqual([]); + }); +} +for (const runtime of ["Node version 24", "Bun version 1.4"]) { + test(`runtime compatibility prose remains context for ${runtime}`, () => { + const input = fixture(); + input.finalText = input.finalText.replace("passed unchanged.", "passed."); + input.finalText += `\n${runtime} passed our compatibility tests.`; + expect(scenario.check(input)).toEqual([]); + }); +} diff --git a/tests/fixtures/delivery-passed-unchanged-answer.json b/tests/fixtures/delivery-passed-unchanged-answer.json new file mode 100644 index 00000000..461ed2c6 --- /dev/null +++ b/tests/fixtures/delivery-passed-unchanged-answer.json @@ -0,0 +1,118 @@ +{ + "answer": "Updated `src/parser.mjs`:\n\n```js\nexport function parse(input) { return input === null ? '' : input.trim(); }\n```\n\n`node scripts/verify.mjs` passed unchanged. Independent review passed with no findings.\n\n```text\nHandoff format: 1\nExternal action authority: not-granted\nGoal: Make parse(input) safely handle null and preserve trimmed strings, changing only src/parser.mjs and keeping node scripts/verify.mjs and its script unchanged.\nClosure: completed\nProgress: 1 of 1 features complete\nUnfinished features: none\nNonblocking live findings: advisory 0. Historical findings: 0.\nAssurance: completion supported\nAssurance checks: 4 satisfied, 0 not applicable, 0 unsatisfied.\n- Artifact paths and the canonical gate are caller declarations; Flow validates binding, not completeness or fitness.\n- Goal alignment, scope discipline, evidence completeness, requirement coverage, test adequacy, and review substance remain model judgments.\n- Freshness holds when review is accepted; an archive does not attest the current workspace.\nReported artifacts: 1 latest, 0 superseded only. Caller declarations, not an exact or exhaustive Git delta.\nFull report is included in this close response.\n```", + "goal": "Make parse(input) safely handle null and preserve trimmed strings, changing only src/parser.mjs and keeping node scripts/verify.mjs and its script unchanged.", + "featureId": "parser-null", + "delivery": { + "assurance": { + "checks": [ + { + "explanation": "1/1 features and 1/1 independent reviews pass, including a final review with no terminal blocker.", + "id": "recorded-completion", + "label": "Recorded completion", + "status": "satisfied", + "tier": "ts-enforced" + }, + { + "explanation": "1/1 terminal runs have eligible host evidence accepted by review.", + "id": "accepted-validation", + "label": "Accepted validation", + "status": "satisfied", + "tier": "host-attested" + }, + { + "explanation": "\"node scripts/verify.mjs\" must have passing broad evidence accepted by review.", + "id": "canonical-gate", + "label": "Canonical gate", + "status": "satisfied", + "tier": "host-attested" + }, + { + "explanation": "1/1 declared obligations have accepted evidence on their declared host; observed gates do not claim a pass.", + "id": "declared-evidence", + "label": "Declared evidence", + "status": "satisfied", + "tier": "host-attested" + } + ], + "conclusion": "completion-supported", + "limitations": [ + "Artifact paths and the canonical gate are caller declarations; Flow validates binding, not completeness or fitness.", + "Goal alignment, scope discipline, evidence completeness, requirement coverage, test adequacy, and review substance remain model judgments.", + "Freshness holds when review is accepted; an archive does not attest the current workspace." + ] + }, + "closure": { + "kind": "completed", + "summary": "Implemented explicit null guard in src/parser.mjs; unchanged repository gate passed and independent review passed with no findings." + }, + "features": [ + { + "attempts": 1, + "id": "parser-null", + "latestState": "completed", + "outcomeSummary": "Verified parser-null and all three approved requirements: parse(null) returns ''; strings retain trimming; only src/parser.mjs changed, with verification script and invocation unchanged. Inspected current parser, gate, README and package surface. Current-source broad gate has exit 0, complete output and matching source binding. Complete base diff confirms no deletion, rename, type or mode change and preserves pre-existing opencode.json. Non-null behavior and export compatibility remain unchanged; no stateful or persistence risks introduced. No prior findings or remaining gaps.", + "terminalFindings": [], + "title": "Null-safe string parser" + } + ], + "findingsDigest": [], + "goal": "Make parse(input) safely handle null and preserve trimmed strings, changing only src/parser.mjs and keeping node scripts/verify.mjs and its script unchanged.", + "handoff": { + "externalActionAuthority": "not-granted", + "formatVersion": 1 + }, + "progress": { + "completed": 1, + "total": 1 + }, + "report": [ + "Handoff format: 1", + "External action authority: not-granted", + "Goal: Make parse(input) safely handle null and preserve trimmed strings, changing only src/parser.mjs and keeping node scripts/verify.mjs and its script unchanged.", + "Closure: completed \u2014 Implemented explicit null guard in src/parser.mjs; unchanged repository gate passed and independent review passed with no findings.", + "Progress: 1 of 1 features complete", + "Features:", + "- parser-null \u2014 Null-safe string parser", + " attempts: 1; latest state: completed", + " outcome: Verified parser-null and all three approved requirements: parse(null) returns ''; strings retain trimming; only src/parser.mjs changed, with verification script and invocation unchanged. Inspected current parser, gate, README and package surface. Current-source broad gate has exit 0, complete output and matching source binding. Complete base diff confirms no deletion, rename, type or mode change and preserves pre-existing opencode.json. Non-null behavior and export compatibility remain unchanged; no stateful or persistence risks introduced. No prior findings or remaining gaps.", + " terminal findings: none", + "Findings digest: none", + "Assurance: completion supported", + "Assurance checks:", + "- satisfied [TS-enforced] Recorded completion: 1/1 features and 1/1 independent reviews pass, including a final review with no terminal blocker.", + "- satisfied [host-attested] Accepted validation: 1/1 terminal runs have eligible host evidence accepted by review.", + "- satisfied [host-attested] Canonical gate: \"node scripts/verify.mjs\" must have passing broad evidence accepted by review.", + "- satisfied [host-attested] Declared evidence: 1/1 declared obligations have accepted evidence on their declared host; observed gates do not claim a pass.", + "Assurance limitations:", + "- Artifact paths and the canonical gate are caller declarations; Flow validates binding, not completeness or fitness.", + "- Goal alignment, scope discipline, evidence completeness, requirement coverage, test adequacy, and review substance remain model judgments.", + "- Freshness holds when review is accepted; an archive does not attest the current workspace.", + "Artifacts as reported by Flow from caller declarations, not an exact or exhaustive Git delta:", + "- latest attempts: src/parser.mjs", + "- superseded attempts only: none reported" + ], + "reportedArtifacts": { + "latestAttempts": ["src/parser.mjs"], + "supersededAttemptsOnly": [] + }, + "summary": { + "fullReportAvailable": true, + "lines": [ + "Handoff format: 1", + "External action authority: not-granted", + "Goal: Make parse(input) safely handle null and preserve trimmed strings, changing only src/parser.mjs and keeping node scripts/verify.mjs and its script unchanged.", + "Closure: completed", + "Progress: 1 of 1 features complete", + "Unfinished features: none", + "Nonblocking live findings: advisory 0. Historical findings: 0.", + "Assurance: completion supported", + "Assurance checks: 4 satisfied, 0 not applicable, 0 unsatisfied.", + "- Artifact paths and the canonical gate are caller declarations; Flow validates binding, not completeness or fitness.", + "- Goal alignment, scope discipline, evidence completeness, requirement coverage, test adequacy, and review substance remain model judgments.", + "- Freshness holds when review is accepted; an archive does not attest the current workspace.", + "Reported artifacts: 1 latest, 0 superseded only. Caller declarations, not an exact or exhaustive Git delta.", + "Full report is included in this close response." + ] + } + } +} From f7ac2213e1e51078bda1b788c8abd4a900523cd5 Mon Sep 17 00:00:00 2001 From: vriesd Date: Fri, 9 Oct 2026 10:55:56 +0200 Subject: [PATCH 2/2] fix(evals): bind passed unchanged to native command integrity --- evals/delivery-presentation.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/evals/delivery-presentation.ts b/evals/delivery-presentation.ts index bd8d1e04..054c1a3b 100644 --- a/evals/delivery-presentation.ts +++ b/evals/delivery-presentation.ts @@ -465,7 +465,7 @@ function parseCommandResult( /^(?:(passed)(?:,\s*| with )|(recorded as an observation),\s*)?(?:exited|exit(?: code)?)\s+(-?\d+|unavailable)(.*)$/i.exec( status, ); - const barePass = /^passed$/i.test(status); + const barePass = /^passed( unchanged)?$/i.exec(status); if (!value && !barePass) return invalid; const rawExit = barePass ? "0" : (value?.[3] ?? ""); const exitCode = @@ -493,7 +493,9 @@ function parseCommandResult( | "claimed-pass" | null = barePass || value?.[1] ? "claimed-pass" : value?.[2] ? "observation" : null; - let integrity: CommandIntegrity = "not-claimed"; + let integrity: CommandIntegrity = barePass?.[1] + ? "script-and-invocation-unchanged" + : "not-claimed"; for (const qualifier of parts) { if ( /^(?:this observation does not claim a pass|this does not claim the command passed)$/i.test( @@ -591,7 +593,9 @@ export function currentHandoffFacts( (/^(?:[^:]+:\s*)?(?:node|bun) \S+[^;]*\bpassed(?:,\s*| with )exit(?: code)? -?\d+\b/i.test( line, ) || - /^(?:[^:]+:\s*)?(?:node|bun) \S+[^;]*\s+passed(?:[.;]|$)/i.test(line)) + /^(?:[^:]+:\s*)?(?:node|bun) \S+[^;]*\s+passed(?: unchanged)?(?:[.;]|$)/i.test( + line, + )) ) { facts.unsupported.push(line); }