diff --git a/.github/workflows/agent-isolation.yml b/.github/workflows/agent-isolation.yml index f8d00633..838004a6 100644 --- a/.github/workflows/agent-isolation.yml +++ b/.github/workflows/agent-isolation.yml @@ -1,44 +1,9 @@ name: Agent isolation +# Runs on every pull request and on pushes to the same branches as CI, so `real-docker` can be a required check. on: push: - branches: [main] - paths: - - 'agents/**' - - 'git/clone.ts' - - 'test/agent-*.test.ts' - - 'runner/question-*.ts' - - 'runner/questions.ts' - - 'runner/planning*.ts' - - 'runner/workspace.ts' - - 'runner/runner-repository.ts' - - 'runner/execution.ts' - - 'core/run-audit.ts' - - 'runner/review.ts' - - 'runner/store.ts' - - 'git/history.ts' - - 'test/runner-workspace.test.ts' - - 'test/vitest-config.test.ts' - - 'vitest.config.ts' - - '.github/workflows/agent-isolation.yml' + branches: [main, 'codex/**'] pull_request: - paths: - - 'agents/**' - - 'git/clone.ts' - - 'test/agent-*.test.ts' - - 'runner/question-*.ts' - - 'runner/questions.ts' - - 'runner/planning*.ts' - - 'runner/workspace.ts' - - 'runner/runner-repository.ts' - - 'runner/execution.ts' - - 'core/run-audit.ts' - - 'runner/review.ts' - - 'runner/store.ts' - - 'git/history.ts' - - 'test/runner-workspace.test.ts' - - 'test/vitest-config.test.ts' - - 'vitest.config.ts' - - '.github/workflows/agent-isolation.yml' permissions: contents: read jobs: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7709445..a731a63a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,9 @@ permissions: jobs: test: runs-on: ubuntu-latest + # The longest recent run took 20 min, about half of this limit. A hung browser suite would otherwise hold the + # runner for GitHub's default of 360 min. + timeout-minutes: 45 steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -16,8 +19,9 @@ jobs: cache: npm - run: npm ci --ignore-scripts - run: npm run typecheck - # The Docker agent suites run one file at a time in the Agent isolation workflow; running them here - # would put them in parallel against the same image tag and daemon. - - run: npm test -- --exclude test/agent-container.test.ts --exclude test/agent-network.test.ts --exclude test/agent-adapter.test.ts --exclude test/agent-supervisor.test.ts --exclude test/agent-gate.test.ts --exclude test/agent-question.test.ts --exclude test/agent-planning.test.ts --exclude test/runner-workspace.test.ts + # The Docker-backed files run one at a time in the Agent isolation workflow; running them here would put them in + # parallel against the same image tag and daemon. Exclude exactly the dockerTests list in vitest.config.ts; + # test/vitest-config.test.ts fails if the two differ. + - run: npm test -- --exclude test/agent-container.test.ts --exclude test/agent-gate.test.ts --exclude test/agent-network.test.ts --exclude test/agent-planning.test.ts --exclude test/agent-question.test.ts --exclude test/agent-supervisor.test.ts --exclude test/runner-workspace.test.ts - run: npx playwright install --with-deps chromium - run: npm run test:browser diff --git a/.github/workflows/live-probes.yml b/.github/workflows/live-probes.yml new file mode 100644 index 00000000..78e18b52 --- /dev/null +++ b/.github/workflows/live-probes.yml @@ -0,0 +1,48 @@ +name: Live probes +# Runs the live Claude startup probes against the real Claude service. It is not a pull request check: it runs +# nightly on the default branch and when started by hand. See docs/implementation/ci.md. +on: + workflow_dispatch: + schedule: + - cron: '23 6 * * *' +permissions: + contents: read +concurrency: + group: live-probes + cancel-in-progress: false +jobs: + live-probes: + runs-on: ubuntu-latest + # The probes' own time limits total 33 min (6, 8, 8 and 11). agent-container and agent-supervisor may each spend + # up to 10 min building the image before their probes; agent-question builds inside its probe's own 11 min. The + # teardown hooks have Vitest limits that total 9 min: agent-container 2 after each test and 2 at the end, + # agent-supervisor 3 and agent-question 2. That is 62 min, plus setup. These are only Vitest's limits: the afterAll + # hooks in agent-container and agent-question call docker through spawnSync with no timeout of their own, and + # Vitest's hook timer cannot fire while such a call blocks. The 75 min job limit caps the run. + timeout-minutes: 75 + steps: + # Fail at once when the secret is missing, so a missing secret never looks like a passing run. + - name: Require the Claude token + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + run: | + if [ -z "$CLAUDE_CODE_OAUTH_TOKEN" ]; then + echo "::error::The repository secret CLAUDE_CODE_OAUTH_TOKEN is not set. Add it to run the live probes." + exit 1 + fi + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '26.7.0' + cache: npm + - run: npm ci --ignore-scripts + # The docker project runs these files one at a time, as in the Agent isolation workflow. The filter keeps only + # the tests whose titles end in "(live Claude)". + - name: Run the live Claude probes + env: + CODEBOOST_RUN_AUTH_PROBES: '1' + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + run: npx vitest run --project docker -t '\(live Claude\)$' --reporter=default --reporter=json --outputFile=live-probes.json test/agent-container.test.ts test/agent-question.test.ts test/agent-supervisor.test.ts + # Vitest exits 0 when it skips every probe, so check that each probe passed and nothing else ran. + - name: Check that every probe passed + run: node scripts/live-probes.ts check live-probes.json diff --git a/README.md b/README.md index 8aa4776f..f83204f2 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ npm run typecheck npm test ``` -Tests create disposable local repositories and exercise command checks in disposable credential-free containers. They do not invoke model providers or access GitHub. +Tests create disposable local repositories and exercise command checks in disposable credential-free containers. They do not invoke model providers or access GitHub. [CI](docs/implementation/ci.md) lists the workflows, the required checks and the live Claude probes. ## Guarded GitHub merge diff --git a/docs/designs/codeboost-plan-indexed-review.md b/docs/designs/codeboost-plan-indexed-review.md index f7d283bc..1ce0aa2d 100644 --- a/docs/designs/codeboost-plan-indexed-review.md +++ b/docs/designs/codeboost-plan-indexed-review.md @@ -1922,7 +1922,7 @@ Read each row left to right: finish and validate step 1 before step 2 within tha | C — guarded merge gate | **C1.** Required-check and branch-rule reads (T12). **C2.** Snapshot/evidence blockers, with unavailable T6 execution evidence blocking merge. **C3.** Head-pinned, base-protected merge and refusal handling (T7). **C4.** Review UI, race regressions and final #21 / PR #23 review. | Before C1, record B0 evidence for the foundation contracts C consumes; existing work must supply that evidence before C4 completion. Continue existing work rather than restarting implemented steps. Release shared runner/UI files after C4 merges. | | D — agent isolation | **D1.** Invocation contract and isolated task clone (T1). **D2.** Pinned, restricted container and startup self-test (T1). **D3.** Vendor-only egress and phase/tool enforcement (T2). **D4.** Claude/Codex adapters, cancellation settlement and bounded output. **D5.** Full real-Docker and hostile-input gate for this boundary (T9). | Can run alongside C and E. F requires D5 merged; G's production invocation requires D5. Add regressions with each step; D5 integrates them rather than postponing testing. | | E — planning logic | **E1.** Audit existing T18 schema/parser/prompt behavior and remaining #6 gaps. **E2.** Read-only authoring-provider contract and safe prompt/response handling. **E3.** Identity/revision-bound suggestion orchestration using the existing store interface. **E4.** Import, replay, malformed-response and hostile-input acceptance fixtures (T18). | Can run alongside C and D with injected providers. G consumes E4; live invocation waits for D5. Shared schema/store fixes must go through the assigned integration owner. | -| F — runner and pre-merge automation | **F1.** Before implementation, publish and review the lifecycle/state-holder contract: pending, running, completed, failed, cancelled, stale and closing; legal transitions; ownership and settlement for persisted records, in-memory jobs, subprocesses, admitted HTTP requests and rendered UI; guarded retry; reject-admission → drain requests → cancel/await jobs → close storage. Then implement it and the feedback-event contract under the AGENTS.md async rules. **F2.** Per-item execution, review/reject rounds, pre-PR already-fixed checks, PR opening and hostile-issue eval (build step 5; T9). **F3.** Trusted rebase and ledger mapping (remaining T3). **F4.** Foreign-commit conflict handling (T11). **F5.** Post-rebase attribution/approval refresh and head-bound command execution (T6). **F6.** Required-check refresh, already-fixed check, guarded merge handoff and #22 integration regressions. F owns common CI after C: integrate every T9 suite (Docker, adapter, hostile-input/issue, recorded-output, unit and browser) into required CI, coordinating D's dedicated workflow. T9 remains incomplete until the combined head demonstrably runs and passes every suite. | Starts after C4 and D5 merge and B0 evidence is handed off for F's consumed contracts. Recheck that evidence against merged main before F1; existing T4/T5/T10 behavior is reused rather than rebuilt. F1 owns planning persistence/API additions needed by G. F2's working reject loop supplies the learning dependency. Ask already uses D's invocation contract (`runner/question-container.ts`), so F reuses that path rather than adding a second one. | +| F — runner and pre-merge automation | **F1.** Before implementation, publish and review the lifecycle/state-holder contract: pending, running, completed, failed, cancelled, stale and closing; legal transitions; ownership and settlement for persisted records, in-memory jobs, subprocesses, admitted HTTP requests and rendered UI; guarded retry; reject-admission → drain requests → cancel/await jobs → close storage. Then implement it and the feedback-event contract under the AGENTS.md async rules. **F2.** Per-item execution, review/reject rounds, pre-PR already-fixed checks, PR opening and hostile-issue eval (build step 5; T9). **F3.** Trusted rebase and ledger mapping (remaining T3). **F4.** Foreign-commit conflict handling (T11). **F5.** Post-rebase attribution/approval refresh and head-bound command execution (T6). **F6.** Required-check refresh, already-fixed check, guarded merge handoff and #22 integration regressions. F owns common CI after C: integrate every T9 suite (Docker, adapter, hostile-input/issue, recorded-output, unit and browser) into required CI, coordinating D's dedicated workflow. T9 remains incomplete until the combined head demonstrably runs and passes every suite. *F6c status:* the workflows are in place (`docs/implementation/ci.md`); required-check settings, a passing run of both required jobs on one head and a green `Live probes` run remain. | Starts after C4 and D5 merge and B0 evidence is handed off for F's consumed contracts. Recheck that evidence against merged main before F1; existing T4/T5/T10 behavior is reused rather than rebuilt. F1 owns planning persistence/API additions needed by G. F2's working reject loop supplies the learning dependency. Ask already uses D's invocation contract (`runner/question-container.ts`), so F reuses that path rather than adding a second one. | | G — planning screen | **G1.** Import and plan display UI. **G2.** Authoring and suggestion cards. **G3.** Revision-bound Apply and draft/attachment preservation. **G4.** Real provider/store integration and complete T18 browser/adapter acceptance. | G1 starts after E4 and C4 merge; G1–G3 may use fixtures. G4 waits for D5 and F1's production planning API/persistence contract. Release shared web files after G4. | | H — issue prioritization | **H1.** Decide and record ranking policy. **H2.** Issue retrieval/normalization. **H3.** Deterministic ranking with reasons and failure/stale states. **H4.** Issue-list UI and end-to-end checks (build step 8). | H1–H3 can run alongside F/G after the issue-access contract is inspected. H4 waits for G4 to release shared web files. No existing T-ID covers this entire milestone. | | I — queue, schedule and recovery | **I1.** Queue admission and persisted transitions. **I2.** Run-window scheduling and cancellation. **I3.** Restart recovery, stale attempts and shutdown draining. **I4.** UI integration and controlled race acceptance (build step 7). | Starts after F6; owns shared runner/store files. UI work waits for G/H to release its exact files. No existing T-ID covers this entire milestone. | @@ -1981,7 +1981,7 @@ F, G and H ran together within these ownership boundaries. G and H are done; for Built from this review's findings. Each task comes from a specific decision above. Run with Claude Code or Codex, and tick each one as you ship it. Effort ratios assumed: features about 30x, tests about 50x, architecture about 5x. -**Status (checked against `main` on 2026-09-26, lane B0).** T1, T2, T4, T5, T10, T13 and T14 meet their Verify lines and are ticked, with evidence under each (T1, T2 and T14 re-checked 2026-09-26 after lane D merged). T9 stays open until lane F6 runs every suite in required CI. Ask calls the `agents/` boundary from `runner/question-container.ts`; it has no host adapter. The planned files `core/segments`, `core/choices` and `core/attribution` were never created. That logic lives in `core/linking.ts` (segments and ledger attribution) and `core/approvals.ts` (approvals and duplicate-segment choices). The Files lines below now name the real files. +**Status (checked against `main` on 2026-09-26, lane B0).** T1, T2, T4, T5, T10, T13 and T14 meet their Verify lines and are ticked, with evidence under each (T1, T2 and T14 re-checked 2026-09-26 after lane D merged). T9 stays open. Lane F6 (common CI) has added the workflows in `docs/implementation/ci.md`: `test` and `real-docker` run on every pull request with no path filter, and `Live probes` runs the live Claude probes nightly and by hand. T9 can be ticked when a repository administrator requires `test` and `real-docker` on `main`, both pass on the same head, and a `Live probes` run passes with the `CLAUDE_CODE_OAUTH_TOKEN` secret. Ask calls the `agents/` boundary from `runner/question-container.ts`; it has no host adapter. The planned files `core/segments`, `core/choices` and `core/attribution` were never created. That logic lives in `core/linking.ts` (segments and ledger attribution) and `core/approvals.ts` (approvals and duplicate-segment choices). The Files lines below now name the real files. These `T` IDs are requirement identifiers, not the build-order numbers. Current merge-gate work is **build step 4, increment 1 (#21)** and spans parts of T6, T7, and T12; it is unrelated to the numbering of T4. See “Build step 4: scope and progress” for the current increment and remaining milestone criteria. An increment must not mark a broader requirement complete while any of its acceptance criteria remain deferred. diff --git a/docs/implementation/agent-isolation.md b/docs/implementation/agent-isolation.md index c3858887..3760e1af 100644 --- a/docs/implementation/agent-isolation.md +++ b/docs/implementation/agent-isolation.md @@ -15,13 +15,16 @@ npm test `vitest.config.ts` finishes the ordinary test project first, then runs the Docker-backed files one at a time because they share one image tag and daemon. Use `npx vitest run --project docker` to rerun only that serial project. The `Agent isolation` -workflow runs the complete gate. The main `CI` workflow skips the Docker project so that -it can cover the ordinary project independently. +workflow's `real-docker` job runs the complete gate on every pull request. The main `CI` +workflow's `test` job skips the Docker-backed files so that it can cover the ordinary +project independently. [CI](ci.md) describes both jobs. -The live vendor probes need real credentials, so CI does not run them. To run them, -set `CODEBOOST_RUN_AUTH_PROBES=1` and `CLAUDE_CODE_OAUTH_TOKEN`. Codex has no live -probe while it is refused (#93). Do not put credentials in an issue, a pull request -or chat. +The live vendor probes need real credentials, so pull requests do not run them. The +`Live probes` workflow runs them every night and when started by hand, with the +repository secret `CLAUDE_CODE_OAUTH_TOKEN` ([CI](ci.md#live-probes)). To run them +locally, set `CODEBOOST_RUN_AUTH_PROBES=1` and `CLAUDE_CODE_OAUTH_TOKEN`. Codex has no +live probe while it is refused (#93). Do not put credentials in an issue, a pull +request or chat. ## What the gate proves @@ -407,10 +410,16 @@ against real Docker, including two reviews with different Ask owners on one daem ## Limits of this gate -- CI does not run the live vendor probes. Run them locally with credentials before - a release that changes the image, the adapters or the prompts. Codex has no live - probe while it is refused in every phase (#93). -- T9 as a whole is complete only when lane F runs every suite in required CI (F6). +- Pull requests do not run the live vendor probes. The `Live probes` workflow runs them + every night and by hand, and fails if the `CLAUDE_CODE_OAUTH_TOKEN` secret is missing. + A pull request can therefore merge before a probe shows that it broke the Claude + startup path. Start the workflow by hand, or run the probes locally, before a release + that changes the image, the adapters or the prompts. Codex has no live probe while it + is refused in every phase (#93). +- T9 as a whole is not complete yet. Lane F6 has added the workflows ([CI](ci.md)). What + remains: a repository administrator must require `test` and `real-docker` on `main`; + both jobs must pass on the same head after that; and a `Live probes` run must pass + once the secret exists. - Gitlink mounts and the pre-launch check apply to execute and fix only. In planning, questions and review, `/work` is a read-only volume, so nothing can be written beneath a gitlink there; that includes `check` attempts, which run as review. Running an item's `cmd` in a writable task filesystem is not built yet; when it is, each run must call diff --git a/docs/implementation/ci.md b/docs/implementation/ci.md new file mode 100644 index 00000000..b10e663f --- /dev/null +++ b/docs/implementation/ci.md @@ -0,0 +1,218 @@ +# Continuous integration + +This page describes which GitHub Actions workflows run which tests, and when. It +also lists the settings that only a repository administrator can make. These +settings are not stored in the repository. + +## Workflows + +| Workflow file | Job (check name) | Runs on | What it runs | +| --- | --- | --- | --- | +| `.github/workflows/ci.yml` | `test` | every pull request; pushes to `main` and `codex/**` | Type check; every top-level `test/*.test.ts` file except the Docker-backed files; the Playwright browser suite | +| `.github/workflows/agent-isolation.yml` | `real-docker` | every pull request; pushes to `main` and `codex/**` | Type check; the complete Vitest suite against real Docker | +| `.github/workflows/live-probes.yml` | `live-probes` | started by hand (`workflow_dispatch`); every night at 06:23 UTC | The four live Claude startup probes only | + +`test` and `real-docker` have no path filters. They run on every pull request, +so both can be required checks. A required check that a path filter skips +never reports, and GitHub then blocks the merge. + +A check takes its name from the job id, or from the job's `name` when it has +one, and a matrix adds its values to the name. So do not rename the `test` or +`real-docker` job, and do not give either one a `name` or a `strategy` (matrix): +the required check would then never report, and no pull request could merge. +`test/vitest-config.test.ts` fails if either job has any key other than +`runs-on`, `timeout-minutes` and `steps`, or if either workflow has another +job. It also fails if a workflow has a top-level key other than the ones it has +now (`name`, `on`, `permissions`, and `concurrency` in `live-probes.yml` +only), so no workflow can add `defaults` or `env` for every job. + +The test pins each job's steps exactly, in this order, including each step's +`run` text. It also runs the `live-probes` token check with an empty, a missing +and a set token. + +| Job | Steps | +| --- | --- | +| `test` | `actions/checkout@v4`; `actions/setup-node@v4` with Node 26.7.0 and the npm cache; `npm ci --ignore-scripts`; `npm run typecheck`; `npm test` with one `--exclude` for each Docker-backed file; `npx playwright install --with-deps chromium`; `npm run test:browser` | +| `real-docker` | `actions/checkout@v4`; `actions/setup-node@v4` with Node 26.7.0 and the npm cache; `npm ci --ignore-scripts`; `npm run typecheck`; `npm test` | +| `live-probes` | the token check; `actions/checkout@v4`; `actions/setup-node@v4` with Node 26.7.0 and the npm cache; `npm ci --ignore-scripts`; the live probe Vitest run; `node scripts/live-probes.ts check live-probes.json` | + +The `live-probes` job also may have only the keys `runs-on`, +`timeout-minutes` and `steps`, so it has no `defaults`, `env` or `strategy`. + +### `test` + +`test` runs the Vitest files that do not need Docker, and then the browser +suite (`npm run test:browser`). Its time limit is 45 minutes; its longest +recent run took 20 minutes. It excludes the Docker-backed files with +`--exclude`. The Docker-backed files are the `dockerTests` list in +`vitest.config.ts`. The excludes must equal that list: +`test/vitest-config.test.ts` fails if they differ. + +Vitest runs only the top-level `test/*.test.ts` files: the `include` of both +projects in `vitest.config.ts` is `test/*.test.ts` or the Docker-backed list, so +a test file in a subfolder of `test/` does not run in any job. +`test/vitest-config.test.ts` pins the whole Vitest config, so a change to the +`include` makes it fail. + +The browser suite runs every spec in `test/browser/`. `playwright.config.ts` +sets `forbidOnly: true`, so Playwright refuses to run when a spec has a +`test.only` or `describe.only`, locally as well as in CI. Without it, a committed +`.only` would make the job run only that test and pass. `test/vitest-config.test.ts` +pins the whole Playwright config, so it fails if `forbidOnly` is removed or a +`grep`, `testMatch`, `testIgnore` or narrower `testDir` is added. + +When a test file starts to use Docker, add it to `dockerTests` in +`vitest.config.ts`, to `DOCKER_TESTS` in `test/vitest-config.test.ts` and to the +excludes in `ci.yml`. + +`test/agent-adapter.test.ts` runs in `test`. It does not start Docker or a +vendor CLI: its cases test parsers and budgets, or refuse a request before +anything is allocated. It passes with no Docker binary on `PATH`. + +### `real-docker` + +`real-docker` runs `npm test` with no excludes. `vitest.config.ts` runs the +ordinary files first. Then it runs the Docker-backed files one at a time, +because they share one image tag and one Docker daemon. `test/vitest-config.test.ts` +pins the whole of `vitest.config.ts`, so it fails if either project gets any +other key, such as an `exclude`, `testNamePattern` or `passWithNoTests` that +could drop files or tests from both jobs. It also pins the `test`, `typecheck` +and `test:browser` scripts in `package.json` that the jobs call, so a filter +added there fails it too. This job is the T9 +Docker gate. [Agent isolation](agent-isolation.md#what-the-gate-proves) lists +what it proves. + +### Recorded outputs and the hostile-issue eval + +These files are ordinary Vitest files, so both `test` and `real-docker` run +them. They do not call a vendor or GitHub. + +- Recorded Claude planning output: `test/fixtures/planning/recorded/`, replayed + by `test/planning-recorded.test.ts`. +- GitHub (`gh`) responses: fixtures inside `test/merge.test.ts` and the other + merge and publishing tests. +- Hostile issue texts: `test/fixtures/hostile-issues.json`, used by + `test/execution-prompt.test.ts`. + +### `live-probes` + +The live probes call the real Claude service. They are not a pull request +check, and they do not run on a pull request or a push. GitHub runs scheduled +workflows only from the default branch, so the nightly run starts after this +workflow is on `main`. The repository is public, so GitHub disables this +scheduled workflow after 60 days without repository activity. To turn it on +again, open the Actions tab, select the `Live probes` workflow and select +**Enable workflow**. + +The job: + +1. Fails at once if the repository secret `CLAUDE_CODE_OAUTH_TOKEN` is empty or + missing. A missing secret never gives a passing run. +2. Runs the Docker Vitest project with `CODEBOOST_RUN_AUTH_PROBES=1`. It selects + only the tests whose titles end in `(live Claude)`, in + `test/agent-container.test.ts`, `test/agent-question.test.ts` and + `test/agent-supervisor.test.ts`. The files run one at a time. +3. Runs `node scripts/live-probes.ts check` on the Vitest JSON report. Vitest + exits with success when it skips every test, so this step fails unless each + probe listed in `scripts/live-probes.ts` passed and no other test ran. + +`test/vitest-config.test.ts` fails if the `live-probes` job, or the `test` job +in `ci.yml` or the `real-docker` job in `agent-isolation.yml`, or any step of +these jobs, has an `if` condition or `continue-on-error`. Either one could skip +a step or let it fail while the job passes. + +If you add, rename or remove a probe, or rename a `describe` block around one, +update `LIVE_PROBES` in `scripts/live-probes.ts`. `test/vitest-config.test.ts` +parses each probe file. It fails unless the tests that +`CODEBOOST_RUN_AUTH_PROBES` enables have exactly the full names in the list: the +`describe` titles and the test title, joined with spaces, as Vitest names them. +Every `it`, `test` and `describe` call in a probe file must have a string +literal or a template literal without expressions as its title, or the test +fails. The call must reach `it`, `test` or `describe` through plain calls and +plain property names only, such as `it.skip(...)` or `it.each([...])(...)`. +Any other form, such as ``it.each`table`(...)``, `it['skip'](...)` or +`(it as T)(...)`, makes the test fail. A call through another name, such as +`const t = it; t(...)`, is not checked. So the test sees the title of every +such call, and fails if a test that the switch does not enable has a literal +title that ends with the marker. The test reads a `.each` or `.for` title before +Vitest fills its placeholders, so it does not see a marker that a placeholder +adds, as in `it.each(['(live Claude)'])('sneaky %s', ...)`. The `-t` filter of +the `live-probes` job selects such a test, and the report check then fails the +job, because a test that is not a probe ran. Each `describe` around a +gated test, and each gated `it` or `test` call, must have exactly a literal +title, a function expression and optionally a numeric timeout. +`describe.each`, `describe.for`, `describe.skip` or any other modifier, or an +options object such as `{ skip: true }`, makes the test fail. +The switch may only be the condition of an `if` block without `else`, or of +`it.runIf`. Each statement in such an `if` block must be a plain `it` or `test` +call, or a `const` or `let` declaration of plain names. Each +name must have a plain value: a function expression, or a constant. A constant +is a string, number or boolean literal, a template literal whose expressions are +constants, or a binary operation on constants. Any other value, such as a call +or an assignment like `(on = true)`, makes the test fail, and so does a `var`, +because it is visible outside the block. Such a gate must be where Vitest always +registers tests as it loads the file: a statement directly in the body of the +file, or directly in the body of a `describe` callback that is itself such a +statement. A gate inside any other block or expression, such as another `if`, a +loop or a helper that is never called, makes the test fail. The test counts every occurrence of the name +`CODEBOOST_RUN_AUTH_PROBES` in each probe file's text, comments and strings +included, and fails unless each one is such a gate. It also fails if any other +source file under `test/` names the switch: any `.ts`, `.tsx`, `.mts`, `.cts`, +`.js`, `.mjs` or `.cjs` file, helpers and fixtures included, except +`test/vitest-config.test.ts` itself. It +cannot catch a variable name built at runtime from pieces, such as +`process.env['CODEBOOST_' + 'RUN_AUTH_PROBES']`. + +Codex has no probe, because Codex is refused in every phase (#93). + +## Settings for a repository administrator + +The repository cannot make these settings itself. + +1. **Required checks on `main`.** In the branch protection rule or ruleset for + `main`, require the status checks `test` and `real-docker`. Do not require + `live-probes`. +2. **Live probe secret.** Add the repository secret `CLAUDE_CODE_OAUTH_TOKEN` + (Settings, then Secrets and variables, then Actions). Make the token with + `claude setup-token`. Until the secret exists, every `live-probes` run fails + at its first step. + +Never put a token in an issue, a pull request, a commit or chat. + +## Record the planning fixtures again + +`test/planning-recorded.test.ts` fails when a recording no longer matches +today's request: for example, when the prompt, the schema, the Claude command, +the image tag or the CLI version changes. To record again, run this on a +machine with Docker: + +```sh +CODEBOOST_RUN_AUTH_PROBES=1 CLAUDE_CODE_OAUTH_TOKEN=... node scripts/record-planning.ts +``` + +Read each file in `test/fixtures/planning/recorded/` before you commit it. No +workflow records fixtures. [Planning acceptance](planning-acceptance.md#recorded-vendor-output) +describes the recordings. + +## Run the same checks locally + +```sh +npm run typecheck +npm test # all Vitest files; needs Docker +npx vitest run --project parallel # only the files that do not need Docker +npx vitest run --project docker # only the Docker-backed files +npm run test:browser +``` + +To run the live probes locally, set `CODEBOOST_RUN_AUTH_PROBES=1` and +`CLAUDE_CODE_OAUTH_TOKEN`, then run the `vitest` command from +`live-probes.yml`. + +## What remains before T9 is complete + +T9 in the [plan](../designs/codeboost-plan-indexed-review.md) stays open until +all of these are true: + +1. A repository administrator requires `test` and `real-docker` on `main`. +2. Both required jobs pass on the same head after this setup is merged. +3. The `CLAUDE_CODE_OAUTH_TOKEN` secret exists and a `live-probes` run passes. diff --git a/docs/implementation/planning-acceptance.md b/docs/implementation/planning-acceptance.md index 3f23eb24..e177b4f1 100644 --- a/docs/implementation/planning-acceptance.md +++ b/docs/implementation/planning-acceptance.md @@ -86,5 +86,11 @@ The Store guarantees revision/snapshot binding and pending-only settlement acros processes, and E3 requires the caller's snapshot identity before admission. G4 now composes the browser with production planning setup, lane D's provider boundary, F's HTTP lifecycle and Store persistence in `plans-production.spec.ts`. The exact-head -real-Docker planning gate also passes, completing T18. T9 remains incomplete until -its remaining F6 common-CI and all-suite gate passes. +real-Docker planning gate also passes, completing T18. + +T9 is still incomplete. F6 runs the recordings in both required jobs, `test` and +`real-docker`, and the live Claude probes in the nightly `Live probes` workflow +([CI](ci.md)). No workflow records fixtures: when a recording goes stale, run +`scripts/record-planning.ts` again as above. T9 can be ticked when a repository +administrator requires `test` and `real-docker` on `main`, both pass on the same head, +and a `Live probes` run passes with the `CLAUDE_CODE_OAUTH_TOKEN` secret. diff --git a/package-lock.json b/package-lock.json index fde54e92..9884dd3e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "@playwright/test": "1.63.0", "@types/node": "26.6.2", "typescript": "7.0.2", + "vite": "8.3.0", "vitest": "5.0.1" }, "engines": { @@ -77,7 +78,6 @@ "integrity": "sha512-rDS5/31E9HfPl/CIzGrn0DOlvBbXFseQ5URJ9sYMfstbKLD/c6Gm9vmRzRGDdAXyOIL4zmO37lc9RIwYqVruZw==", "dev": true, "license": "MIT", - "peer": true, "funding": { "url": "https://github.com/sponsors/oxc-project" } @@ -111,7 +111,6 @@ "os": [ "android" ], - "peer": true, "engines": { "node": "^20.19.0 || >=22.12.0" } @@ -129,7 +128,6 @@ "os": [ "android" ], - "peer": true, "engines": { "node": "^20.19.0 || >=22.12.0" } @@ -147,7 +145,6 @@ "os": [ "darwin" ], - "peer": true, "engines": { "node": "^20.19.0 || >=22.12.0" } @@ -165,7 +162,6 @@ "os": [ "darwin" ], - "peer": true, "engines": { "node": "^20.19.0 || >=22.12.0" } @@ -183,7 +179,6 @@ "os": [ "freebsd" ], - "peer": true, "engines": { "node": "^20.19.0 || >=22.12.0" } @@ -201,7 +196,6 @@ "os": [ "linux" ], - "peer": true, "engines": { "node": "^20.19.0 || >=22.12.0" } @@ -222,7 +216,6 @@ "os": [ "linux" ], - "peer": true, "engines": { "node": "^20.19.0 || >=22.12.0" } @@ -243,7 +236,6 @@ "os": [ "linux" ], - "peer": true, "engines": { "node": "^20.19.0 || >=22.12.0" } @@ -264,7 +256,6 @@ "os": [ "linux" ], - "peer": true, "engines": { "node": "^20.19.0 || >=22.12.0" } @@ -285,7 +276,6 @@ "os": [ "linux" ], - "peer": true, "engines": { "node": "^20.19.0 || >=22.12.0" } @@ -306,7 +296,6 @@ "os": [ "linux" ], - "peer": true, "engines": { "node": "^20.19.0 || >=22.12.0" } @@ -327,7 +316,6 @@ "os": [ "linux" ], - "peer": true, "engines": { "node": "^20.19.0 || >=22.12.0" } @@ -345,7 +333,6 @@ "os": [ "openharmony" ], - "peer": true, "engines": { "node": "^20.19.0 || >=22.12.0" } @@ -363,7 +350,6 @@ "os": [ "win32" ], - "peer": true, "engines": { "node": "^20.19.0 || >=22.12.0" } @@ -381,7 +367,6 @@ "os": [ "win32" ], - "peer": true, "engines": { "node": "^20.19.0 || >=22.12.0" } @@ -391,8 +376,7 @@ "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@types/chai": { "version": "5.2.3", @@ -849,7 +833,6 @@ "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", "dev": true, "license": "Apache-2.0", - "peer": true, "engines": { "node": ">=8" } @@ -941,7 +924,6 @@ "os": [ "darwin" ], - "peer": true, "engines": { "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } @@ -970,7 +952,6 @@ "integrity": "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA==", "dev": true, "license": "MPL-2.0", - "peer": true, "dependencies": { "detect-libc": "^2.0.3" }, @@ -1008,7 +989,6 @@ "os": [ "android" ], - "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -1030,7 +1010,6 @@ "os": [ "darwin" ], - "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -1052,7 +1031,6 @@ "os": [ "darwin" ], - "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -1074,7 +1052,6 @@ "os": [ "freebsd" ], - "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -1096,7 +1073,6 @@ "os": [ "linux" ], - "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -1121,7 +1097,6 @@ "os": [ "linux" ], - "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -1146,7 +1121,6 @@ "os": [ "linux" ], - "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -1171,7 +1145,6 @@ "os": [ "linux" ], - "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -1196,7 +1169,6 @@ "os": [ "linux" ], - "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -1218,7 +1190,6 @@ "os": [ "win32" ], - "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -1240,7 +1211,6 @@ "os": [ "win32" ], - "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -1271,7 +1241,6 @@ } ], "license": "MIT", - "peer": true, "bin": { "nanoid": "bin/nanoid.cjs" }, @@ -1298,8 +1267,7 @@ "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", "dev": true, - "license": "ISC", - "peer": true + "license": "ISC" }, "node_modules/picomatch": { "version": "4.0.7", @@ -1363,7 +1331,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "nanoid": "^3.3.18", "picocolors": "^1.1.1", @@ -1388,7 +1355,6 @@ "integrity": "sha512-hx/Pv0N1haXRb11qkfnK5MXB/iqr7i0yjWQqmO9uHqZpBgQSqzc8UsSnEpalsh+j1I8qQ2CkXAkJC8Br3dKSlg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@oxc-project/types": "=0.150.0", "@rolldown/pluginutils": "^1.0.0" @@ -1430,7 +1396,6 @@ "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", "dev": true, "license": "BSD-3-Clause", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -1534,7 +1499,6 @@ "integrity": "sha512-lhZBVvEHefgE+HQZC9O7EBJgCU/nVzFNl7vkS4RE0APtWLP02/8QVIkQtzBxPquh7lq5/78NHipTj7ODQ6XuyQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "lightningcss": "^1.33.0", "picomatch": "^4.0.7", diff --git a/package.json b/package.json index b1691203..28588ea9 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "@playwright/test": "1.63.0", "@types/node": "26.6.2", "typescript": "7.0.2", + "vite": "8.3.0", "vitest": "5.0.1" } } diff --git a/playwright.config.ts b/playwright.config.ts index 08f130b7..b0132b41 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -1,4 +1,6 @@ import { defineConfig } from '@playwright/test'; // The review server runs in the test process. Its review loads no longer block that process, so test and assertion // timeouts now include the server's real Git work (a merge click runs several review loads); allow for it. -export default defineConfig({ testDir: './test/browser', fullyParallel: false, workers: 1, timeout: 60_000, expect: { timeout: 15_000 }, use: { browserName:'chromium', viewport:{width:1512,height:982}, trace:'retain-on-failure' } }); +// `forbidOnly` makes Playwright refuse a `test.only` (or `describe.only`) left in a spec, which would otherwise run +// only that test and let the CI `test` job pass. `test/vitest-config.test.ts` pins this whole config. +export default defineConfig({ testDir: './test/browser', forbidOnly: true, fullyParallel: false, workers: 1, timeout: 60_000, expect: { timeout: 15_000 }, use: { browserName:'chromium', viewport:{width:1512,height:982}, trace:'retain-on-failure' } }); diff --git a/scripts/live-probes.ts b/scripts/live-probes.ts new file mode 100644 index 00000000..9547a76a --- /dev/null +++ b/scripts/live-probes.ts @@ -0,0 +1,68 @@ +// The live Claude startup probes, and the check that the Live probes workflow ran all of them. +// +// node scripts/live-probes.ts check +// +// The probes call the real Claude service, so they run only when CODEBOOST_RUN_AUTH_PROBES=1 and +// CLAUDE_CODE_OAUTH_TOKEN are set. Without those, Vitest skips them and still exits 0. This check reads the +// Vitest JSON report and exits 1 unless every probe below passed and nothing else ran. A renamed, removed or +// skipped probe therefore fails the workflow instead of passing it silently. +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +// Every probe title ends with a space and this suffix. The workflow's `-t` filter is this text escaped as a regular +// expression and anchored at the end; test/vitest-config.test.ts fails if the two differ. +export const LIVE_PROBE_MARKER = '(live Claude)'; + +export const LIVE_PROBES = [ + { file: 'test/agent-container.test.ts', + name: 'real Docker agent isolation runs the authenticated Claude startup path with only its OAuth token (live Claude)' }, + { file: 'test/agent-question.test.ts', + name: 'Ask in the agent container answers from a file it can only read in /work (live Claude)' }, + { file: 'test/agent-supervisor.test.ts', + name: 'container invocation supervisor runs the production Claude adapter and parses its bounded envelope (live Claude)' }, + { file: 'test/agent-supervisor.test.ts', + name: 'container invocation supervisor returns a schema-constrained Claude planning answer as bare JSON from the ' + + 'mounted schema (live Claude)' }, +] as const; + +export const LIVE_PROBE_FILES = [...new Set(LIVE_PROBES.map(probe => probe.file))]; + +type Report = { + success?: unknown; numFailedTests?: unknown; + testResults?: Array<{ name?: unknown; assertionResults?: Array<{ fullName?: unknown; status?: unknown }> }>; +}; + +// Returns one line per problem; an empty list means every probe passed and nothing else ran. +export function checkLiveProbeReport(report: Report, root = process.cwd()): string[] { + const problems: string[] = []; + if (report.success !== true) problems.push('Vitest did not report success.'); + if (report.numFailedTests !== 0) problems.push(`Vitest reported ${String(report.numFailedTests)} failed test(s).`); + const passed = new Set(); + for (const file of report.testResults ?? []) { + const name = typeof file.name === 'string' ? file.name : ''; + for (const test of file.assertionResults ?? []) { + if (test.status !== 'passed') continue; + const key = `${name} > ${String(test.fullName)}`; + passed.add(key); + if (!LIVE_PROBES.some(probe => `${resolve(root, probe.file)} > ${probe.name}` === key)) + problems.push(`A test that is not a live probe ran: ${key}`); + } + } + for (const probe of LIVE_PROBES) + if (!passed.has(`${resolve(root, probe.file)} > ${probe.name}`)) + problems.push(`Live probe did not pass: ${probe.file} > ${probe.name}`); + return problems; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + const [command, path] = process.argv.slice(2); + if (command !== 'check' || !path) { + console.error('Usage: node scripts/live-probes.ts check '); + process.exit(2); + } + const problems = checkLiveProbeReport(JSON.parse(readFileSync(path, 'utf8')) as Report); + for (const problem of problems) console.error(problem); + if (problems.length) process.exit(1); + console.log(`All ${LIVE_PROBES.length} live Claude probes passed.`); +} diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 48d809ed..7b6ef613 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -2590,7 +2590,7 @@ describe('real Docker agent isolation', () => { }, 60_000); if (process.env.CODEBOOST_RUN_AUTH_PROBES === '1') { - it('runs the authenticated Claude startup path with only its OAuth token', async () => { + it('runs the authenticated Claude startup path with only its OAuth token (live Claude)', async () => { const data = fixture(), token = process.env.CLAUDE_CODE_OAUTH_TOKEN; if (!token) throw new Error('CLAUDE_CODE_OAUTH_TOKEN is required.'); // Questions answer in plain text; a planning command would also carry the schema as --json-schema. diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 7c5cc5c4..7d1bf7b4 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -641,7 +641,7 @@ describe('container invocation supervisor', () => { // formatting fails the probe. const schemaValue = (output: string) => output.replace(/\r?\n$/, ''); - it('runs the production Claude adapter and parses its bounded envelope', async () => { + it('runs the production Claude adapter and parses its bounded envelope (live Claude)', async () => { const data = fixture(), token = process.env.CLAUDE_CODE_OAUTH_TOKEN; if (!token) throw new Error('CLAUDE_CODE_OAUTH_TOKEN is required.'); // Questions answer in plain text; planning answers are schema-constrained (the next probe). @@ -655,7 +655,7 @@ describe('container invocation supervisor', () => { expect(schemaValue(result.stdout)).toBe('codeboost-adapter-schema-marker'); }, 8 * 60_000); - it('returns a schema-constrained Claude planning answer as bare JSON from the mounted schema', async () => { + it('returns a schema-constrained Claude planning answer as bare JSON from the mounted schema (live Claude)', async () => { // `const` makes the schema itself enforce the marker. Claude can also read the mounted file, so the marker alone // does not prove the flag: the strict structured_output decode does, since without --json-schema there is none. const schema = JSON.stringify({ type: 'object', additionalProperties: false, required: ['marker'], diff --git a/test/vitest-config.test.ts b/test/vitest-config.test.ts index d1851b4e..d010c604 100644 --- a/test/vitest-config.test.ts +++ b/test/vitest-config.test.ts @@ -1,6 +1,16 @@ +import { spawnSync } from 'node:child_process'; +import { readdirSync, readFileSync } from 'node:fs'; +import { join, sep } from 'node:path'; +import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; +import { parseSync } from 'vite'; +import { parse } from 'yaml'; +import { checkLiveProbeReport, LIVE_PROBE_FILES, LIVE_PROBE_MARKER, LIVE_PROBES } from '../scripts/live-probes.ts'; +import playwrightConfig from '../playwright.config.ts'; import config from '../vitest.config.ts'; +const ROOT = fileURLToPath(new URL('..', import.meta.url)); + const DOCKER_TESTS = [ 'test/agent-container.test.ts', 'test/agent-gate.test.ts', @@ -13,19 +23,401 @@ const DOCKER_TESTS = [ describe('Vitest scheduling', () => { it('finishes ordinary tests before running Docker-backed files one at a time', () => { - const projects = (config as { test?: { projects?: Array<{ test?: Record }> } }).test?.projects; - expect(projects).toHaveLength(2); - expect(projects?.[0]?.test).toMatchObject({ - name: 'parallel', - include: ['test/*.test.ts'], - exclude: DOCKER_TESTS, - sequence: { groupOrder: 0 }, - }); - expect(projects?.[1]?.test).toMatchObject({ - name: 'docker', - include: DOCKER_TESTS, - fileParallelism: false, - sequence: { groupOrder: 1 }, + // The whole config, exactly: any other key, such as an `exclude`, `testNamePattern` or `passWithNoTests` in a + // project, could drop files or tests from both CI jobs while every job still passes. + expect(config).toEqual({ + test: { + projects: [ + { test: { name: 'parallel', include: ['test/*.test.ts'], exclude: DOCKER_TESTS, sequence: { groupOrder: 0 } } }, + { test: { name: 'docker', include: DOCKER_TESTS, fileParallelism: false, sequence: { groupOrder: 1 } } }, + ], + }, }); }); }); + +describe('Playwright config', () => { + it('runs every browser spec and refuses a committed test.only', () => { + // The whole config, exactly: a `grep`, `testMatch`, `testIgnore` or narrower `testDir` could drop specs from the + // CI test job while it still passes, and without `forbidOnly` a committed `test.only` would run only that test. + // defineConfig adds one symbol-keyed marker; Playwright reads only string keys. + expect(Object.getOwnPropertySymbols(playwrightConfig).map(String)).toEqual(['Symbol(defineConfigWasUsed)']); + expect(Object.fromEntries(Object.entries(playwrightConfig))).toEqual({ testDir: './test/browser', forbidOnly: true, fullyParallel: false, workers: 1, + timeout: 60_000, expect: { timeout: 15_000 }, + use: { browserName: 'chromium', viewport: { width: 1512, height: 982 }, trace: 'retain-on-failure' } }); + }); +}); + +const workflow = (name: string) => + parse(readFileSync(join(ROOT, '.github/workflows', name), 'utf8')) as { + on: Record; + permissions: unknown; + jobs: Record & { env?: unknown; + steps: Array & { name?: string; run?: string; env?: Record }> }>; + }; +// The `-t` pattern that selects the titles ending in the marker. +const MARKER_FILTER = `${LIVE_PROBE_MARKER.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`; +const NODE_VERSION = '26.7.0'; +const SETUP = [{ uses: 'actions/checkout@v4' }, + { uses: 'actions/setup-node@v4', with: { 'node-version': NODE_VERSION, cache: 'npm' } }]; +const runs = (steps: Array<{ run?: string }>) => steps.flatMap(step => step.run === undefined ? [] : [step.run]); + +type Node = { type: string; [key: string]: unknown }; +type ProbeFileTest = { name: string; gated: boolean }; +// An enclosing `describe`: its literal title, or why its title is not known. +type Suite = { title: string } | { problem: string }; +const PROBE_SWITCH = 'CODEBOOST_RUN_AUTH_PROBES'; +const SOURCE = /\.(?:ts|tsx|mts|cts|js|mjs|cjs)$/; +const isNode = (value: unknown): value is Node => + typeof value === 'object' && value !== null && typeof (value as { type?: unknown }).type === 'string'; +const FUNCTIONS = ['ArrowFunctionExpression', 'FunctionExpression', 'FunctionDeclaration']; +const children = (node: Node) => Object.values(node).flatMap(value => Array.isArray(value) ? value : [value]).filter(isNode); +const isIdentifier = (node: unknown, name: string) => isNode(node) && node.type === 'Identifier' && node.name === name; +const isMember = (node: unknown, object: (node: unknown) => boolean, property: string): boolean => + isNode(node) && node.type === 'MemberExpression' && node.computed === false && object(node.object) + && isIdentifier(node.property, property); +// Exactly `process.env.CODEBOOST_RUN_AUTH_PROBES === '1'`. +const isSwitchOn = (node: unknown) => isNode(node) && node.type === 'BinaryExpression' && node.operator === '===' + && isMember(node.left, env => isMember(env, process => isIdentifier(process, 'process'), 'env'), PROBE_SWITCH) + && isNode(node.right) && node.right.type === 'Literal' && node.right.value === '1'; +const literalText = (node: unknown) => !isNode(node) ? undefined + : node.type === 'Literal' && typeof node.value === 'string' ? node.value + : node.type === 'TemplateLiteral' && (node.expressions as unknown[]).length === 0 + ? (node.quasis as Array<{ value: { cooked: string } }>)[0]!.value.cooked : undefined; +// The switch as an identifier, a string literal or a template literal without expressions. +const isSwitchName = (node: Node) => (node.type === 'Identifier' && node.name === PROBE_SWITCH) || literalText(node) === PROBE_SWITCH; +const mentionsSwitch = (node: Node): boolean => isSwitchName(node) || children(node).some(mentionsSwitch); +// Links that can lead from a callee to `it`, `test` or `describe` but that `suiteCall` does not model, such as the tag +// in ``it.each`table`(...)``, a computed member such as `it['skip']`, a non-null assertion or parentheses. +const UNMODELED_LINKS: Record = { TaggedTemplateExpression: 'tag', MemberExpression: 'object', + CallExpression: 'callee', ChainExpression: 'expression', ParenthesizedExpression: 'expression', + TSNonNullExpression: 'expression', TSAsExpression: 'expression', TSSatisfiesExpression: 'expression', + TSTypeAssertion: 'expression', TSInstantiationExpression: 'expression' }; +// A call whose callee chain starts at `it`, `test` or `describe` through plain calls and non-computed members only, such +// as `it.runIf(...)('title', ...)` or `it.each([...])('title', ...)`: the kind, the methods and arguments of the inner +// calls, and the outer call's arguments. Throws if the chain reaches `it`, `test` or `describe` through any other link +// (see `UNMODELED_LINKS`), because the caller could not check that call's title. +function suiteCall(node: Node, file: string) { + if (node.type !== 'CallExpression') return undefined; + const methods: string[] = [], innerArgs: Node[][] = []; + let callee = node.callee as Node; + for (;;) { + if (callee.type === 'CallExpression') { innerArgs.unshift((callee.arguments as Node[])); callee = callee.callee as Node; } + else if (callee.type === 'MemberExpression' && callee.computed === false && isNode(callee.property) + && callee.property.type === 'Identifier') { methods.unshift(callee.property.name as string); callee = callee.object as Node; } + else break; + } + if (callee.type !== 'Identifier' || !['it', 'test', 'describe'].includes(callee.name as string)) { + for (let link: unknown = callee; isNode(link); link = link[UNMODELED_LINKS[link.type] ?? '']) { + if (link.type === 'Identifier' && ['it', 'test', 'describe'].includes(link.name as string)) + throw new Error(`A call that reaches ${String(link.name)} through ${callee.type} at offset ${String(node.start)} in ${file}.`); + } + return undefined; + } + return { kind: callee.name as string, methods, innerArgs, args: node.arguments as Node[] }; +} +// A value that a gated block may give a variable: a function expression, which runs only when called, or a constant: a +// string, number or boolean literal, a template literal whose expressions are constants, or a binary operation on +// constants. Nothing else is allowed: a call could register a test this check does not see, and an assignment or +// update, such as `(on = true)`, could turn on a test outside the gate. +const isConstant = (node: Node): boolean => (node.type === 'Literal' && ['string', 'number', 'boolean'].includes(typeof node.value)) + || (node.type === 'TemplateLiteral' && (node.expressions as Node[]).every(isConstant)) + || (node.type === 'BinaryExpression' && isConstant(node.left as Node) && isConstant(node.right as Node)); +const isPlainValue = (node: Node) => ['ArrowFunctionExpression', 'FunctionExpression'].includes(node.type) || isConstant(node); +// A timeout: a number literal, or a binary operation on numbers, such as `22 * 60_000`. +const isNumber = (node: Node): boolean => (node.type === 'Literal' && typeof node.value === 'number') + || (node.type === 'BinaryExpression' && isNumber(node.left as Node) && isNumber(node.right as Node)); +// Exactly (literal title, function expression[, timeout]). An options object, such as `{ skip: true }`, could skip or +// change the test, so it is refused. +const isPlainSuiteArgs = (args: Node[]) => (args.length === 2 || (args.length === 3 && isNumber(args[2]!))) + && literalText(args[0]) !== undefined && ['ArrowFunctionExpression', 'FunctionExpression'].includes(args[1]!.type); +// Throws unless the statement is a `const` or `let` declaration of plain names, each with a plain value. A `var` is +// refused because it is visible outside the block. +function assertPlainDeclaration(statement: Node, file: string) { + const declarations = statement.declarations as Array<{ id: Node; init: Node | null }>; + if (!['const', 'let'].includes(statement.kind as string) || !declarations.every(declaration => + declaration.id.type === 'Identifier' && declaration.init !== null && isPlainValue(declaration.init))) + throw new Error(`A gated block declares a variable that is not a const or let with a plain value at offset ${String(statement.start)} in ${file}.`); +} + +// Every test in a probe file with its Vitest full name (describe titles and the test title, joined with spaces), and +// whether CODEBOOST_RUN_AUTH_PROBES enables it. Every call whose callee chain reaches `it`, `test` or `describe` must +// reach it through plain calls and non-computed members only, such as `it.skip(...)` or `it.each([...])(...)`, or this +// throws: ``it.each`table`(...)``, `it['skip'](...)` and `(it as T)(...)` throw. Each such call must have a string +// literal or a template literal without expressions as its title, or this throws, so that the caller can check every +// title for the marker. A call through an alias, such as `const t = it; t(...)`, is not seen. Every `describe` around a gated test must be a plain +// `describe(...)` with exactly a literal title, a function expression and optionally a numeric timeout, or this +// throws: a modifier such as `describe.each`, `.for` or `.skip`, or an options object such as `{ skip: true }`, could +// skip the test or make its full name differ from the one this computes. A gated test call must have the same +// arguments. A gate counts only where Vitest always +// registers its tests while it loads the file: as a statement directly in the module body or directly in the body of a +// plain `describe` callback that is itself such a statement. A gate inside any other block or expression, such as +// another `if`, a loop or a helper that is never called, throws. The switch may appear only as the exact condition of an +// `if (...) { ... }` block without `else`, whose statements are each a plain `it`/`test` call or a +// `const` or `let` declaration of plain names with plain values (see `isPlainValue`), or of +// `it.runIf(...)`/`test.runIf(...)`. Every occurrence of the switch's name in the source text, +// comments and strings included, must be one of these gates, or this throws. A variable name built at runtime from +// pieces, such as `process.env['CODEBOOST_' + 'RUN_AUTH_PROBES']`, is not caught. +function probeFileTests(file: string, source: string): ProbeFileTest[] { + const { program, errors } = parseSync(file, source); + if (errors.length) throw new Error(`${file} does not parse: ${errors[0]!.message}`); + const tests: ProbeFileTest[] = []; + let gates = 0; + const record = (describes: Suite[], args: Node[], gated: boolean, offset: unknown) => { + const title = literalText(args[0]); + if (title === undefined) throw new Error(`A test without a literal title at offset ${String(offset)} in ${file}.`); + if (gated && !isPlainSuiteArgs(args)) + throw new Error(`A gated test whose arguments are not a literal title, a function and a timeout at offset ${String(offset)} in ${file}.`); + const unknown = describes.find(suite => 'problem' in suite); + if (gated && unknown) throw new Error(`Gated test in ${file} inside a describe ${unknown.problem}.`); + const titles = describes.map(suite => 'title' in suite ? suite.title : '?'); + tests.push({ name: [...titles, title].join(' '), gated }); + }; + // `loading` is true only for a statement that always runs as Vitest loads the file, and for its expression: a statement + // directly in the module body or directly in the body of a `describe` callback that is itself such a statement. + const visit = (node: Node, describes: Suite[], loading: boolean): void => { + if (FUNCTIONS.includes(node.type)) { children(node).forEach(child => visit(child, describes, false)); return; } + if (node.type === 'IfStatement' && mentionsSwitch(node.test as Node)) { + const block = node.consequent as Node; + if (!loading) + throw new Error(`A ${PROBE_SWITCH} gate that does not always run as Vitest loads the file at offset ${String(node.start)} in ${file}.`); + if (!isSwitchOn(node.test) || block.type !== 'BlockStatement' || node.alternate !== null) + throw new Error(`Unrecognized ${PROBE_SWITCH} condition at offset ${String(node.start)} in ${file}.`); + gates += 1; + for (const statement of block.body as Node[]) { + const call = statement.type === 'ExpressionStatement' ? suiteCall(statement.expression as Node, file) : undefined; + if (call && call.kind !== 'describe' && call.methods.length === 0) { + record(describes, call.args, true, statement.start); + call.args.slice(1).forEach(arg => visit(arg, describes, false)); + } else if (statement.type === 'VariableDeclaration') { + assertPlainDeclaration(statement, file); + visit(statement, describes, false); + } else throw new Error(`A gated block holds a statement that is not a test at offset ${String(statement.start)} in ${file}.`); + } + return; + } + const call = suiteCall(node, file); + if (call) { + const gate = call.kind !== 'describe' && call.methods.length === 1 && call.methods[0] === 'runIf' + && call.innerArgs.length === 1 && call.innerArgs[0]!.length === 1 && isSwitchOn(call.innerArgs[0]![0]); + if (gate && !loading) + throw new Error(`A ${PROBE_SWITCH} gate that does not always run as Vitest loads the file at offset ${String(node.start)} in ${file}.`); + if (gate) gates += 1; + else call.innerArgs.flat().forEach(arg => visit(arg, describes, false)); + if (call.kind === 'describe') { + const title = literalText(call.args[0]); + if (title === undefined) throw new Error(`A describe without a literal title at offset ${String(node.start)} in ${file}.`); + const suite: Suite = call.methods.length > 0 + ? { problem: `with the modifier .${call.methods.join('.')} at offset ${String(node.start)}` } + : !isPlainSuiteArgs(call.args) + ? { problem: `whose arguments are not a literal title, a function and a timeout at offset ${String(node.start)}` } + : { title }; + const inner = [...describes, suite]; + // Vitest calls a describe callback while it loads the file, so the statements directly in the callback's body keep + // `loading`. Everything else in the arguments does not. + for (const arg of call.args) { + const body = arg.body as Node | undefined; + if (!FUNCTIONS.includes(arg.type) || body?.type !== 'BlockStatement') { visit(arg, inner, false); continue; } + (arg.params as Node[]).forEach(param => visit(param, inner, false)); + (body.body as Node[]).forEach(statement => visit(statement, inner, loading)); + } + return; + } + record(describes, call.args, gate, node.start); + call.args.forEach(arg => visit(arg, describes, false)); + return; + } + if (isSwitchName(node)) + throw new Error(`Unrecognized use of ${PROBE_SWITCH} at offset ${String(node.start)} in ${file}.`); + // Only the module body and an expression statement pass `loading` on to their direct children. + const keep = loading && (node.type === 'Program' || node.type === 'ExpressionStatement'); + children(node).forEach(child => visit(child, describes, keep)); + }; + visit(program as unknown as Node, [], true); + const occurrences = source.split(PROBE_SWITCH).length - 1; + if (occurrences !== gates) + throw new Error(`${file} names ${PROBE_SWITCH} ${String(occurrences)} time(s) but has ${String(gates)} recognized gate(s).`); + return tests; +} + +describe('CI workflows', () => { + it('excludes exactly the Docker-backed files from the CI test job', () => { + const ci = workflow('ci.yml'); + const commands = runs(ci.jobs.test!.steps).filter(run => run.startsWith('npm test')); + expect(commands).toHaveLength(1); + // `npm test --` and then only `--exclude ` pairs: any other argument, such as a file filter, would narrow + // the job. + const [npm, test, separator, ...rest] = commands[0]!.trim().split(/\s+/); + expect([npm, test, separator]).toEqual(['npm', 'test', '--']); + expect(rest.length % 2).toBe(0); + expect(rest.filter((_, index) => index % 2 === 0).every(word => word === '--exclude')).toBe(true); + expect(rest.filter((_, index) => index % 2 === 1).toSorted()).toEqual(DOCKER_TESTS); + }); + + it('runs the complete Vitest suite in real-docker on every pull request and on the CI push branches', () => { + const ci = workflow('ci.yml'), isolation = workflow('agent-isolation.yml'); + expect(Object.keys(isolation.on).toSorted()).toEqual(['pull_request', 'push']); + // No path filter: a required check that a filter skips never reports, and the pull request cannot merge. + expect(isolation.on.pull_request).toBeNull(); + expect(ci.on.pull_request).toBeNull(); + expect(isolation.on.push).toEqual(ci.on.push); + expect(isolation.on.push).toEqual({ branches: ['main', 'codex/**'] }); + expect(isolation.jobs['real-docker']!.steps).toEqual( + [...SETUP, { run: 'npm ci --ignore-scripts' }, { run: 'npm run typecheck' }, { run: 'npm test' }]); + }); + + it('runs exactly the type check, the non-Docker Vitest files and the browser suite in the CI test job', () => { + expect(workflow('ci.yml').jobs.test!.steps).toEqual([...SETUP, { run: 'npm ci --ignore-scripts' }, { run: 'npm run typecheck' }, + { run: `npm test -- ${DOCKER_TESTS.map(file => `--exclude ${file}`).join(' ')}` }, + { run: 'npx playwright install --with-deps chromium' }, { run: 'npm run test:browser' }]); + }); + + it('keeps the required check names: one job per workflow, with no name and no matrix', () => { + // A check is named after its job id, or after its `name` when it has one; a matrix adds the matrix values to the + // name. Either change would leave the required check `test` or `real-docker` never reporting. The test job's + // timeout is about twice its longest recent run (20 min). + const ci = workflow('ci.yml').jobs, isolation = workflow('agent-isolation.yml').jobs; + expect(Object.keys(ci)).toEqual(['test']); + expect(Object.keys(isolation)).toEqual(['real-docker']); + expect(ci.test).toMatchObject({ 'runs-on': 'ubuntu-latest', 'timeout-minutes': 45 }); + expect(isolation['real-docker']).toMatchObject({ 'runs-on': 'ubuntu-latest', 'timeout-minutes': 45 }); + for (const job of [ci.test!, isolation['real-docker']!]) + expect(Object.keys(job).toSorted()).toEqual(['runs-on', 'steps', 'timeout-minutes']); + }); + + it('has no workflow keys other than the ones it uses now', () => { + // A workflow-level `defaults` or `env`, such as `defaults: { run: { shell: 'true {0}' } }`, would change every + // `run:` step of every job, for example into a no-op that passes. + const top = (name: string) => Object.keys(parse(readFileSync(join(ROOT, '.github/workflows', name), 'utf8')) as object); + expect(top('ci.yml')).toEqual(['name', 'on', 'permissions', 'jobs']); + expect(top('agent-isolation.yml')).toEqual(['name', 'on', 'permissions', 'jobs']); + expect(top('live-probes.yml')).toEqual(['name', 'on', 'permissions', 'concurrency', 'jobs']); + for (const name of ['ci.yml', 'agent-isolation.yml', 'live-probes.yml']) + expect(workflow(name).permissions, name).toEqual({ contents: 'read' }); + }); + + it('runs the package scripts the gating jobs call exactly as written here', () => { + // Both required jobs call these scripts; a filter, -t, --project or --passWithNoTests here would narrow them silently. + const scripts = (JSON.parse(readFileSync(join(ROOT, 'package.json'), 'utf8')) as { scripts: Record }).scripts; + expect({ test: scripts.test, typecheck: scripts.typecheck, 'test:browser': scripts['test:browser'] }) + .toEqual({ test: 'vitest run', typecheck: 'tsc --noEmit', 'test:browser': 'playwright test' }); + }); + + it('uses the same Node version in every workflow', () => { + const versions = ['ci.yml', 'agent-isolation.yml', 'live-probes.yml'].flatMap(name => Object.values(workflow(name).jobs) + .flatMap(job => job.steps.filter(step => String(step.uses).startsWith('actions/setup-node@')) + .map(step => (step.with as Record | undefined)?.['node-version']))); + expect(versions).toEqual([NODE_VERSION, NODE_VERSION, NODE_VERSION]); + }); + + it('runs only the live Claude probes, by hand or nightly, never on a pull request or push', () => { + const live = workflow('live-probes.yml'); + expect(Object.keys(live.on).toSorted()).toEqual(['schedule', 'workflow_dispatch']); + expect(Object.keys(live.jobs)).toEqual(['live-probes']); + const job = live.jobs['live-probes']!; + // No `defaults`, `env`, `strategy` or `container`: any of them could change what the steps run. + expect(Object.keys(job).toSorted()).toEqual(['runs-on', 'steps', 'timeout-minutes']); + expect(job).toMatchObject({ 'runs-on': 'ubuntu-latest', 'timeout-minutes': 75 }); + const token = { CLAUDE_CODE_OAUTH_TOKEN: '${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}' }; + expect(job.steps.map(step => Object.keys(step).toSorted())).toEqual([['env', 'name', 'run'], ['uses'], ['uses', 'with'], + ['run'], ['env', 'name', 'run'], ['name', 'run']]); + expect(job.steps.slice(1, 4)).toEqual([...SETUP, { run: 'npm ci --ignore-scripts' }]); + expect(job.steps[0]).toEqual({ name: 'Require the Claude token', env: token, + run: 'if [ -z "$CLAUDE_CODE_OAUTH_TOKEN" ]; then\n' + + ' echo "::error::The repository secret CLAUDE_CODE_OAUTH_TOKEN is not set. Add it to run the live probes."\n' + + ' exit 1\nfi\n' }); + expect(job.steps[4]).toEqual({ name: 'Run the live Claude probes', env: { CODEBOOST_RUN_AUTH_PROBES: '1', ...token }, + run: `npx vitest run --project docker -t '${MARKER_FILTER}' --reporter=default --reporter=json ` + + `--outputFile=live-probes.json ${LIVE_PROBE_FILES.join(' ')}` }); + expect(job.steps[5]).toEqual({ name: 'Check that every probe passed', run: 'node scripts/live-probes.ts check live-probes.json' }); + const vitest = job.steps.filter(step => step.run?.startsWith('npx vitest')); + expect(vitest).toHaveLength(1); + const words = vitest[0]!.run!.split(/\s+/); + // The docker project runs the files one at a time; the filter keeps only titles that end in the marker. + expect(vitest[0]!.run!.startsWith(`npx vitest run --project docker -t '${MARKER_FILTER}' `)).toBe(true); + expect(words.filter(word => word.startsWith('test/'))).toEqual(LIVE_PROBE_FILES); + expect(vitest[0]!.env).toEqual({ CODEBOOST_RUN_AUTH_PROBES: '1', + CLAUDE_CODE_OAUTH_TOKEN: '${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}' }); + expect(runs(job.steps).at(-1)).toBe('node scripts/live-probes.ts check live-probes.json'); + for (const file of LIVE_PROBE_FILES) expect(DOCKER_TESTS).toContain(file); + }); + + it('never skips or soft-fails a gating job or any of its steps', () => { + // A condition or continue-on-error on the required checks or on the live probe job could skip a step or let it + // fail while the job still passes. + for (const [name, job] of [['ci.yml', 'test'], ['agent-isolation.yml', 'real-docker'], ['live-probes.yml', 'live-probes']] as const) { + const jobConfig = workflow(name).jobs[job]!; + for (const part of [jobConfig, ...jobConfig.steps]) { + expect(Object.keys(part), `${name} ${job}`).not.toContain('if'); + expect(Object.keys(part), `${name} ${job}`).not.toContain('continue-on-error'); + } + } + }); + + it('fails the live probe job at once when the token secret is empty', () => { + const guard = workflow('live-probes.yml').jobs['live-probes']!.steps[0]!; + expect(guard.env).toEqual({ CLAUDE_CODE_OAUTH_TOKEN: '${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}' }); + const run = (token: string | undefined) => spawnSync('bash', ['-e', '-c', guard.run!], { encoding: 'utf8', + env: { PATH: process.env.PATH, ...(token === undefined ? {} : { CLAUDE_CODE_OAUTH_TOKEN: token }) } }); + for (const token of [undefined, '']) { + const result = run(token); + expect(result.status).toBe(1); + expect(result.stdout).toContain('::error::The repository secret CLAUDE_CODE_OAUTH_TOKEN is not set'); + } + expect(run('placeholder').status).toBe(0); + }); + + it('names every live probe exactly as its test file titles it', () => { + // The tests that the probe switch enables are exactly the listed probes, by Vitest full name, and every one ends in + // the suffix the workflow's -t filter selects. No other test in the file carries the suffix in its literal title: + // every title in the file is a literal, or probeFileTests throws, so this sees each one. A `.each` or `.for` title is + // read before Vitest fills its placeholders, so `it.each(['(live Claude)'])('sneaky %s', ...)` passes here; the + // live-probes job's report check (`checkLiveProbeReport`) then refuses that test, because it ran and is not a probe. + for (const file of LIVE_PROBE_FILES) { + const tests = probeFileTests(file, readFileSync(join(ROOT, file), 'utf8')); + const gated = tests.filter(test => test.gated).map(test => test.name); + expect(gated.toSorted(), file).toEqual(LIVE_PROBES.filter(probe => probe.file === file).map(probe => probe.name).toSorted()); + for (const name of gated) { + expect(name.endsWith(` ${LIVE_PROBE_MARKER}`), name).toBe(true); + expect(name, file).toMatch(new RegExp(MARKER_FILTER)); + } + expect(tests.filter(test => !test.gated && new RegExp(MARKER_FILTER).test(test.name)), file) + .toEqual([]); + } + // Every JavaScript or TypeScript source file under test/ except this one, helpers and fixtures included, since a + // probe file could import a gate. + const named = readdirSync(join(ROOT, 'test'), { recursive: true, encoding: 'utf8' }) + .map(name => `test/${name.split(sep).join('/')}`) + .filter(file => SOURCE.test(file) && file !== 'test/vitest-config.test.ts') + .filter(file => readFileSync(join(ROOT, file), 'utf8').includes(PROBE_SWITCH)); + expect(named.toSorted()).toEqual(LIVE_PROBE_FILES.toSorted()); + }); +}); + +describe('live probe report check', () => { + const report = (tests: Array<[string, string, string]>, failed = 0) => ({ + success: failed === 0, numFailedTests: failed, + testResults: [...new Set(tests.map(([file]) => file))].map(file => ({ name: join(ROOT, file), + assertionResults: tests.filter(([testFile]) => testFile === file).map(([, fullName, status]) => ({ fullName, status })) })), + }); + const all = LIVE_PROBES.map(probe => [probe.file, probe.name, 'passed'] as [string, string, string]); + + it('accepts a report in which every probe passed and nothing else ran', () => { + expect(checkLiveProbeReport(report([...all, [LIVE_PROBES[0].file, 'other test', 'skipped']]), ROOT)).toEqual([]); + }); + + it('refuses a report in which the probes were skipped, as when the probe switch is off', () => { + const skipped = all.map(([file, name]) => [file, name, 'skipped'] as [string, string, string]); + expect(checkLiveProbeReport(report(skipped), ROOT)).toHaveLength(LIVE_PROBES.length); + expect(checkLiveProbeReport({}, ROOT).length).toBeGreaterThan(LIVE_PROBES.length); + }); + + it('refuses a missing probe, a failure or a test that is not a probe', () => { + expect(checkLiveProbeReport(report(all.slice(1)), ROOT)).toEqual( + [`Live probe did not pass: ${LIVE_PROBES[0].file} > ${LIVE_PROBES[0].name}`]); + expect(checkLiveProbeReport(report(all, 1), ROOT)).toEqual( + ['Vitest did not report success.', 'Vitest reported 1 failed test(s).']); + expect(checkLiveProbeReport(report([...all, [LIVE_PROBES[0].file, 'other test', 'passed']]), ROOT)).toEqual( + [`A test that is not a live probe ran: ${join(ROOT, LIVE_PROBES[0].file)} > other test`]); + }); +});