From 19a818348ac6717755d1c6ba1c47bf5147deba42 Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 17:52:21 -0700 Subject: [PATCH 01/75] Remove test objects made without a runner label however the test ends (#189) `withoutRunnerLabel` now labels each object it strips with a per-call test-run ID and removes everything with that ID when the test finishes, passed, failed or timed out. The owner sweep also removes containers that hold a swept volume. In agent-network, the stand-in proxy and the blocker are removed in `finally`. A pair of tests checks that a keeper left by a test that stopped early is gone. Closes #189 Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 38 ++++++++++++++++++++++++++++++----- test/agent-network.test.ts | 20 +++++++++++------- test/fixtures/docker-sweep.ts | 38 ++++++++++++++++++++++++++--------- 3 files changed, 75 insertions(+), 21 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 83646a9e..3d7c1f0a 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -1,5 +1,5 @@ import { execFileSync, spawnSync } from 'node:child_process'; -import { removeRunnerObjects } from './fixtures/docker-sweep.ts'; +import { removeLabelledObjects, removeRunnerObjects, TEST_RUN_LABEL } from './fixtures/docker-sweep.ts'; import { fixtureGit } from './fixtures/git.ts'; import { createHash, randomBytes, randomUUID } from 'node:crypto'; import { chmodSync, existsSync, mkdtempSync, mkdirSync, readdirSync, readFileSync, realpathSync, renameSync, rmSync, statSync, @@ -7,7 +7,7 @@ import { chmodSync, existsSync, mkdtempSync, mkdirSync, readdirSync, readFileSyn import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { pathToFileURL } from 'node:url'; -import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; +import { afterAll, afterEach, beforeAll, describe, expect, it, onTestFinished, vi } from 'vitest'; import { captureInvocation, type InvocationInput, type Phase } from '../agents/contract.ts'; import { AGENT_IMAGE, assertBuiltAgentImage, buildAgentImage } from '../agents/container/image.ts'; import { assertContainerProfile, createContainerProfile, disposeContainerProfile, @@ -144,8 +144,9 @@ afterAll(async () => { } } finally { // The runner owner is this run's own, so nothing else would find what this run leaves behind. Objects a test - // makes under another owner or without the runner label are not found here; those tests remove their own. It may - // not run before a hook timeout. + // makes under another owner or without the runner label are not found here, unless they hold one of this run's + // volumes; those tests remove their own (`withoutRunnerLabel` when the test finishes). It may not run before a hook + // timeout. removeRunnerObjects(TEST_RUNNER_OWNER); } }, 120_000); @@ -625,8 +626,13 @@ describe('real Docker agent isolation', () => { }, 60_000); // A docker wrapper that drops our runner label from commands starting with one of `prefixes`, so the object it - // creates is genuinely mislabelled and only the owner-label checks can catch it. + // creates is genuinely mislabelled and only the owner-label checks can catch it. Neither afterAll's runner sweep nor + // production cleanup finds such an object, so the wrapper labels it with a test-run ID instead, and the test removes + // everything with that ID when it finishes, passed, failed or timed out (#189). const withoutRunnerLabel = async (prefixes: string[][], run: () => Promise | T): Promise => { + if (!prefixes.every(prefix => prefix.includes('create'))) throw new Error('Only create commands can be unlabelled.'); + const testRun = randomUUID(); + onTestFinished(() => removeLabelledObjects(TEST_RUN_LABEL, testRun, 'onTestFinished')); const shim = mkdtempSync(join(tmpdir(), 'codeboost-unlabelled-docker-')); roots.push(shim); mkdirSync(join(shim, 'input')); // afterAll resets this path's mode const realDocker = execFileSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).trim(); @@ -636,6 +642,7 @@ describe('real Docker agent isolation', () => { `const strip = ${JSON.stringify(prefixes)}.some(prefix => prefix.every((word, i) => args[i] === word));`, 'const out = [];', "for (let i = 0; i < args.length; i++) { if (strip && args[i] === '--label' && String(args[i + 1]).startsWith('io.codeboost.runner=')) { i++; continue; } out.push(args[i]); }", + `if (strip) out.splice(out.indexOf('create') + 1, 0, '--label', ${JSON.stringify(`${TEST_RUN_LABEL}=${testRun}`)});`, `const result = spawnSync(${JSON.stringify(realDocker)}, out, { stdio: 'inherit' });`, 'process.exit(result.status ?? 1);'].join('\n'), { mode: 0o755 }); const path = process.env.PATH; @@ -683,6 +690,27 @@ describe('real Docker agent isolation', () => { expect(spawnSync('docker', ['container', 'inspect', live.name]).status).not.toBe(0); }, 60_000); + // #189: these two run in order. The first stops after making a keeper without its runner label, before the removal + // its sibling test above ends with; the second checks that the keeper went when the first finished. + let stoppedEarly: ReturnType | undefined; + it.fails('stops after making a keeper without its runner label (expected to fail)', async () => { + stoppedEarly = await withoutRunnerLabel([['create', '--name']], () => fixture()); + const keeper = JSON.parse(docker('container', 'inspect', stoppedEarly.filesystems.keeper))[0] as + { Config: { Labels: Record } }; + expect(keeper.Config.Labels).not.toHaveProperty('io.codeboost.runner'); + expect(keeper.Config.Labels).toHaveProperty(TEST_RUN_LABEL); + throw new Error('This test stops before it removes the keeper.'); + }, 60_000); + + it('removed the keeper that a test without its runner label left when it stopped', () => { + const left = stoppedEarly!.filesystems, allocationId = taskFilesystemOwner(left).allocationId; + expect(spawnSync('docker', ['container', 'inspect', left.keeper], { stdio: 'ignore' }).status).not.toBe(0); + // Its volumes carry the runner label, so ordinary cleanup can remove them now that no keeper holds them. + taskFilesystems.splice(taskFilesystems.indexOf(left), 1); + removeTaskFilesystems(left); + expect(byAllocation(allocationId)).toEqual([]); + }, 60_000); + it('refuses an allocation ID that still labels Docker objects, as after a restart, before creating any storage', () => { const data = fixture(), reused = taskFilesystemOwner(data.filesystems); const before = docker('volume', 'ls', '--quiet', '--filter', `label=io.codeboost.allocation=${reused.allocationId}`); diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index ee2b4ed2..632daf78 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -1,6 +1,6 @@ import { execFileSync, spawnSync } from 'node:child_process'; import { randomBytes, randomUUID } from 'node:crypto'; -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; @@ -190,13 +190,19 @@ describe('vendor-only egress', () => { deadline: Date.now() + 60_000 }); const path = process.env.PATH; process.env.PATH = `${shim}:${path}`; - let impostor = ''; + let impostor = '', survived = false; try { await expect(createVendorNetwork(failing, imageId, randomUUID())).rejects.toThrow(); - impostor = readFileSync(recorded, 'utf8'); - } finally { process.env.PATH = path; rmSync(shim, { recursive: true, force: true }); } - const survived = spawnSync('docker', ['container', 'inspect', impostor], { stdio: 'ignore' }).status === 0; - spawnSync('docker', ['rm', '--force', impostor], { stdio: 'ignore' }); + } finally { + process.env.PATH = path; + // The stand-in has no runner label, so no sweep finds it: remove it here, however the test ends (#189). + impostor = existsSync(recorded) ? readFileSync(recorded, 'utf8') : ''; + if (impostor) { + survived = spawnSync('docker', ['container', 'inspect', impostor], { stdio: 'ignore' }).status === 0; + spawnSync('docker', ['rm', '--force', impostor], { stdio: 'ignore' }); + } + rmSync(shim, { recursive: true, force: true }); + } expect(impostor).toMatch(/^codeboost-proxy-/); expect(survived).toBe(true); }, 60_000); @@ -209,8 +215,8 @@ describe('vendor-only egress', () => { expect(proxy).toMatchObject({ kind: 'container', name: created.proxyContainer }); expect(net).toMatchObject({ kind: 'network', name: created.name }); // A foreign endpoint keeps the network busy, so the proxy is removed but `network rm` fails. - docker('run', '--detach', '--name', blocker, '--network', created.name, '--entrypoint', 'sleep', imageId, '300'); try { + docker('run', '--detach', '--name', blocker, '--network', created.name, '--entrypoint', 'sleep', imageId, '300'); await expect(removeVendorNetwork(created, 10_000)).rejects.toThrow('did not settle'); expect(vendorNetworkResources(created)).toEqual([net]); } finally { spawnSync('docker', ['rm', '--force', blocker], { stdio: 'ignore' }); } diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index 97c0bf0e..9d25556a 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -2,24 +2,44 @@ import { spawnSync } from 'node:child_process'; import { RUNNER_LABEL } from '../../agents/labels.ts'; /** - * Removes the containers, networks and volumes labelled with one runner owner, containers first, as they hold the - * others in use. Containers and volumes go by force; a network that another container still uses stays. It reports - * what it finds and any Docker call that fails, and never throws, so a test file's afterAll can call it in a finally - * block without hiding an error from its ordinary cleanup. Each call has a 30 s limit, so all six can take up to - * 3 minutes. + * The label a test's Docker shim adds to an object it makes without the runner label (#189), so the test can still + * find and remove that object however the test ends. No codeboost code writes or reads it. + */ +export const TEST_RUN_LABEL = 'io.codeboost.test-run'; + +/** + * Removes the containers, networks and volumes labelled with one runner owner (see `removeLabelledObjects`). A test + * file's afterAll calls it in a finally block. */ export function removeRunnerObjects(runnerOwner: string): void { + removeLabelledObjects(RUNNER_LABEL, runnerOwner, 'afterAll'); +} + +/** + * Removes the containers, networks and volumes that carry `label=value`, containers first, as they hold the others in + * use. Before it removes a volume it also removes every container that mounts it, labelled or not: a test can make a + * volume with one owner and the container that holds it without that owner. Containers and volumes go by force; a + * network that another container still uses stays. It reports what it finds and any Docker call that fails, prefixed + * with `when`, and never throws, so cleanup can call it without hiding an earlier error. Each call has a 30 s limit. + */ +export function removeLabelledObjects(label: string, value: string, when: string): void { const run = (args: string[]) => { const result = spawnSync('docker', args, { encoding: 'utf8', timeout: 30_000, killSignal: 'SIGKILL' }); - if (result.status !== 0) console.warn(`afterAll: docker ${args.slice(0, 2).join(' ')} failed:`, + if (result.status !== 0) console.warn(`${when}: docker ${args.slice(0, 2).join(' ')} failed:`, result.error?.message ?? result.stderr.trim()); return result.status === 0 ? result.stdout.split('\n').filter(Boolean) : []; }; for (const kind of ['container', 'network', 'volume'] as const) { - const ids = run([kind, 'ls', ...(kind === 'container' ? ['--all'] : []), '--quiet', '--filter', - `label=${RUNNER_LABEL}=${runnerOwner}`]); + const ids = run([kind, 'ls', ...(kind === 'container' ? ['--all'] : []), '--quiet', '--filter', `label=${label}=${value}`]); if (!ids.length) continue; - console.warn(`afterAll removes this run's leftover ${kind}s: ${ids.join(' ')}`); + console.warn(`${when} removes leftover ${kind}s labelled ${label}=${value}: ${ids.join(' ')}`); + if (kind === 'volume') { + const holders = [...new Set(ids.flatMap(id => run(['ps', '--all', '--quiet', '--filter', `volume=${id}`])))]; + if (holders.length) { + console.warn(`${when} removes the containers that hold them: ${holders.join(' ')}`); + run(['rm', '--force', ...holders]); + } + } run([kind, 'rm', ...(kind === 'network' ? [] : ['--force']), ...ids]); } } From 6c1034aa376b1244d8e3a485aad0a5090841c27b Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 17:59:32 -0700 Subject: [PATCH 02/75] Give the test-run sweep its own budget and an afterAll pass (#189) Review round 1: the onTestFinished removal gets 3 minutes, not the 10 s hook default; afterAll sweeps every test-run ID again, for a create that lands after a timed-out test's removal; the regression pair records the storage only after its checks and takes it out of afterAll's list first; one ps call finds every holder. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 30 ++++++++++++++++++++---------- test/fixtures/docker-sweep.ts | 6 ++++-- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 3d7c1f0a..68b7ed47 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -47,6 +47,8 @@ const testOwner = (attemptId = 'fixture') => ({ runnerOwner: TEST_RUNNER_OWNER, const roots: string[] = []; const taskFilesystems: ReturnType[] = []; const containers = new Set(); +/** The test-run IDs `withoutRunnerLabel` put on objects it made without the runner label (#189). */ +const testRuns: string[] = []; const profiles: Awaited>[] = []; let imageId = ''; const vendorNetworks: VendorNetwork[] = []; @@ -145,8 +147,9 @@ afterAll(async () => { } finally { // The runner owner is this run's own, so nothing else would find what this run leaves behind. Objects a test // makes under another owner or without the runner label are not found here, unless they hold one of this run's - // volumes; those tests remove their own (`withoutRunnerLabel` when the test finishes). It may not run before a hook - // timeout. + // volumes: those made through `withoutRunnerLabel` are found by their test-run IDs, and the other tests remove + // their own. It may not run before a hook timeout. + for (const testRun of testRuns) removeLabelledObjects(TEST_RUN_LABEL, testRun, 'afterAll'); removeRunnerObjects(TEST_RUNNER_OWNER); } }, 120_000); @@ -626,13 +629,16 @@ describe('real Docker agent isolation', () => { }, 60_000); // A docker wrapper that drops our runner label from commands starting with one of `prefixes`, so the object it - // creates is genuinely mislabelled and only the owner-label checks can catch it. Neither afterAll's runner sweep nor - // production cleanup finds such an object, so the wrapper labels it with a test-run ID instead, and the test removes - // everything with that ID when it finishes, passed, failed or timed out (#189). + // creates is genuinely mislabelled and only the owner-label checks can catch it. Production cleanup refuses such an + // object, and afterAll's runner sweep finds it only while it holds one of this run's volumes, so the wrapper labels it + // with a test-run ID instead (#189). Whatever has that ID is removed when the test finishes, passed, failed or timed + // out, and again in afterAll, for a create that lands after a timed-out test's removal (its body runs on). const withoutRunnerLabel = async (prefixes: string[][], run: () => Promise | T): Promise => { if (!prefixes.every(prefix => prefix.includes('create'))) throw new Error('Only create commands can be unlabelled.'); const testRun = randomUUID(); - onTestFinished(() => removeLabelledObjects(TEST_RUN_LABEL, testRun, 'onTestFinished')); + testRuns.push(testRun); + // Each Docker call may take 30 s, so the removal gets more than the 10 s a hook gets by default. + onTestFinished(() => removeLabelledObjects(TEST_RUN_LABEL, testRun, 'onTestFinished'), 3 * 60_000); const shim = mkdtempSync(join(tmpdir(), 'codeboost-unlabelled-docker-')); roots.push(shim); mkdirSync(join(shim, 'input')); // afterAll resets this path's mode const realDocker = execFileSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).trim(); @@ -691,22 +697,26 @@ describe('real Docker agent isolation', () => { }, 60_000); // #189: these two run in order. The first stops after making a keeper without its runner label, before the removal - // its sibling test above ends with; the second checks that the keeper went when the first finished. + // its sibling test above ends with; the second checks that the keeper went when the first finished. `it.fails` passes + // on any failure, so the first records the storage only once its checks passed, just before its deliberate stop. let stoppedEarly: ReturnType | undefined; it.fails('stops after making a keeper without its runner label (expected to fail)', async () => { - stoppedEarly = await withoutRunnerLabel([['create', '--name']], () => fixture()); - const keeper = JSON.parse(docker('container', 'inspect', stoppedEarly.filesystems.keeper))[0] as + const data = await withoutRunnerLabel([['create', '--name']], () => fixture()); + const keeper = JSON.parse(docker('container', 'inspect', data.filesystems.keeper))[0] as { Config: { Labels: Record } }; expect(keeper.Config.Labels).not.toHaveProperty('io.codeboost.runner'); expect(keeper.Config.Labels).toHaveProperty(TEST_RUN_LABEL); + stoppedEarly = data; throw new Error('This test stops before it removes the keeper.'); }, 60_000); it('removed the keeper that a test without its runner label left when it stopped', () => { + expect(stoppedEarly, 'the test before did not reach its deliberate stop').toBeDefined(); const left = stoppedEarly!.filesystems, allocationId = taskFilesystemOwner(left).allocationId; + // First, so afterAll does not refuse a keeper this test finds left; its runner sweep removes what is. + taskFilesystems.splice(taskFilesystems.indexOf(left), 1); expect(spawnSync('docker', ['container', 'inspect', left.keeper], { stdio: 'ignore' }).status).not.toBe(0); // Its volumes carry the runner label, so ordinary cleanup can remove them now that no keeper holds them. - taskFilesystems.splice(taskFilesystems.indexOf(left), 1); removeTaskFilesystems(left); expect(byAllocation(allocationId)).toEqual([]); }, 60_000); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index 9d25556a..0f2f80c2 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -20,7 +20,8 @@ export function removeRunnerObjects(runnerOwner: string): void { * use. Before it removes a volume it also removes every container that mounts it, labelled or not: a test can make a * volume with one owner and the container that holds it without that owner. Containers and volumes go by force; a * network that another container still uses stays. It reports what it finds and any Docker call that fails, prefixed - * with `when`, and never throws, so cleanup can call it without hiding an earlier error. Each call has a 30 s limit. + * with `when`, and never throws, so cleanup can call it without hiding an earlier error. Each call has a 30 s limit, so + * all seven can take up to 3.5 minutes. */ export function removeLabelledObjects(label: string, value: string, when: string): void { const run = (args: string[]) => { @@ -34,7 +35,8 @@ export function removeLabelledObjects(label: string, value: string, when: string if (!ids.length) continue; console.warn(`${when} removes leftover ${kind}s labelled ${label}=${value}: ${ids.join(' ')}`); if (kind === 'volume') { - const holders = [...new Set(ids.flatMap(id => run(['ps', '--all', '--quiet', '--filter', `volume=${id}`])))]; + // Filters with one key match any of their values. + const holders = run(['ps', '--all', '--quiet', ...ids.flatMap(id => ['--filter', `volume=${id}`])]); if (holders.length) { console.warn(`${when} removes the containers that hold them: ${holders.join(' ')}`); run(['rm', '--force', ...holders]); From 8828d582ba99f3fdd9ca3baf89eb7bd7b98416b6 Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 18:05:20 -0700 Subject: [PATCH 03/75] Remove containers that hold a swept network; record the stand-in first (#189) Review round 2: a network made without the runner label stayed when a runner-labelled proxy held it, so the sweep now removes the containers on a swept network too (looked up by name, which also finds a created container). The hook budget covers the sweep's ten calls, and the impostor's name is recorded before the stand-in exists. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 16 +++++++++------- test/agent-network.test.ts | 4 +++- test/fixtures/docker-sweep.ts | 18 ++++++++++-------- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 68b7ed47..64e7acfe 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -147,8 +147,9 @@ afterAll(async () => { } finally { // The runner owner is this run's own, so nothing else would find what this run leaves behind. Objects a test // makes under another owner or without the runner label are not found here, unless they hold one of this run's - // volumes: those made through `withoutRunnerLabel` are found by their test-run IDs, and the other tests remove - // their own. It may not run before a hook timeout. + // volumes or networks: those made through `withoutRunnerLabel` are found by their test-run IDs, and the other tests + // remove their own. It may not run before a hook timeout, and its budget does not cover every Docker call taking + // its full 30 s. for (const testRun of testRuns) removeLabelledObjects(TEST_RUN_LABEL, testRun, 'afterAll'); removeRunnerObjects(TEST_RUNNER_OWNER); } @@ -630,15 +631,16 @@ describe('real Docker agent isolation', () => { // A docker wrapper that drops our runner label from commands starting with one of `prefixes`, so the object it // creates is genuinely mislabelled and only the owner-label checks can catch it. Production cleanup refuses such an - // object, and afterAll's runner sweep finds it only while it holds one of this run's volumes, so the wrapper labels it - // with a test-run ID instead (#189). Whatever has that ID is removed when the test finishes, passed, failed or timed - // out, and again in afterAll, for a create that lands after a timed-out test's removal (its body runs on). + // object, and afterAll's runner sweep finds it only while it holds one of this run's volumes or networks, so the + // wrapper labels it with a test-run ID instead (#189). Whatever has that ID is removed when the test finishes, passed, + // failed or timed out, and again in afterAll, for a create that lands after a timed-out test's removal (its body runs + // on). const withoutRunnerLabel = async (prefixes: string[][], run: () => Promise | T): Promise => { if (!prefixes.every(prefix => prefix.includes('create'))) throw new Error('Only create commands can be unlabelled.'); const testRun = randomUUID(); testRuns.push(testRun); - // Each Docker call may take 30 s, so the removal gets more than the 10 s a hook gets by default. - onTestFinished(() => removeLabelledObjects(TEST_RUN_LABEL, testRun, 'onTestFinished'), 3 * 60_000); + // Up to ten Docker calls of 30 s each, so the removal gets their worst case, not the 10 s a hook gets by default. + onTestFinished(() => removeLabelledObjects(TEST_RUN_LABEL, testRun, 'onTestFinished'), 5 * 60_000); const shim = mkdtempSync(join(tmpdir(), 'codeboost-unlabelled-docker-')); roots.push(shim); mkdirSync(join(shim, 'input')); // afterAll resets this path's mode const realDocker = execFileSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).trim(); diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index 632daf78..0f519f86 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -184,8 +184,10 @@ describe('vendor-only egress', () => { ` name=$('${realDocker}' inspect -f '{{.Name}}' "$2" | sed 's#^/##')`, ` label=$('${realDocker}' inspect -f '{{index .Config.Labels "io.codeboost.egress"}}' "$2")`, ` '${realDocker}' rm --force "$2" >/dev/null`, + // Recorded before the stand-in exists, so the test can remove it even if the run is cut short. + ` printf %s "$name" > '${recorded}'`, ` '${realDocker}' run --detach --name "$name" --label "io.codeboost.egress=$label" --entrypoint sleep ${imageId} 300 >/dev/null`, - ` printf %s "$name" > '${recorded}'; exit 1`, 'fi', `exec '${realDocker}' "$@"`].join('\n'), { mode: 0o755 }); + ' exit 1', 'fi', `exec '${realDocker}' "$@"`].join('\n'), { mode: 0o755 }); const failing = captureInvocation({ ...invocation, attemptId: `failed-setup-${randomUUID()}`, deadline: Date.now() + 60_000 }); const path = process.env.PATH; diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index 0f2f80c2..3dc51bd5 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -17,11 +17,11 @@ export function removeRunnerObjects(runnerOwner: string): void { /** * Removes the containers, networks and volumes that carry `label=value`, containers first, as they hold the others in - * use. Before it removes a volume it also removes every container that mounts it, labelled or not: a test can make a - * volume with one owner and the container that holds it without that owner. Containers and volumes go by force; a - * network that another container still uses stays. It reports what it finds and any Docker call that fails, prefixed - * with `when`, and never throws, so cleanup can call it without hiding an earlier error. Each call has a 30 s limit, so - * all seven can take up to 3.5 minutes. + * use. Before it removes a network or a volume it also removes every container attached to it or mounting it, labelled + * or not: a test can make a network or volume with one owner and the container that holds it with another, or without + * one. Containers and volumes go by force; a network another container joins meanwhile stays. It reports what it finds + * and any Docker call that fails, prefixed with `when`, and never throws, so cleanup can call it without hiding an + * earlier error. Each call has a 30 s limit, so all ten can take up to 5 minutes. */ export function removeLabelledObjects(label: string, value: string, when: string): void { const run = (args: string[]) => { @@ -31,12 +31,14 @@ export function removeLabelledObjects(label: string, value: string, when: string return result.status === 0 ? result.stdout.split('\n').filter(Boolean) : []; }; for (const kind of ['container', 'network', 'volume'] as const) { - const ids = run([kind, 'ls', ...(kind === 'container' ? ['--all'] : []), '--quiet', '--filter', `label=${label}=${value}`]); + // Networks by name: `ps --filter network=` finds a created, never started container by the name only (Docker 27.4). + const ids = run([kind, 'ls', ...(kind === 'container' ? ['--all'] : []), ...(kind === 'network' ? ['--format', '{{.Name}}'] + : ['--quiet']), '--filter', `label=${label}=${value}`]); if (!ids.length) continue; console.warn(`${when} removes leftover ${kind}s labelled ${label}=${value}: ${ids.join(' ')}`); - if (kind === 'volume') { + if (kind !== 'container') { // Filters with one key match any of their values. - const holders = run(['ps', '--all', '--quiet', ...ids.flatMap(id => ['--filter', `volume=${id}`])]); + const holders = run(['ps', '--all', '--quiet', ...ids.flatMap(id => ['--filter', `${kind}=${id}`])]); if (holders.length) { console.warn(`${when} removes the containers that hold them: ${holders.join(' ')}`); run(['rm', '--force', ...holders]); From c6b27fe1299b6b7a034a1d4746451fbd463658be Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 18:24:37 -0700 Subject: [PATCH 04/75] Remove only exact holders of a swept network; test the holder sweep (#189) Review round 3: Docker's network filter also matches the start of another network's ID, so holders are checked by network name; a real-Docker test covers created and running holders and bystanders; hook budgets cover the sweep; label-less creates in agent-container and agent-supervisor are tracked before they are made. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 59 ++++++++++++++++++++++++++++++----- test/agent-gate.test.ts | 2 +- test/agent-network.test.ts | 2 +- test/agent-supervisor.test.ts | 11 ++++--- test/fixtures/docker-sweep.ts | 11 +++++-- test/runner-workspace.test.ts | 2 +- 6 files changed, 68 insertions(+), 19 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 64e7acfe..cb9f0070 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -563,8 +563,10 @@ describe('real Docker agent isolation', () => { const moved = `${live.name}-moved`, replacement = live.name; // Someone renames our container and puts a same-named, same-labelled container in its place. docker('rename', live.name, moved); containers.add(moved); + // Tracked before the create, so afterAll removes it even if the create's client fails after the daemon made it. + containers.add(replacement); docker('create', '--name', replacement, '--label', `io.codeboost.invocation=${live.ownershipId}`, - '--entrypoint', 'true', imageId); containers.add(replacement); + '--entrypoint', 'true', imageId); const replacementId = docker('container', 'inspect', '--format', '{{.Id}}', replacement); await disposeValidatedContainer(live); expect(spawnSync('docker', ['container', 'inspect', originalId], { stdio: 'ignore' }).status).not.toBe(0); @@ -582,8 +584,10 @@ describe('real Docker agent isolation', () => { await expect(disposeValidatedContainer(live)).rejects.toThrow('did not settle'); expect(isContainerProfileAuthentic(live)).toBe(true); // A same-named, same-labelled container appears before the retry. + // Tracked before the create, so afterAll removes it even if the create's client fails after the daemon made it. + containers.add(replacement); docker('create', '--name', replacement, '--label', `io.codeboost.invocation=${live.ownershipId}`, - '--entrypoint', 'true', imageId); containers.add(replacement); + '--entrypoint', 'true', imageId); const replacementId = docker('container', 'inspect', '--format', '{{.Id}}', replacement); docker('rm', '--force', blocker); containers.delete(blocker); await disposeValidatedContainer(live); @@ -597,8 +601,10 @@ describe('real Docker agent isolation', () => { const originalId = docker('container', 'inspect', '--format', '{{.Id}}', live.name); const moved = `${live.name}-moved`, replacement = live.name; docker('rename', live.name, moved); containers.add(moved); + // Tracked before the create, so afterAll removes it even if the create's client fails after the daemon made it. + containers.add(replacement); docker('create', '--name', replacement, '--label', `io.codeboost.invocation=${live.ownershipId}`, - '--entrypoint', 'true', imageId); containers.add(replacement); + '--entrypoint', 'true', imageId); expect(await startValidatedContainer(live)).toBe('stdout-marker'); // The replacement was never started, and cleanup removed only ours. expect(docker('container', 'inspect', '--format', '{{.State.StartedAt}}', replacement)).toBe('0001-01-01T00:00:00Z'); @@ -639,7 +645,8 @@ describe('real Docker agent isolation', () => { if (!prefixes.every(prefix => prefix.includes('create'))) throw new Error('Only create commands can be unlabelled.'); const testRun = randomUUID(); testRuns.push(testRun); - // Up to ten Docker calls of 30 s each, so the removal gets their worst case, not the 10 s a hook gets by default. + // Up to ten synchronous Docker calls of 30 s each, which a timer cannot cut short: the budget covers them, so a slow + // removal is not reported as a hook timeout (10 s by default). onTestFinished(() => removeLabelledObjects(TEST_RUN_LABEL, testRun, 'onTestFinished'), 5 * 60_000); const shim = mkdtempSync(join(tmpdir(), 'codeboost-unlabelled-docker-')); roots.push(shim); mkdirSync(join(shim, 'input')); // afterAll resets this path's mode @@ -698,6 +705,41 @@ describe('real Docker agent isolation', () => { expect(spawnSync('docker', ['container', 'inspect', live.name]).status).not.toBe(0); }, 60_000); + // #189: the sweep removes what holds a swept network or volume, even a container created and never started, and + // nothing on a network or volume it does not sweep, even one whose ID starts with a swept network's name. + it('removes the containers that hold a swept network or volume, and nothing else', () => { + const run = randomUUID(), label = `${TEST_RUN_LABEL}=${run}`, made: { kind: string; ref: string }[] = []; + const make = (kind: 'container' | 'network' | 'volume', ...args: string[]) => { + const ref = docker(...(kind === 'container' ? [] : [kind]), ...args); + made.push({ kind, ref }); + return ref; + }; + try { + const bystanderNetwork = make('network', 'create', `codeboost-bystander-${randomUUID()}`); + const bystanderVolume = make('volume', 'create', `codeboost-bystander-${randomUUID()}`); + const bystanders = [make('container', 'run', '--detach', '--network', bystanderNetwork, '--entrypoint', 'sleep', imageId, '300'), + make('container', 'create', '--mount', `type=volume,source=${bystanderVolume},target=/x`, '--entrypoint', 'true', imageId)]; + const network = make('network', 'create', '--label', label, `codeboost-swept-${randomUUID()}`); + // Named like the start of the bystander network's ID: Docker's network filter also matches that prefix. + make('network', 'create', '--label', label, bystanderNetwork.slice(0, 12)); + const volume = make('volume', 'create', '--label', label, `codeboost-swept-${randomUUID()}`); + const holders = [make('container', 'create', '--network', network, '--entrypoint', 'true', imageId), + make('container', 'run', '--detach', '--network', network, '--entrypoint', 'sleep', imageId, '300'), + make('container', 'create', '--mount', `type=volume,source=${volume},target=/x`, '--entrypoint', 'true', imageId)]; + removeLabelledObjects(TEST_RUN_LABEL, run, 'test'); + const exists = (kind: string, ref: string) => spawnSync('docker', [kind, 'inspect', ref], { stdio: 'ignore' }).status === 0; + for (const holder of holders) expect(exists('container', holder)).toBe(false); + expect([exists('network', network), exists('volume', volume)]).toEqual([false, false]); + for (const bystander of bystanders) expect(exists('container', bystander)).toBe(true); + expect([exists('network', bystanderNetwork), exists('volume', bystanderVolume)]).toEqual([true, true]); + } finally { + for (const { kind, ref } of made.filter(object => object.kind === 'container')) + spawnSync('docker', ['rm', '--force', ref], { stdio: 'ignore' }); + for (const { kind, ref } of made.filter(object => object.kind !== 'container')) + spawnSync('docker', [kind, 'rm', ...(kind === 'volume' ? ['--force'] : []), ref], { stdio: 'ignore' }); + } + }, 120_000); + // #189: these two run in order. The first stops after making a keeper without its runner label, before the removal // its sibling test above ends with; the second checks that the keeper went when the first finished. `it.fails` passes // on any failure, so the first records the storage only once its checks passed, just before its deliberate stop. @@ -2375,15 +2417,16 @@ describe('real Docker agent isolation', () => { const data = fixture(), filesystems = data.filesystems, owner = taskFilesystemOwner(filesystems); taskFilesystems.splice(taskFilesystems.indexOf(filesystems), 1); docker('rm', '--force', filesystems.keeper); - docker('run', '--detach', '--name', filesystems.keeper, '--label', `io.codeboost.runner=${'f'.repeat(32)}`, - '--label', `io.codeboost.attempt=${owner.attemptId}`, '--label', `io.codeboost.allocation=${owner.allocationId}`, - '--network=none', '--entrypoint', 'sleep', imageId, 'infinity'); try { + // Inside the try: this keeper carries another runner's label, so only the finally below removes it (#189). + docker('run', '--detach', '--name', filesystems.keeper, '--label', `io.codeboost.runner=${'f'.repeat(32)}`, + '--label', `io.codeboost.attempt=${owner.attemptId}`, '--label', `io.codeboost.allocation=${owner.allocationId}`, + '--network=none', '--entrypoint', 'sleep', imageId, 'infinity'); expect(() => removeTaskFilesystems(filesystems)).toThrow('did not settle'); expect(docker('container', 'inspect', '--format', '{{.State.Running}}', filesystems.keeper)).toBe('true'); expect(spawnSync('docker', ['volume', 'inspect', filesystems.workVolume]).status).not.toBe(0); } finally { - docker('rm', '--force', filesystems.keeper); + spawnSync('docker', ['rm', '--force', filesystems.keeper], { stdio: 'ignore' }); removeTaskFilesystems(filesystems); } }, 60_000); diff --git a/test/agent-gate.test.ts b/test/agent-gate.test.ts index 1bd77fdd..5315b851 100644 --- a/test/agent-gate.test.ts +++ b/test/agent-gate.test.ts @@ -52,7 +52,7 @@ const boundedScratch = tmpfs('/tmp', 'size=32m,nr_inodes=4096'); beforeAll(() => { imageId = buildAgentImage(); }, 10 * 60_000); // The file has no other cleanup: a probe container whose client timed out may still be running. -afterAll(() => removeRunnerObjects(TEST_RUNNER_OWNER), 4 * 60_000); +afterAll(() => removeRunnerObjects(TEST_RUNNER_OWNER), 6 * 60_000); describe('isolation gate detects breaches', () => { it.each([ diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index 0f519f86..56627b31 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -42,7 +42,7 @@ afterAll(async () => { // timeout. removeRunnerObjects(TEST_RUNNER_OWNER); } -}, 4 * 60_000); +}, 6 * 60_000); describe('vendor-only egress', () => { it('keeps failed allocation and its cleanup inside the caller deadline', async () => { diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 1db2faa0..66504d15 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -81,7 +81,7 @@ afterAll(async () => { // ignores SIGTERM, or storage whose removal did not settle. It may not run before a hook timeout. removeRunnerObjects(TEST_RUNNER_OWNER); } -}, 3 * 60_000); +}, 6 * 60_000); describe('container invocation supervisor', () => { it('preserves the first cancellation reason while retained setup cleanup settles', async () => { @@ -385,9 +385,10 @@ describe('container invocation supervisor', () => { it('releases its own resources, and leaves a same-named container another invocation owns', async () => { const current = await profile(fixture(), 'finite-output', 'foreign-name'); - execFileSync('docker', ['create', '--name', current.name, '--label', 'io.codeboost.invocation=foreign', - '--entrypoint', 'true', imageId], { stdio: 'ignore' }); try { + // Inside the try, so its removal runs even if the create's client fails after the daemon made it (#189). + execFileSync('docker', ['create', '--name', current.name, '--label', 'io.codeboost.invocation=foreign', + '--entrypoint', 'true', imageId], { stdio: 'ignore' }); const result = await startProfileInvocation(current).settled; expect(result.stopReason).toBe('capture-failure'); expect(result.stderr).toMatch(/already in use|Conflict/); @@ -436,9 +437,9 @@ describe('container invocation supervisor', () => { it('releases only the profile when rejecting before creation, even if its name is held elsewhere', async () => { const current = await profile(fixture(), 'noop'); // A foreign container occupies the deterministic name, so container-level cleanup could never settle. - execFileSync('docker', ['create', '--name', current.name, '--label', 'io.codeboost.invocation=someone-else', - '--entrypoint', 'true', imageId], { stdio: 'ignore' }); try { + execFileSync('docker', ['create', '--name', current.name, '--label', 'io.codeboost.invocation=someone-else', + '--entrypoint', 'true', imageId], { stdio: 'ignore' }); await expectRejected(startProfileInvocation(current, { timeoutMs: 10 * 60_000 + 1 }), 'ceiling'); expect(isContainerProfileAuthentic(current)).toBe(false); expect(spawnSync('docker', ['container', 'inspect', current.name], { stdio: 'ignore' }).status).toBe(0); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index 3dc51bd5..f1946a83 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -21,7 +21,7 @@ export function removeRunnerObjects(runnerOwner: string): void { * or not: a test can make a network or volume with one owner and the container that holds it with another, or without * one. Containers and volumes go by force; a network another container joins meanwhile stays. It reports what it finds * and any Docker call that fails, prefixed with `when`, and never throws, so cleanup can call it without hiding an - * earlier error. Each call has a 30 s limit, so all ten can take up to 5 minutes. + * earlier error. Each call has a 30 s limit, so all eleven can take up to 5.5 minutes. */ export function removeLabelledObjects(label: string, value: string, when: string): void { const run = (args: string[]) => { @@ -37,8 +37,13 @@ export function removeLabelledObjects(label: string, value: string, when: string if (!ids.length) continue; console.warn(`${when} removes leftover ${kind}s labelled ${label}=${value}: ${ids.join(' ')}`); if (kind !== 'container') { - // Filters with one key match any of their values. - const holders = run(['ps', '--all', '--quiet', ...ids.flatMap(id => ['--filter', `${kind}=${id}`])]); + // Filters with one key match any of their values. The network filter also matches the start of another network's + // ID, so a container counts only if it is on one of these networks by name. + const listed = run(['ps', '--all', '--quiet', ...ids.flatMap(id => ['--filter', `${kind}=${id}`])]); + const holders = kind === 'volume' || !listed.length ? listed : run(['container', 'inspect', '--format', + '{{.Id}} {{range $name, $_ := .NetworkSettings.Networks}}{{$name}} {{end}}', ...listed]) + .map(line => line.split(' ').filter(Boolean)).filter(([, ...names]) => names.some(name => ids.includes(name))) + .map(([id]) => id!); if (holders.length) { console.warn(`${when} removes the containers that hold them: ${holders.join(' ')}`); run(['rm', '--force', ...holders]); diff --git a/test/runner-workspace.test.ts b/test/runner-workspace.test.ts index 91e78548..35da2761 100644 --- a/test/runner-workspace.test.ts +++ b/test/runner-workspace.test.ts @@ -39,7 +39,7 @@ afterAll(() => { // identity probe carries no owner labels, so it is not found here. It may not run before a hook timeout. removeRunnerObjects(RUNNER_OWNER); } -}, 4 * 60_000); +}, 6 * 60_000); const docker = (...args: string[]) => execFileSync('docker', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); async function setup(agent: Record) { From 489569e357e99e019774a21814ef57db77ec1a35 Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 18:32:05 -0700 Subject: [PATCH 05/75] Close the remaining label-less leaks the review found (#189) Review round 4: the rogue volume is made inside its try and removed after its container; the foreign network is listed again in finally; the holder test records each object before it exists; the sweep reads a partial inspect; the hook budgets cover eleven calls. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 68 +++++++++++++++++++++-------------- test/agent-network.test.ts | 4 +-- test/fixtures/docker-sweep.ts | 3 +- 3 files changed, 46 insertions(+), 29 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index cb9f0070..560e9670 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -149,11 +149,11 @@ afterAll(async () => { // makes under another owner or without the runner label are not found here, unless they hold one of this run's // volumes or networks: those made through `withoutRunnerLabel` are found by their test-run IDs, and the other tests // remove their own. It may not run before a hook timeout, and its budget does not cover every Docker call taking - // its full 30 s. + // its full 30 s (eleven per sweep). for (const testRun of testRuns) removeLabelledObjects(TEST_RUN_LABEL, testRun, 'afterAll'); removeRunnerObjects(TEST_RUNNER_OWNER); } -}, 120_000); +}, 10 * 60_000); describe('real Docker agent isolation', () => { it.each([ @@ -422,13 +422,19 @@ describe('real Docker agent isolation', () => { await expect(validateContainer(valid.name, valid)).rejects.toThrow('unexpected external mount'); docker('rm', '--force', valid.name); containers.delete(valid.name); - const rogue = `codeboost-work-${randomUUID()}`; docker('volume', 'create', rogue); + const rogue = `codeboost-work-${randomUUID()}`; try { + // Inside the try: the volume has no labels, so only the finally below removes it (#189). + docker('volume', 'create', rogue); const rogueArgs = valid.args.map(value => value.replace(data.filesystems.workVolume, rogue)); docker(...rogueArgs); containers.add(valid.name); await expect(validateContainer(valid.name, valid)).rejects.toThrow('captured identity'); docker('rm', '--force', valid.name); containers.delete(valid.name); - } finally { spawnSync('docker', ['volume', 'rm', '--force', rogue], { stdio: 'ignore' }); } + } finally { + // The container first: a volume it mounts cannot be removed. + spawnSync('docker', ['rm', '--force', valid.name], { stdio: 'ignore' }); + spawnSync('docker', ['volume', 'rm', '--force', rogue], { stdio: 'ignore' }); + } }, 60_000); it('rejects cloned profiles while sealed snapshots ignore later host changes', async () => { @@ -645,9 +651,9 @@ describe('real Docker agent isolation', () => { if (!prefixes.every(prefix => prefix.includes('create'))) throw new Error('Only create commands can be unlabelled.'); const testRun = randomUUID(); testRuns.push(testRun); - // Up to ten synchronous Docker calls of 30 s each, which a timer cannot cut short: the budget covers them, so a slow - // removal is not reported as a hook timeout (10 s by default). - onTestFinished(() => removeLabelledObjects(TEST_RUN_LABEL, testRun, 'onTestFinished'), 5 * 60_000); + // Up to eleven synchronous Docker calls of 30 s each, which a timer cannot cut short: the budget covers them, so a + // slow removal is not reported as a hook timeout (10 s by default). + onTestFinished(() => removeLabelledObjects(TEST_RUN_LABEL, testRun, 'onTestFinished'), 6 * 60_000); const shim = mkdtempSync(join(tmpdir(), 'codeboost-unlabelled-docker-')); roots.push(shim); mkdirSync(join(shim, 'input')); // afterAll resets this path's mode const realDocker = execFileSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).trim(); @@ -708,24 +714,27 @@ describe('real Docker agent isolation', () => { // #189: the sweep removes what holds a swept network or volume, even a container created and never started, and // nothing on a network or volume it does not sweep, even one whose ID starts with a swept network's name. it('removes the containers that hold a swept network or volume, and nothing else', () => { - const run = randomUUID(), label = `${TEST_RUN_LABEL}=${run}`, made: { kind: string; ref: string }[] = []; - const make = (kind: 'container' | 'network' | 'volume', ...args: string[]) => { - const ref = docker(...(kind === 'container' ? [] : [kind]), ...args); - made.push({ kind, ref }); - return ref; + // The test-run ID goes to afterAll too, and each object is recorded by name before it is made, so a failure here + // leaves nothing behind (#189). + const run = randomUUID(), label = `${TEST_RUN_LABEL}=${run}`, made: { kind: 'container' | 'network' | 'volume'; ref: string }[] = []; + testRuns.push(run); + const make = (kind: 'container' | 'network' | 'volume', name: string, ...args: string[]) => { + made.push({ kind, ref: name }); + return docker(...(kind === 'container' ? [args[0]!, '--name', name, ...args.slice(1)] : [kind, args[0]!, ...args.slice(1), name])); }; + const named = (what: string) => `codeboost-${what}-${randomUUID()}`; try { - const bystanderNetwork = make('network', 'create', `codeboost-bystander-${randomUUID()}`); - const bystanderVolume = make('volume', 'create', `codeboost-bystander-${randomUUID()}`); - const bystanders = [make('container', 'run', '--detach', '--network', bystanderNetwork, '--entrypoint', 'sleep', imageId, '300'), - make('container', 'create', '--mount', `type=volume,source=${bystanderVolume},target=/x`, '--entrypoint', 'true', imageId)]; - const network = make('network', 'create', '--label', label, `codeboost-swept-${randomUUID()}`); + const bystanderNetwork = make('network', named('bystander'), 'create'); + const bystanderVolume = make('volume', named('bystander'), 'create'); + const bystanders = [make('container', named('bystander'), 'run', '--detach', '--network', bystanderNetwork, '--entrypoint', 'sleep', imageId, '300'), + make('container', named('bystander'), 'create', '--mount', `type=volume,source=${bystanderVolume},target=/x`, '--entrypoint', 'true', imageId)]; + const network = make('network', named('swept'), 'create', '--label', label); // Named like the start of the bystander network's ID: Docker's network filter also matches that prefix. - make('network', 'create', '--label', label, bystanderNetwork.slice(0, 12)); - const volume = make('volume', 'create', '--label', label, `codeboost-swept-${randomUUID()}`); - const holders = [make('container', 'create', '--network', network, '--entrypoint', 'true', imageId), - make('container', 'run', '--detach', '--network', network, '--entrypoint', 'sleep', imageId, '300'), - make('container', 'create', '--mount', `type=volume,source=${volume},target=/x`, '--entrypoint', 'true', imageId)]; + make('network', bystanderNetwork.slice(0, 12), 'create', '--label', label); + const volume = make('volume', named('swept'), 'create', '--label', label); + const holders = [make('container', named('holder'), 'create', '--network', network, '--entrypoint', 'true', imageId), + make('container', named('holder'), 'run', '--detach', '--network', network, '--entrypoint', 'sleep', imageId, '300'), + make('container', named('holder'), 'create', '--mount', `type=volume,source=${volume},target=/x`, '--entrypoint', 'true', imageId)]; removeLabelledObjects(TEST_RUN_LABEL, run, 'test'); const exists = (kind: string, ref: string) => spawnSync('docker', [kind, 'inspect', ref], { stdio: 'ignore' }).status === 0; for (const holder of holders) expect(exists('container', holder)).toBe(false); @@ -733,14 +742,14 @@ describe('real Docker agent isolation', () => { for (const bystander of bystanders) expect(exists('container', bystander)).toBe(true); expect([exists('network', bystanderNetwork), exists('volume', bystanderVolume)]).toEqual([true, true]); } finally { - for (const { kind, ref } of made.filter(object => object.kind === 'container')) + for (const { ref } of made.filter(object => object.kind === 'container')) spawnSync('docker', ['rm', '--force', ref], { stdio: 'ignore' }); for (const { kind, ref } of made.filter(object => object.kind !== 'container')) spawnSync('docker', [kind, 'rm', ...(kind === 'volume' ? ['--force'] : []), ref], { stdio: 'ignore' }); } }, 120_000); - // #189: these two run in order. The first stops after making a keeper without its runner label, before the removal + // #189: these two run in order (run alone, the second fails: it checks what the first left). The first stops after making a keeper without its runner label, before the removal // its sibling test above ends with; the second checks that the keeper went when the first finished. `it.fails` passes // on any failure, so the first records the storage only once its checks passed, just before its deliberate stop. let stoppedEarly: ReturnType | undefined; @@ -830,13 +839,20 @@ describe('real Docker agent isolation', () => { ].join('\n'); const created = await withDockerShim(['network', 'create'], foreign, () => createVendorNetwork(invocation(data.clone, 'planning'), imageId, allocationId).then(() => undefined, error => error)); - const left = docker('network', 'ls', '--quiet', '--filter', `label=io.codeboost.allocation=${allocationId}`); + // Listed in the finally too: the network carries another runner's label, so no sweep of this run finds it (#189). + const listed = () => spawnSync('docker', ['network', 'ls', '--quiet', '--filter', `label=io.codeboost.allocation=${allocationId}`], + { encoding: 'utf8' }).stdout.trim(); + let left = ''; try { + left = listed(); expect(created).toBeInstanceOf(VendorNetworkCreationCleanupError); const cleanup = (created as VendorNetworkCreationCleanupError).errors[1] as AggregateError; expect(cleanup.errors.map(error => (error as Error).message)).toEqual(['Refused to remove unowned vendor network.']); expect(left).not.toBe(''); - } finally { if (left) spawnSync('docker', ['network', 'rm', ...left.split('\n')], { stdio: 'ignore' }); } + } finally { + const remaining = listed(); + if (remaining) spawnSync('docker', ['network', 'rm', ...remaining.split('\n')], { stdio: 'ignore' }); + } }, 60_000); it('never removes an agent container found by name after a killed create when a label is missing', async () => { diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index 56627b31..6db5caf9 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -38,8 +38,8 @@ afterAll(async () => { try { if (network) await removeVendorNetwork(network); } finally { // Then remove whatever this run still owns, even when a test failed or the removal above threw. Objects a test - // makes without the runner label are not found here; those tests remove their own. It may not run before a hook - // timeout. + // makes without the runner label are found here only while they hold one of this run's networks or volumes; those + // tests remove their own. It may not run before a hook timeout. removeRunnerObjects(TEST_RUNNER_OWNER); } }, 6 * 60_000); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index f1946a83..36213c51 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -28,7 +28,8 @@ export function removeLabelledObjects(label: string, value: string, when: string const result = spawnSync('docker', args, { encoding: 'utf8', timeout: 30_000, killSignal: 'SIGKILL' }); if (result.status !== 0) console.warn(`${when}: docker ${args.slice(0, 2).join(' ')} failed:`, result.error?.message ?? result.stderr.trim()); - return result.status === 0 ? result.stdout.split('\n').filter(Boolean) : []; + // An inspect of several objects exits 1 if one is gone, yet prints the rest, so stdout is read whatever the status. + return result.stdout?.split('\n').filter(Boolean) ?? []; }; for (const kind of ['container', 'network', 'volume'] as const) { // Networks by name: `ps --filter network=` finds a created, never started container by the name only (Docker 27.4). From d0556259a0cb4ef27bf05c8c7f3e09641065c92c Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 18:44:17 -0700 Subject: [PATCH 06/75] Record label-less objects first in agent-network; remove networks by ID (#189) Review round 5: the network and proxy shims record a name before they make it, and the tests remove it in finally; the sweep removes networks by full ID, reads output only from a call that exited by itself, and the holder test runs its inspect with a missing ID, as the sweep meets it; the supervisor's finally removals never throw. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 19 ++++++++++++++---- test/agent-network.test.ts | 36 +++++++++++++++++++++++------------ test/agent-supervisor.test.ts | 12 +++++++----- test/fixtures/docker-sweep.ts | 24 +++++++++++++---------- 4 files changed, 60 insertions(+), 31 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 560e9670..d2f04df9 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -735,7 +735,17 @@ describe('real Docker agent isolation', () => { const holders = [make('container', named('holder'), 'create', '--network', network, '--entrypoint', 'true', imageId), make('container', named('holder'), 'run', '--detach', '--network', network, '--entrypoint', 'sleep', imageId, '300'), make('container', named('holder'), 'create', '--mount', `type=volume,source=${volume},target=/x`, '--entrypoint', 'true', imageId)]; - removeLabelledObjects(TEST_RUN_LABEL, run, 'test'); + // The holders' inspect also names a container that does not exist, as one gone since the listing would be: Docker + // prints the others and exits 1, and the sweep must still remove them. + const shim = mkdtempSync(join(tmpdir(), 'codeboost-sweep-docker-')); roots.push(shim); + mkdirSync(join(shim, 'input')); // afterAll resets this path's mode + const realDocker = execFileSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).trim(); + writeFileSync(join(shim, 'docker'), ['#!/bin/sh', + `if [ "$1 $2" = "container inspect" ]; then exec '${realDocker}' "$@" codeboost-gone-${randomUUID()}; fi`, + `exec '${realDocker}' "$@"`].join('\n'), { mode: 0o755 }); + const path = process.env.PATH; + process.env.PATH = `${shim}:${path}`; + try { removeLabelledObjects(TEST_RUN_LABEL, run, 'test'); } finally { process.env.PATH = path; } const exists = (kind: string, ref: string) => spawnSync('docker', [kind, 'inspect', ref], { stdio: 'ignore' }).status === 0; for (const holder of holders) expect(exists('container', holder)).toBe(false); expect([exists('network', network), exists('volume', volume)]).toEqual([false, false]); @@ -749,9 +759,10 @@ describe('real Docker agent isolation', () => { } }, 120_000); - // #189: these two run in order (run alone, the second fails: it checks what the first left). The first stops after making a keeper without its runner label, before the removal - // its sibling test above ends with; the second checks that the keeper went when the first finished. `it.fails` passes - // on any failure, so the first records the storage only once its checks passed, just before its deliberate stop. + // #189: these two run in order (run alone, the second fails: it checks what the first left). The first stops after + // making a keeper without its runner label, before the removal that "rejects a keeper without its runner label" + // ends with; the second checks that the keeper went when the first finished. `it.fails` passes on any failure, so the + // first records the storage only once its checks passed, just before its deliberate stop. let stoppedEarly: ReturnType | undefined; it.fails('stops after making a keeper without its runner label (expected to fail)', async () => { const data = await withoutRunnerLabel([['create', '--name']], () => fixture()); diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index 6db5caf9..f780e99c 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -89,20 +89,26 @@ describe('vendor-only egress', () => { // The name is taken by someone else's network just before our create, so the daemon refuses it. writeFileSync(join(shim, 'docker'), ['#!/bin/sh', `if [ "$1" = network ] && [ "$2" = create ]; then for arg; do last="$arg"; done; ` - + `'${realDocker}' network create "$last" >/dev/null; printf %s "$last" > '${recorded}'; ` + + `printf %s "$last" > '${recorded}'; '${realDocker}' network create "$last" >/dev/null; ` + `echo "Error response from daemon: network with name $last already exists" >&2; exit 1; fi`, `exec '${realDocker}' "$@"`].join('\n'), { mode: 0o755 }); const refused = captureInvocation({ ...invocation, attemptId: `refused-network-${randomUUID()}`, deadline: Date.now() + 60_000 }); const path = process.env.PATH; process.env.PATH = `${shim}:${path}`; - let foreign = '', error: unknown; + let foreign = '', error: unknown, survived = false; try { try { await createVendorNetwork(refused, imageId, randomUUID()); } catch (caught) { error = caught; } - foreign = readFileSync(recorded, 'utf8'); - } finally { process.env.PATH = path; rmSync(shim, { recursive: true, force: true }); } - const survived = spawnSync('docker', ['network', 'inspect', foreign], { stdio: 'ignore' }).status === 0; - spawnSync('docker', ['network', 'rm', foreign], { stdio: 'ignore' }); + } finally { + process.env.PATH = path; + // Recorded before it is made, and removed here: the network has no labels, so no sweep finds it (#189). + foreign = existsSync(recorded) ? readFileSync(recorded, 'utf8') : ''; + if (foreign) { + survived = spawnSync('docker', ['network', 'inspect', foreign], { stdio: 'ignore' }).status === 0; + spawnSync('docker', ['network', 'rm', foreign], { stdio: 'ignore' }); + } + rmSync(shim, { recursive: true, force: true }); + } // The refused create is reported as itself, with no cleanup left to retry, and the foreign network survives. expect(error).toBeInstanceOf(Error); expect(error).not.toBeInstanceOf(VendorNetworkCreationCleanupError); @@ -139,19 +145,25 @@ describe('vendor-only egress', () => { // Someone else's container takes the proxy name just before our create, so the daemon refuses it. writeFileSync(join(shim, 'docker'), ['#!/bin/sh', `if [ "$1" = create ] && [ "$2" = --name ] && case "$3" in codeboost-proxy-*) true;; *) false;; esac; then ` - + `'${realDocker}' create --name "$3" --entrypoint true ${imageId} >/dev/null; printf %s "$3" > '${recorded}'; fi`, + + `printf %s "$3" > '${recorded}'; '${realDocker}' create --name "$3" --entrypoint true ${imageId} >/dev/null; fi`, `exec '${realDocker}' "$@"`].join('\n'), { mode: 0o755 }); const refused = captureInvocation({ ...invocation, attemptId: `refused-proxy-${randomUUID()}`, deadline: Date.now() + 60_000 }); const path = process.env.PATH; process.env.PATH = `${shim}:${path}`; - let foreign = '', error: unknown; + let foreign = '', error: unknown, survived = false; try { try { await createVendorNetwork(refused, imageId, randomUUID()); } catch (caught) { error = caught; } - foreign = readFileSync(recorded, 'utf8'); - } finally { process.env.PATH = path; rmSync(shim, { recursive: true, force: true }); } - const survived = spawnSync('docker', ['container', 'inspect', foreign], { stdio: 'ignore' }).status === 0; - spawnSync('docker', ['rm', '--force', foreign], { stdio: 'ignore' }); + } finally { + process.env.PATH = path; + // Recorded before it is made, and removed here: the container has no labels, so no sweep finds it (#189). + foreign = existsSync(recorded) ? readFileSync(recorded, 'utf8') : ''; + if (foreign) { + survived = spawnSync('docker', ['container', 'inspect', foreign], { stdio: 'ignore' }).status === 0; + spawnSync('docker', ['rm', '--force', foreign], { stdio: 'ignore' }); + } + rmSync(shim, { recursive: true, force: true }); + } // The refused create is reported as itself: our network was removed, nothing is left to retry or report. expect(error).toBeInstanceOf(Error); expect(error).not.toBeInstanceOf(VendorNetworkCreationCleanupError); diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 66504d15..d83f43ad 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -399,7 +399,8 @@ describe('container invocation supervisor', () => { expect(execFileSync('docker', ['container', 'inspect', '--format', '{{index .Config.Labels "io.codeboost.invocation"}}', current.name], { encoding: 'utf8' }).trim()).toBe('foreign'); } finally { - execFileSync('docker', ['rm', '--force', current.name], { stdio: 'ignore' }); + // Never throws: the create may have failed before Docker made anything. + spawnSync('docker', ['rm', '--force', current.name], { stdio: 'ignore' }); } }, 3 * 60_000); @@ -642,11 +643,12 @@ describe('container invocation supervisor', () => { it('writes the deferred output acknowledgement idempotently, repairing a partial one', () => { const token = randomUUID(), file = `/run/codeboost-control/collected-${token}`; const name = `codeboost-ack-script-${randomUUID()}`; - execFileSync('docker', ['run', '-d', '--rm', '--name', name, '--network', 'none', '--cap-drop=ALL', - '--security-opt=no-new-privileges', '--user', '10001:10001', '--entrypoint', 'sleep', - '--tmpfs', '/run/codeboost-control:rw,nosuid,nodev,noexec,size=65536,nr_inodes=16,uid=0,gid=0,mode=0711', - imageId, '120'], { stdio: 'ignore' }); try { + // Inside the try: the container has no labels, so only the finally below removes it (#189). + execFileSync('docker', ['run', '-d', '--rm', '--name', name, '--network', 'none', '--cap-drop=ALL', + '--security-opt=no-new-privileges', '--user', '10001:10001', '--entrypoint', 'sleep', + '--tmpfs', '/run/codeboost-control:rw,nosuid,nodev,noexec,size=65536,nr_inodes=16,uid=0,gid=0,mode=0711', + imageId, '120'], { stdio: 'ignore' }); const acknowledge = () => spawnSync('docker', ['exec', '--user', '0', name, 'node', '-e', ACKNOWLEDGEMENT_SCRIPT, token], { encoding: 'utf8' }); const content = () => execFileSync('docker', ['exec', name, 'cat', file], { encoding: 'utf8' }); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index 36213c51..3367efee 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -28,22 +28,26 @@ export function removeLabelledObjects(label: string, value: string, when: string const result = spawnSync('docker', args, { encoding: 'utf8', timeout: 30_000, killSignal: 'SIGKILL' }); if (result.status !== 0) console.warn(`${when}: docker ${args.slice(0, 2).join(' ')} failed:`, result.error?.message ?? result.stderr.trim()); - // An inspect of several objects exits 1 if one is gone, yet prints the rest, so stdout is read whatever the status. - return result.stdout?.split('\n').filter(Boolean) ?? []; + // An inspect of several objects exits 1 if one is gone, yet prints the rest, so the output of a call that exited by + // itself is read whatever its status. A killed call's output may be cut short, so it is not. + return result.error || result.status === null ? [] : result.stdout.split('\n').filter(Boolean); }; for (const kind of ['container', 'network', 'volume'] as const) { - // Networks by name: `ps --filter network=` finds a created, never started container by the name only (Docker 27.4). - const ids = run([kind, 'ls', ...(kind === 'container' ? ['--all'] : []), ...(kind === 'network' ? ['--format', '{{.Name}}'] - : ['--quiet']), '--filter', `label=${label}=${value}`]); + // Networks with their names: `ps --filter network=` finds a created, never started container by the name only + // (Docker 27.4). Each is removed by its full ID, which no other network's name can be mistaken for. + const listed = run([kind, 'ls', ...(kind === 'container' ? ['--all'] : []), ...(kind === 'network' + ? ['--no-trunc', '--format', '{{.ID}} {{.Name}}'] : ['--quiet']), '--filter', `label=${label}=${value}`]) + .map(line => line.split(' ')); + const ids = listed.map(([id]) => id!), names = listed.map(([id, name]) => name ?? id!); if (!ids.length) continue; - console.warn(`${when} removes leftover ${kind}s labelled ${label}=${value}: ${ids.join(' ')}`); + console.warn(`${when} removes leftover ${kind}s labelled ${label}=${value}: ${names.join(' ')}`); if (kind !== 'container') { // Filters with one key match any of their values. The network filter also matches the start of another network's // ID, so a container counts only if it is on one of these networks by name. - const listed = run(['ps', '--all', '--quiet', ...ids.flatMap(id => ['--filter', `${kind}=${id}`])]); - const holders = kind === 'volume' || !listed.length ? listed : run(['container', 'inspect', '--format', - '{{.Id}} {{range $name, $_ := .NetworkSettings.Networks}}{{$name}} {{end}}', ...listed]) - .map(line => line.split(' ').filter(Boolean)).filter(([, ...names]) => names.some(name => ids.includes(name))) + const candidates = run(['ps', '--all', '--quiet', ...names.flatMap(name => ['--filter', `${kind}=${name}`])]); + const holders = kind === 'volume' || !candidates.length ? candidates : run(['container', 'inspect', '--format', + '{{.Id}} {{range $name, $_ := .NetworkSettings.Networks}}{{$name}} {{end}}', ...candidates]) + .map(line => line.split(' ').filter(Boolean)).filter(([, ...on]) => on.some(name => names.includes(name))) .map(([id]) => id!); if (holders.length) { console.warn(`${when} removes the containers that hold them: ${holders.join(' ')}`); From e6fd22c027656438be2054b452d2197fea7d2121 Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 18:55:41 -0700 Subject: [PATCH 07/75] Test removal by full network ID; say what the runner sweep removes (#189) Review round 6. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 7 +++++-- test/fixtures/docker-sweep.ts | 9 +++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index d2f04df9..d3c36d87 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -732,11 +732,13 @@ describe('real Docker agent isolation', () => { // Named like the start of the bystander network's ID: Docker's network filter also matches that prefix. make('network', bystanderNetwork.slice(0, 12), 'create', '--label', label); const volume = make('volume', named('swept'), 'create', '--label', label); + // And a bystander named like the start of the swept network's ID: removing by a short ID would remove it. + const lookalike = make('network', network.slice(0, 12), 'create'); const holders = [make('container', named('holder'), 'create', '--network', network, '--entrypoint', 'true', imageId), make('container', named('holder'), 'run', '--detach', '--network', network, '--entrypoint', 'sleep', imageId, '300'), make('container', named('holder'), 'create', '--mount', `type=volume,source=${volume},target=/x`, '--entrypoint', 'true', imageId)]; // The holders' inspect also names a container that does not exist, as one gone since the listing would be: Docker - // prints the others and exits 1, and the sweep must still remove them. + // 27.4 prints the others and exits 1, and the sweep must still remove them. const shim = mkdtempSync(join(tmpdir(), 'codeboost-sweep-docker-')); roots.push(shim); mkdirSync(join(shim, 'input')); // afterAll resets this path's mode const realDocker = execFileSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).trim(); @@ -750,7 +752,8 @@ describe('real Docker agent isolation', () => { for (const holder of holders) expect(exists('container', holder)).toBe(false); expect([exists('network', network), exists('volume', volume)]).toEqual([false, false]); for (const bystander of bystanders) expect(exists('container', bystander)).toBe(true); - expect([exists('network', bystanderNetwork), exists('volume', bystanderVolume)]).toEqual([true, true]); + expect([exists('network', bystanderNetwork), exists('volume', bystanderVolume), exists('network', lookalike)]) + .toEqual([true, true, true]); } finally { for (const { ref } of made.filter(object => object.kind === 'container')) spawnSync('docker', ['rm', '--force', ref], { stdio: 'ignore' }); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index 3367efee..fd33b97e 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -8,8 +8,9 @@ import { RUNNER_LABEL } from '../../agents/labels.ts'; export const TEST_RUN_LABEL = 'io.codeboost.test-run'; /** - * Removes the containers, networks and volumes labelled with one runner owner (see `removeLabelledObjects`). A test - * file's afterAll calls it in a finally block. + * Removes the containers, networks and volumes labelled with one runner owner, and every container that holds one of + * those networks or volumes, whatever its labels (see `removeLabelledObjects`). A test file's afterAll calls it in a + * finally block. */ export function removeRunnerObjects(runnerOwner: string): void { removeLabelledObjects(RUNNER_LABEL, runnerOwner, 'afterAll'); @@ -28,8 +29,8 @@ export function removeLabelledObjects(label: string, value: string, when: string const result = spawnSync('docker', args, { encoding: 'utf8', timeout: 30_000, killSignal: 'SIGKILL' }); if (result.status !== 0) console.warn(`${when}: docker ${args.slice(0, 2).join(' ')} failed:`, result.error?.message ?? result.stderr.trim()); - // An inspect of several objects exits 1 if one is gone, yet prints the rest, so the output of a call that exited by - // itself is read whatever its status. A killed call's output may be cut short, so it is not. + // An inspect of several objects exits 1 if one is gone, yet prints the rest (Docker 27.4), so the output of a call + // that exited by itself is read whatever its status. A killed call's output may be cut short, so it is not. return result.error || result.status === null ? [] : result.stdout.split('\n').filter(Boolean); }; for (const kind of ['container', 'network', 'volume'] as const) { From 1fadd86eb59243fb6d7d899092438512b65240f4 Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 19:04:05 -0700 Subject: [PATCH 08/75] Remove swept containers by full ID; track the blocker before it is made (#189) Review round 7: Docker resolves a container name before an ID prefix, so the sweep lists containers and holders with --no-trunc, and the holder test keeps a container named like a holder's short ID; the blocker is tracked before its run; comment wording. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 10 +++++++--- test/fixtures/docker-sweep.ts | 9 ++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 8425b757..158c9965 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -610,8 +610,9 @@ describe('real Docker agent isolation', () => { await createValidatedContainer(live); const blocker = `codeboost-blocker-${randomUUID()}`, replacement = live.name; // Our container goes, but a foreign endpoint keeps the network busy, so the profile cleanup fails and is retried. - docker('run', '--detach', '--name', blocker, '--network', live.network.name, '--entrypoint', 'sleep', imageId, '300'); + // Tracked before it is made: it has no labels, and once its network is gone no sweep finds it (#189). containers.add(blocker); + docker('run', '--detach', '--name', blocker, '--network', live.network.name, '--entrypoint', 'sleep', imageId, '300'); await expect(disposeValidatedContainer(live)).rejects.toThrow('did not settle'); expect(isContainerProfileAuthentic(live)).toBe(true); // A same-named, same-labelled container appears before the retry. @@ -762,6 +763,8 @@ describe('real Docker agent isolation', () => { const holders = [make('container', named('holder'), 'create', '--network', network, '--entrypoint', 'true', imageId), make('container', named('holder'), 'run', '--detach', '--network', network, '--entrypoint', 'sleep', imageId, '300'), make('container', named('holder'), 'create', '--mount', `type=volume,source=${volume},target=/x`, '--entrypoint', 'true', imageId)]; + // The same for containers: Docker resolves a name before an ID prefix. + const holderLookalike = make('container', holders[0]!.slice(0, 12), 'create', '--entrypoint', 'true', imageId); // The holders' inspect also names a container that does not exist, as one gone since the listing would be: Docker // 27.4 prints the others and exits 1, and the sweep must still remove them. const shim = mkdtempSync(join(tmpdir(), 'codeboost-sweep-docker-')); roots.push(shim); @@ -776,7 +779,7 @@ describe('real Docker agent isolation', () => { const exists = (kind: string, ref: string) => spawnSync('docker', [kind, 'inspect', ref], { stdio: 'ignore' }).status === 0; for (const holder of holders) expect(exists('container', holder)).toBe(false); expect([exists('network', network), exists('volume', volume)]).toEqual([false, false]); - for (const bystander of bystanders) expect(exists('container', bystander)).toBe(true); + for (const bystander of [...bystanders, holderLookalike]) expect(exists('container', bystander)).toBe(true); expect([exists('network', bystanderNetwork), exists('volume', bystanderVolume), exists('network', lookalike)]) .toEqual([true, true, true]); } finally { @@ -805,7 +808,8 @@ describe('real Docker agent isolation', () => { it('removed the keeper that a test without its runner label left when it stopped', () => { expect(stoppedEarly, 'the test before did not reach its deliberate stop').toBeDefined(); const left = stoppedEarly!.filesystems, allocationId = taskFilesystemOwner(left).allocationId; - // First, so afterAll does not refuse a keeper this test finds left; its runner sweep removes what is. + // First, so afterAll does not refuse a keeper this test finds left: its test-run sweep removes that keeper, then its + // runner sweep the volumes. taskFilesystems.splice(taskFilesystems.indexOf(left), 1); expect(spawnSync('docker', ['container', 'inspect', left.keeper], { stdio: 'ignore' }).status).not.toBe(0); // Its volumes carry the runner label, so ordinary cleanup can remove them now that no keeper holds them. diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index fd33b97e..0d404426 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -37,15 +37,18 @@ export function removeLabelledObjects(label: string, value: string, when: string // Networks with their names: `ps --filter network=` finds a created, never started container by the name only // (Docker 27.4). Each is removed by its full ID, which no other network's name can be mistaken for. const listed = run([kind, 'ls', ...(kind === 'container' ? ['--all'] : []), ...(kind === 'network' - ? ['--no-trunc', '--format', '{{.ID}} {{.Name}}'] : ['--quiet']), '--filter', `label=${label}=${value}`]) + ? ['--no-trunc', '--format', '{{.ID}} {{.Name}}'] : kind === 'container' ? ['--quiet', '--no-trunc'] : ['--quiet']), + '--filter', `label=${label}=${value}`]) .map(line => line.split(' ')); const ids = listed.map(([id]) => id!), names = listed.map(([id, name]) => name ?? id!); if (!ids.length) continue; console.warn(`${when} removes leftover ${kind}s labelled ${label}=${value}: ${names.join(' ')}`); if (kind !== 'container') { // Filters with one key match any of their values. The network filter also matches the start of another network's - // ID, so a container counts only if it is on one of these networks by name. - const candidates = run(['ps', '--all', '--quiet', ...names.flatMap(name => ['--filter', `${kind}=${name}`])]); + // ID, so a container counts only if it is on one of these networks by name (Docker refuses a second local network + // of one name). Containers go by full ID: Docker resolves a name before an ID prefix. + const candidates = run(['ps', '--all', '--quiet', '--no-trunc', + ...names.flatMap(name => ['--filter', `${kind}=${name}`])]); const holders = kind === 'volume' || !candidates.length ? candidates : run(['container', 'inspect', '--format', '{{.Id}} {{range $name, $_ := .NetworkSettings.Networks}}{{$name}} {{end}}', ...candidates]) .map(line => line.split(' ').filter(Boolean)).filter(([, ...on]) => on.some(name => names.includes(name))) From faf549d89a523f723854820c2b772ac311ff1c7a Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 19:12:57 -0700 Subject: [PATCH 09/75] Remove the foreign-runner storage in its test; clean every storage in afterAll (#189) Review round 8: the storage made under another runner is removed in finally; afterAll removes every storage even when one fails; the killed-create network is listed by full ID; afterAll budgets have a minute over the sweep. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 19 +++++++++++++------ test/agent-network.test.ts | 3 ++- test/agent-supervisor.test.ts | 3 ++- test/runner-workspace.test.ts | 3 ++- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 158c9965..6b5a7552 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -137,13 +137,18 @@ afterEach(async () => { afterAll(async () => { try { for (const container of containers) spawnSync('docker', ['rm', '--force', container], { stdio: 'ignore' }); - for (const filesystems of taskFilesystems.reverse()) removeTaskFilesystems(filesystems); + // Each one, even when an earlier one fails: storage left here is found by the sweep below only by its label. + const failures: unknown[] = []; + for (const filesystems of taskFilesystems.reverse()) { + try { removeTaskFilesystems(filesystems); } catch (error) { failures.push(error); } + } for (const profile of profiles) await disposeContainerProfile(profile); for (const network of vendorNetworks.splice(0).reverse()) await removeVendorNetwork(network); for (const root of roots.reverse()) { chmodSync(join(root, 'input'), 0o700); rmSync(root, { recursive: true, force: true, maxRetries: 3, retryDelay: 50 }); } + if (failures.length) throw new AggregateError(failures, 'Task storage cleanup did not settle.'); } finally { // The runner owner is this run's own, so nothing else would find what this run leaves behind. Objects a test // makes under another owner or without the runner label are not found here, unless they hold one of this run's @@ -883,8 +888,8 @@ describe('real Docker agent isolation', () => { const created = await withDockerShim(['network', 'create'], foreign, () => createVendorNetwork(invocation(data.clone, 'planning'), imageId, allocationId).then(() => undefined, error => error)); // Listed in the finally too: the network carries another runner's label, so no sweep of this run finds it (#189). - const listed = () => spawnSync('docker', ['network', 'ls', '--quiet', '--filter', `label=io.codeboost.allocation=${allocationId}`], - { encoding: 'utf8' }).stdout.trim(); + const listed = () => spawnSync('docker', ['network', 'ls', '--quiet', '--no-trunc', '--filter', + `label=io.codeboost.allocation=${allocationId}`], { encoding: 'utf8' }).stdout?.trim() ?? ''; let left = ''; try { left = listed(); @@ -2495,9 +2500,11 @@ describe('real Docker agent isolation', () => { const foreign = prepareTaskFilesystems(data.clone, { workBytes: 16 * 1024 * 1024, workInodes: 512, metadataBytes: 16 * 1024 * 1024, metadataInodes: 512 }, imageId, { runnerOwner: 'f'.repeat(32), attemptId: 'other-runner', allocationId: randomUUID() }); - taskFilesystems.push(foreign); - await expect(createContainerProfile({ ...await governed(invocation(data.clone, 'planning')), filesystems: foreign, - inputDirectory: data.input, codexAuthFile: data.fakeAuth, imageId })).rejects.toThrow('another runner'); + // Removed here: it carries another runner's label, so no sweep of this run finds it (#189). + try { + await expect(createContainerProfile({ ...await governed(invocation(data.clone, 'planning')), filesystems: foreign, + inputDirectory: data.input, codexAuthFile: data.fakeAuth, imageId })).rejects.toThrow('another runner'); + } finally { removeTaskFilesystems(foreign); } }, 60_000); it('refuses to seed a clone whose staging directory was replaced after creation', async () => { diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index f780e99c..eee872c2 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -40,9 +40,10 @@ afterAll(async () => { // Then remove whatever this run still owns, even when a test failed or the removal above threw. Objects a test // makes without the runner label are found here only while they hold one of this run's networks or volumes; those // tests remove their own. It may not run before a hook timeout. + // The hook's budget is a minute over this sweep's worst case (eleven calls of 30 s), for the cleanup before it. removeRunnerObjects(TEST_RUNNER_OWNER); } -}, 6 * 60_000); +}, 7 * 60_000); describe('vendor-only egress', () => { it('keeps failed allocation and its cleanup inside the caller deadline', async () => { diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index d83f43ad..1337f257 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -79,9 +79,10 @@ afterAll(async () => { } finally { // Then remove whatever this run still owns, even when a test failed or the cleanup above threw: an agent that // ignores SIGTERM, or storage whose removal did not settle. It may not run before a hook timeout. + // The hook's budget is a minute over this sweep's worst case (eleven calls of 30 s), for the cleanup before it. removeRunnerObjects(TEST_RUNNER_OWNER); } -}, 6 * 60_000); +}, 7 * 60_000); describe('container invocation supervisor', () => { it('preserves the first cancellation reason while retained setup cleanup settles', async () => { diff --git a/test/runner-workspace.test.ts b/test/runner-workspace.test.ts index 35da2761..03520a19 100644 --- a/test/runner-workspace.test.ts +++ b/test/runner-workspace.test.ts @@ -37,9 +37,10 @@ afterAll(() => { } finally { // Then remove the task storage this run still owns, even when a test failed or the cleanup above threw. The path // identity probe carries no owner labels, so it is not found here. It may not run before a hook timeout. + // The hook's budget is a minute over this sweep's worst case (eleven calls of 30 s), for the cleanup before it. removeRunnerObjects(RUNNER_OWNER); } -}, 6 * 60_000); +}, 7 * 60_000); const docker = (...args: string[]) => execFileSync('docker', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); async function setup(agent: Record) { From 8efbd1bfbd7fb749e64c045220678db7850ff39a Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 19:20:07 -0700 Subject: [PATCH 10/75] Read network names as JSON; name the seccomp probe; test --no-trunc (#189) Review round 9: a network name may hold spaces, so the sweep reads names and holders' networks as JSON; the label-less seccomp probe is named and tracked before it runs; the holder test keeps a container named like a swept container's short ID; budget comments state the real margin. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 12 +++++++++--- test/agent-network.test.ts | 3 ++- test/agent-supervisor.test.ts | 3 ++- test/fixtures/docker-sweep.ts | 14 ++++++++------ test/runner-workspace.test.ts | 3 ++- 5 files changed, 23 insertions(+), 12 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 6b5a7552..dbce264f 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -768,8 +768,10 @@ describe('real Docker agent isolation', () => { const holders = [make('container', named('holder'), 'create', '--network', network, '--entrypoint', 'true', imageId), make('container', named('holder'), 'run', '--detach', '--network', network, '--entrypoint', 'sleep', imageId, '300'), make('container', named('holder'), 'create', '--mount', `type=volume,source=${volume},target=/x`, '--entrypoint', 'true', imageId)]; - // The same for containers: Docker resolves a name before an ID prefix. + // The same for containers: Docker resolves a name before an ID prefix. A holder and a swept container each have one. const holderLookalike = make('container', holders[0]!.slice(0, 12), 'create', '--entrypoint', 'true', imageId); + const labelled = make('container', named('swept'), 'create', '--label', label, '--entrypoint', 'true', imageId); + const labelledLookalike = make('container', labelled.slice(0, 12), 'create', '--entrypoint', 'true', imageId); // The holders' inspect also names a container that does not exist, as one gone since the listing would be: Docker // 27.4 prints the others and exits 1, and the sweep must still remove them. const shim = mkdtempSync(join(tmpdir(), 'codeboost-sweep-docker-')); roots.push(shim); @@ -784,7 +786,8 @@ describe('real Docker agent isolation', () => { const exists = (kind: string, ref: string) => spawnSync('docker', [kind, 'inspect', ref], { stdio: 'ignore' }).status === 0; for (const holder of holders) expect(exists('container', holder)).toBe(false); expect([exists('network', network), exists('volume', volume)]).toEqual([false, false]); - for (const bystander of [...bystanders, holderLookalike]) expect(exists('container', bystander)).toBe(true); + expect(exists('container', labelled)).toBe(false); + for (const bystander of [...bystanders, holderLookalike, labelledLookalike]) expect(exists('container', bystander)).toBe(true); expect([exists('network', bystanderNetwork), exists('volume', bystanderVolume), exists('network', lookalike)]) .toEqual([true, true, true]); } finally { @@ -2526,7 +2529,10 @@ describe('real Docker agent isolation', () => { }, 60_000); it('startup probe refuses to exec when seccomp filtering is disabled', async () => { - const result = spawnSync('docker', ['run', '--rm', '--read-only', '--user', '10001:10001', '--cap-drop=ALL', + // Named and tracked before it is made: it has no labels, so afterAll's removal by name is the only one (#189). + const name = `codeboost-seccomp-probe-${randomUUID()}`; + containers.add(name); + const result = spawnSync('docker', ['run', '--rm', '--name', name, '--read-only', '--user', '10001:10001', '--cap-drop=ALL', '--security-opt=no-new-privileges', '--security-opt=seccomp=unconfined', '--network=none', imageId, 'true'], { encoding: 'utf8', timeout: 60_000 }); expect(result.status).toBe(78); diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index eee872c2..a686d5f2 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -40,7 +40,8 @@ afterAll(async () => { // Then remove whatever this run still owns, even when a test failed or the removal above threw. Objects a test // makes without the runner label are found here only while they hold one of this run's networks or volumes; those // tests remove their own. It may not run before a hook timeout. - // The hook's budget is a minute over this sweep's worst case (eleven calls of 30 s), for the cleanup before it. + // The hook's 7 minutes leave a minute and a half over this sweep's worst case (eleven calls of 30 s), for the + // cleanup before it. removeRunnerObjects(TEST_RUNNER_OWNER); } }, 7 * 60_000); diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 1337f257..6d94e551 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -79,7 +79,8 @@ afterAll(async () => { } finally { // Then remove whatever this run still owns, even when a test failed or the cleanup above threw: an agent that // ignores SIGTERM, or storage whose removal did not settle. It may not run before a hook timeout. - // The hook's budget is a minute over this sweep's worst case (eleven calls of 30 s), for the cleanup before it. + // The hook's 7 minutes leave a minute and a half over this sweep's worst case (eleven calls of 30 s), for the + // cleanup before it. removeRunnerObjects(TEST_RUNNER_OWNER); } }, 7 * 60_000); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index 0d404426..d26835ff 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -36,11 +36,12 @@ export function removeLabelledObjects(label: string, value: string, when: string for (const kind of ['container', 'network', 'volume'] as const) { // Networks with their names: `ps --filter network=` finds a created, never started container by the name only // (Docker 27.4). Each is removed by its full ID, which no other network's name can be mistaken for. + // As JSON: a network's name may hold spaces. const listed = run([kind, 'ls', ...(kind === 'container' ? ['--all'] : []), ...(kind === 'network' - ? ['--no-trunc', '--format', '{{.ID}} {{.Name}}'] : kind === 'container' ? ['--quiet', '--no-trunc'] : ['--quiet']), + ? ['--no-trunc', '--format', '{{json .}}'] : kind === 'container' ? ['--quiet', '--no-trunc'] : ['--quiet']), '--filter', `label=${label}=${value}`]) - .map(line => line.split(' ')); - const ids = listed.map(([id]) => id!), names = listed.map(([id, name]) => name ?? id!); + .map(line => kind === 'network' ? JSON.parse(line) as { ID: string; Name: string } : { ID: line, Name: line }); + const ids = listed.map(object => object.ID), names = listed.map(object => object.Name); if (!ids.length) continue; console.warn(`${when} removes leftover ${kind}s labelled ${label}=${value}: ${names.join(' ')}`); if (kind !== 'container') { @@ -50,9 +51,10 @@ export function removeLabelledObjects(label: string, value: string, when: string const candidates = run(['ps', '--all', '--quiet', '--no-trunc', ...names.flatMap(name => ['--filter', `${kind}=${name}`])]); const holders = kind === 'volume' || !candidates.length ? candidates : run(['container', 'inspect', '--format', - '{{.Id}} {{range $name, $_ := .NetworkSettings.Networks}}{{$name}} {{end}}', ...candidates]) - .map(line => line.split(' ').filter(Boolean)).filter(([, ...on]) => on.some(name => names.includes(name))) - .map(([id]) => id!); + '{{json .Id}} {{json .NetworkSettings.Networks}}', ...candidates]).map(line => { + const at = line.indexOf(' '); + return { id: JSON.parse(line.slice(0, at)) as string, on: Object.keys(JSON.parse(line.slice(at + 1)) ?? {}) }; + }).filter(holder => holder.on.some(name => names.includes(name))).map(holder => holder.id); if (holders.length) { console.warn(`${when} removes the containers that hold them: ${holders.join(' ')}`); run(['rm', '--force', ...holders]); diff --git a/test/runner-workspace.test.ts b/test/runner-workspace.test.ts index 03520a19..d7ad551e 100644 --- a/test/runner-workspace.test.ts +++ b/test/runner-workspace.test.ts @@ -37,7 +37,8 @@ afterAll(() => { } finally { // Then remove the task storage this run still owns, even when a test failed or the cleanup above threw. The path // identity probe carries no owner labels, so it is not found here. It may not run before a hook timeout. - // The hook's budget is a minute over this sweep's worst case (eleven calls of 30 s), for the cleanup before it. + // The hook's 7 minutes leave a minute and a half over this sweep's worst case (eleven calls of 30 s), for the + // cleanup before it. removeRunnerObjects(RUNNER_OWNER); } }, 7 * 60_000); From 7075dd8ae30aad652e35e3fe544f2a4cb6c4c0dd Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 19:28:25 -0700 Subject: [PATCH 11/75] Never throw on output the sweep cannot parse; foreign creates inside try (#189) Review round 10: a line that is not JSON is reported and skipped; the foreign-runner network and storage are made inside their try and removed by allocation; the recovery-handle test removes by full ID through the sweep; doc wording. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 33 +++++++++++++++++---------------- test/fixtures/docker-sweep.ts | 16 ++++++++++------ 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index dbce264f..44b7f5fd 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -888,21 +888,20 @@ describe('real Docker agent isolation', () => { 'result = run(args);', "process.kill(process.pid, 'SIGKILL');", ].join('\n'); - const created = await withDockerShim(['network', 'create'], foreign, () => - createVendorNetwork(invocation(data.clone, 'planning'), imageId, allocationId).then(() => undefined, error => error)); - // Listed in the finally too: the network carries another runner's label, so no sweep of this run finds it (#189). const listed = () => spawnSync('docker', ['network', 'ls', '--quiet', '--no-trunc', '--filter', `label=io.codeboost.allocation=${allocationId}`], { encoding: 'utf8' }).stdout?.trim() ?? ''; let left = ''; try { + const created = await withDockerShim(['network', 'create'], foreign, () => + createVendorNetwork(invocation(data.clone, 'planning'), imageId, allocationId).then(() => undefined, error => error)); left = listed(); expect(created).toBeInstanceOf(VendorNetworkCreationCleanupError); const cleanup = (created as VendorNetworkCreationCleanupError).errors[1] as AggregateError; expect(cleanup.errors.map(error => (error as Error).message)).toEqual(['Refused to remove unowned vendor network.']); expect(left).not.toBe(''); } finally { - const remaining = listed(); - if (remaining) spawnSync('docker', ['network', 'rm', ...remaining.split('\n')], { stdio: 'ignore' }); + // The network carries another runner's label, so no sweep of this run finds it: removed by its allocation (#189). + removeLabelledObjects('io.codeboost.allocation', allocationId, 'test'); } }, 60_000); @@ -2472,11 +2471,8 @@ describe('real Docker agent isolation', () => { for (const [kind, ref] of [['container', left.keeper], ['volume', left.workVolume], ['volume', left.metadataVolume]] as const) expect(spawnSync('docker', [kind, 'inspect', ref], { stdio: 'ignore' }).status).not.toBe(0); } finally { - // Never throws, so a failure above is the one reported. - const labelled = (...args: string[]) => spawnSync('docker', [...args, '--quiet', '--filter', - `label=io.codeboost.allocation=${owner.allocationId}`], { encoding: 'utf8' }).stdout?.split('\n').filter(Boolean) ?? []; - for (const name of labelled('ps', '--all')) spawnSync('docker', ['rm', '--force', name], { stdio: 'ignore' }); - for (const name of labelled('volume', 'ls')) spawnSync('docker', ['volume', 'rm', '--force', name], { stdio: 'ignore' }); + // Never throws, so a failure above is the one reported. By full ID: Docker resolves a name before an ID prefix. + removeLabelledObjects('io.codeboost.allocation', owner.allocationId, 'test'); } }, 180_000); @@ -2499,15 +2495,20 @@ describe('real Docker agent isolation', () => { }, 60_000); it('refuses task storage that belongs to another runner', async () => { - const data = fixture(); - const foreign = prepareTaskFilesystems(data.clone, { workBytes: 16 * 1024 * 1024, workInodes: 512, - metadataBytes: 16 * 1024 * 1024, metadataInodes: 512 }, imageId, - { runnerOwner: 'f'.repeat(32), attemptId: 'other-runner', allocationId: randomUUID() }); - // Removed here: it carries another runner's label, so no sweep of this run finds it (#189). + const data = fixture(), allocationId = randomUUID(); + let foreign: ReturnType | undefined; + // Removed here: it carries another runner's label, so no sweep of this run finds it (#189). Then whatever still + // carries its allocation, should its creation or removal not settle. try { + foreign = prepareTaskFilesystems(data.clone, { workBytes: 16 * 1024 * 1024, workInodes: 512, + metadataBytes: 16 * 1024 * 1024, metadataInodes: 512 }, imageId, + { runnerOwner: 'f'.repeat(32), attemptId: 'other-runner', allocationId }); await expect(createContainerProfile({ ...await governed(invocation(data.clone, 'planning')), filesystems: foreign, inputDirectory: data.input, codexAuthFile: data.fakeAuth, imageId })).rejects.toThrow('another runner'); - } finally { removeTaskFilesystems(foreign); } + } finally { + try { if (foreign) removeTaskFilesystems(foreign); } + finally { removeLabelledObjects('io.codeboost.allocation', allocationId, 'test'); } + } }, 60_000); it('refuses to seed a clone whose staging directory was replaced after creation', async () => { diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index d26835ff..1ee308f4 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -9,8 +9,8 @@ export const TEST_RUN_LABEL = 'io.codeboost.test-run'; /** * Removes the containers, networks and volumes labelled with one runner owner, and every container that holds one of - * those networks or volumes, whatever its labels (see `removeLabelledObjects`). A test file's afterAll calls it in a - * finally block. + * those networks or volumes, whatever its labels (see `removeLabelledObjects`). A test file's afterAll calls it, after + * its other cleanup. */ export function removeRunnerObjects(runnerOwner: string): void { removeLabelledObjects(RUNNER_LABEL, runnerOwner, 'afterAll'); @@ -33,6 +33,10 @@ export function removeLabelledObjects(label: string, value: string, when: string // that exited by itself is read whatever its status. A killed call's output may be cut short, so it is not. return result.error || result.status === null ? [] : result.stdout.split('\n').filter(Boolean); }; + // A line Docker printed that is not JSON is reported and skipped, so the sweep never throws. + const parsed = (text: string): T[] => { + try { return [JSON.parse(text) as T]; } catch { console.warn(`${when}: docker printed a line that is not JSON:`, text); return []; } + }; for (const kind of ['container', 'network', 'volume'] as const) { // Networks with their names: `ps --filter network=` finds a created, never started container by the name only // (Docker 27.4). Each is removed by its full ID, which no other network's name can be mistaken for. @@ -40,7 +44,7 @@ export function removeLabelledObjects(label: string, value: string, when: string const listed = run([kind, 'ls', ...(kind === 'container' ? ['--all'] : []), ...(kind === 'network' ? ['--no-trunc', '--format', '{{json .}}'] : kind === 'container' ? ['--quiet', '--no-trunc'] : ['--quiet']), '--filter', `label=${label}=${value}`]) - .map(line => kind === 'network' ? JSON.parse(line) as { ID: string; Name: string } : { ID: line, Name: line }); + .flatMap(line => kind === 'network' ? parsed<{ ID: string; Name: string }>(line) : [{ ID: line, Name: line }]); const ids = listed.map(object => object.ID), names = listed.map(object => object.Name); if (!ids.length) continue; console.warn(`${when} removes leftover ${kind}s labelled ${label}=${value}: ${names.join(' ')}`); @@ -51,9 +55,9 @@ export function removeLabelledObjects(label: string, value: string, when: string const candidates = run(['ps', '--all', '--quiet', '--no-trunc', ...names.flatMap(name => ['--filter', `${kind}=${name}`])]); const holders = kind === 'volume' || !candidates.length ? candidates : run(['container', 'inspect', '--format', - '{{json .Id}} {{json .NetworkSettings.Networks}}', ...candidates]).map(line => { - const at = line.indexOf(' '); - return { id: JSON.parse(line.slice(0, at)) as string, on: Object.keys(JSON.parse(line.slice(at + 1)) ?? {}) }; + '{{json .Id}} {{json .NetworkSettings.Networks}}', ...candidates]).flatMap(line => { + const at = line.indexOf(' '), id = parsed(line.slice(0, at)), on = parsed>(line.slice(at + 1)); + return id.length && on.length ? [{ id: id[0]!, on: Object.keys(on[0] ?? {}) }] : []; }).filter(holder => holder.on.some(name => names.includes(name))).map(holder => holder.id); if (holders.length) { console.warn(`${when} removes the containers that hold them: ${holders.join(' ')}`); From ee972b816190a091b321ca98003bc6cdafa8d065 Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 19:35:01 -0700 Subject: [PATCH 12/75] Sweep the failed-allocation test by allocation; guard unreadable lines (#189) Review round 11: the foreign-runner failed-allocation test removes its objects through the sweep, by full ID and never throwing; network lines that are not a network are skipped, with a unit test; in-test sweeps have budgets for their worst case and their own warning prefix. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 13 ++++++------- test/docker-sweep.test.ts | 29 +++++++++++++++++++++++++++++ test/fixtures/docker-sweep.ts | 10 ++++++---- test/runner-workspace.test.ts | 7 ++++--- 4 files changed, 45 insertions(+), 14 deletions(-) create mode 100644 test/docker-sweep.test.ts diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 44b7f5fd..b0611789 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -796,7 +796,7 @@ describe('real Docker agent isolation', () => { for (const { kind, ref } of made.filter(object => object.kind !== 'container')) spawnSync('docker', [kind, 'rm', ...(kind === 'volume' ? ['--force'] : []), ref], { stdio: 'ignore' }); } - }, 120_000); + }, 8 * 60_000); // #189: these two run in order (run alone, the second fails: it checks what the first left). The first stops after // making a keeper without its runner label, before the removal that "rejects a keeper without its runner label" @@ -2302,12 +2302,11 @@ describe('real Docker agent isolation', () => { owner)).toThrow('cleanup did not settle')); await expect(recoverLeftovers(runnerOwner)).rejects.toThrow('still holds task storage'); } finally { - for (const name of docker('ps', '--all', '--quiet', '--filter', `label=io.codeboost.allocation=${owner.allocationId}`) - .split('\n').filter(Boolean)) spawnSync('docker', ['rm', '--force', name], { stdio: 'ignore' }); - for (const name of docker('volume', 'ls', '--quiet', '--filter', `label=io.codeboost.allocation=${owner.allocationId}`) - .split('\n').filter(Boolean)) spawnSync('docker', ['volume', 'rm', '--force', name], { stdio: 'ignore' }); + // Another runner's objects, so no sweep of this run finds them: removed by allocation, by full ID, never throwing + // (#189). The sweep may take up to 5.5 minutes, which the test's budget allows for. + removeLabelledObjects('io.codeboost.allocation', owner.allocationId, 'test'); } - }, 60_000); + }, 7 * 60_000); it('recovers only its own runner after a restart, keeping its storage for removal by handle', async () => { const data = fixture(), runners = [randomBytes(16).toString('hex'), randomBytes(16).toString('hex')] as const; @@ -2509,7 +2508,7 @@ describe('real Docker agent isolation', () => { try { if (foreign) removeTaskFilesystems(foreign); } finally { removeLabelledObjects('io.codeboost.allocation', allocationId, 'test'); } } - }, 60_000); + }, 7 * 60_000); it('refuses to seed a clone whose staging directory was replaced after creation', async () => { const data = fixture(); diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts new file mode 100644 index 00000000..7375f96e --- /dev/null +++ b/test/docker-sweep.test.ts @@ -0,0 +1,29 @@ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, expect, it, vi } from 'vitest'; +import { removeLabelledObjects } from './fixtures/docker-sweep.ts'; + +// The test sweep's handling of output it cannot read (#189). The `docker` on PATH answers as no real daemon does: for +// `network ls` it prints a line that is not JSON, a JSON null and one network without a name, and it answers every +// other call with nothing and exit 0. What Docker 27.4 really prints is covered against the daemon in agent-container. +const roots: string[] = [], savedPath = process.env.PATH; +afterEach(() => { + process.env.PATH = savedPath; + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); + vi.restoreAllMocks(); +}); + +it('reports and skips output that is not a network, and never throws', () => { + const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); + const calls = join(root, 'calls'); + writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${calls}'`, + `if [ "$1 $2" = "network ls" ]; then printf '%s\\n' 'not json' 'null' '{"ID":"abc"}'; fi`, 'exit 0'].join('\n'), + { mode: 0o755 }); + process.env.PATH = `${root}:${savedPath}`; + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + expect(() => removeLabelledObjects('io.codeboost.test-run', 'x', 'test')).not.toThrow(); + expect(warn.mock.calls.map(call => String(call[0]))).toContain('test: docker printed a line that is not JSON:'); + // Nothing it could not read is removed or looked up as a holder. + expect(readFileSync(calls, 'utf8').split('\n').filter(line => /^(ps|rm|network rm|volume rm) /.test(line))).toEqual([]); +}); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index 1ee308f4..f5bd900b 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -12,8 +12,8 @@ export const TEST_RUN_LABEL = 'io.codeboost.test-run'; * those networks or volumes, whatever its labels (see `removeLabelledObjects`). A test file's afterAll calls it, after * its other cleanup. */ -export function removeRunnerObjects(runnerOwner: string): void { - removeLabelledObjects(RUNNER_LABEL, runnerOwner, 'afterAll'); +export function removeRunnerObjects(runnerOwner: string, when = 'afterAll'): void { + removeLabelledObjects(RUNNER_LABEL, runnerOwner, when); } /** @@ -44,7 +44,9 @@ export function removeLabelledObjects(label: string, value: string, when: string const listed = run([kind, 'ls', ...(kind === 'container' ? ['--all'] : []), ...(kind === 'network' ? ['--no-trunc', '--format', '{{json .}}'] : kind === 'container' ? ['--quiet', '--no-trunc'] : ['--quiet']), '--filter', `label=${label}=${value}`]) - .flatMap(line => kind === 'network' ? parsed<{ ID: string; Name: string }>(line) : [{ ID: line, Name: line }]); + .flatMap(line => kind === 'network' ? parsed<{ ID?: unknown; Name?: unknown } | null>(line) + .filter((object): object is { ID: string; Name: string } => typeof object?.ID === 'string' && typeof object.Name === 'string') + : [{ ID: line, Name: line }]); const ids = listed.map(object => object.ID), names = listed.map(object => object.Name); if (!ids.length) continue; console.warn(`${when} removes leftover ${kind}s labelled ${label}=${value}: ${names.join(' ')}`); @@ -57,7 +59,7 @@ export function removeLabelledObjects(label: string, value: string, when: string const holders = kind === 'volume' || !candidates.length ? candidates : run(['container', 'inspect', '--format', '{{json .Id}} {{json .NetworkSettings.Networks}}', ...candidates]).flatMap(line => { const at = line.indexOf(' '), id = parsed(line.slice(0, at)), on = parsed>(line.slice(at + 1)); - return id.length && on.length ? [{ id: id[0]!, on: Object.keys(on[0] ?? {}) }] : []; + return typeof id[0] === 'string' && on.length ? [{ id: id[0], on: Object.keys(on[0] ?? {}) }] : []; }).filter(holder => holder.on.some(name => names.includes(name))).map(holder => holder.id); if (holders.length) { console.warn(`${when} removes the containers that hold them: ${holders.join(' ')}`); diff --git a/test/runner-workspace.test.ts b/test/runner-workspace.test.ts index 7eb45060..493c1e11 100644 --- a/test/runner-workspace.test.ts +++ b/test/runner-workspace.test.ts @@ -255,10 +255,11 @@ describe('task storage path identity (#162)', () => { for (const ref of [other.name, unmarked]) expect(docker('container', 'inspect', '--format', '{{.State.Status}}', ref)).toBe('created'); expect(docker('volume', 'inspect', '--format', '{{.Name}}', other.volume)).toBe(other.volume); } finally { - removeRunnerObjects(mine); - removeRunnerObjects(theirs); + // Each sweep may take up to 5.5 minutes, which the test's budget allows for. + removeRunnerObjects(mine, 'test'); + removeRunnerObjects(theirs, 'test'); } - }, 120_000); + }, 13 * 60_000); it('runs only the image the trusted builder produced', async () => { await expect(probeTaskStoragePathIdentity(`sha256:${'0'.repeat(64)}`, RUNNER_OWNER)).rejects.toThrow('Agent image was not produced by the trusted validated builder.'); await expect(probeTaskStoragePathIdentity('alpine', RUNNER_OWNER)).rejects.toThrow('requires the immutable built image ID'); From 31e669f29ac9054378fffcfffe91e76ae1216c89 Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 19:41:23 -0700 Subject: [PATCH 13/75] Budget the in-test sweeps; exact wording (#189) Review round 12. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 18 ++++++++++-------- test/docker-sweep.test.ts | 4 ++-- test/runner-workspace.test.ts | 2 +- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index b0611789..e2546c78 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -673,11 +673,11 @@ describe('real Docker agent isolation', () => { }, 60_000); // A docker wrapper that drops our runner label from commands starting with one of `prefixes`, so the object it - // creates is genuinely mislabelled and only the owner-label checks can catch it. Production cleanup refuses such an - // object, and afterAll's runner sweep finds it only while it holds one of this run's volumes or networks, so the - // wrapper labels it with a test-run ID instead (#189). Whatever has that ID is removed when the test finishes, passed, - // failed or timed out, and again in afterAll, for a create that lands after a timed-out test's removal (its body runs - // on). + // creates is genuinely mislabelled and only the owner-label checks can catch it. Production cleanup may refuse such + // an object (task storage has no captured IDs), and afterAll's runner sweep finds it only while it holds one of this + // run's volumes or networks, so the wrapper labels it with a test-run ID instead (#189). Whatever has that ID is + // removed when the test finishes, passed, failed or timed out, and again in afterAll, for a create that lands after a + // timed-out test's removal (its body runs on). const withoutRunnerLabel = async (prefixes: string[][], run: () => Promise | T): Promise => { if (!prefixes.every(prefix => prefix.includes('create'))) throw new Error('Only create commands can be unlabelled.'); const testRun = randomUUID(); @@ -901,9 +901,10 @@ describe('real Docker agent isolation', () => { expect(left).not.toBe(''); } finally { // The network carries another runner's label, so no sweep of this run finds it: removed by its allocation (#189). + // The sweep may take up to 5.5 minutes, which the test's budget allows for. removeLabelledObjects('io.codeboost.allocation', allocationId, 'test'); } - }, 60_000); + }, 7 * 60_000); it('never removes an agent container found by name after a killed create when a label is missing', async () => { const data = fixture(), live = await profile(data, 'planning', 'must-not-run'); @@ -2470,10 +2471,11 @@ describe('real Docker agent isolation', () => { for (const [kind, ref] of [['container', left.keeper], ['volume', left.workVolume], ['volume', left.metadataVolume]] as const) expect(spawnSync('docker', [kind, 'inspect', ref], { stdio: 'ignore' }).status).not.toBe(0); } finally { - // Never throws, so a failure above is the one reported. By full ID: Docker resolves a name before an ID prefix. + // Never throws, so a failure above is the one reported. By full ID: Docker resolves a name before an ID prefix. The + // sweep may take up to 5.5 minutes, which the test's budget allows for. removeLabelledObjects('io.codeboost.allocation', owner.allocationId, 'test'); } - }, 180_000); + }, 9 * 60_000); it('never removes a task keeper replaced by another runner, but still removes the owned volumes', () => { const data = fixture(), filesystems = data.filesystems, owner = taskFilesystemOwner(filesystems); diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index 7375f96e..b7f82bec 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -14,7 +14,7 @@ afterEach(() => { vi.restoreAllMocks(); }); -it('reports and skips output that is not a network, and never throws', () => { +it('skips output that is not a network, reports what is not JSON, and never throws', () => { const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); const calls = join(root, 'calls'); writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${calls}'`, @@ -25,5 +25,5 @@ it('reports and skips output that is not a network, and never throws', () => { expect(() => removeLabelledObjects('io.codeboost.test-run', 'x', 'test')).not.toThrow(); expect(warn.mock.calls.map(call => String(call[0]))).toContain('test: docker printed a line that is not JSON:'); // Nothing it could not read is removed or looked up as a holder. - expect(readFileSync(calls, 'utf8').split('\n').filter(line => /^(ps|rm|network rm|volume rm) /.test(line))).toEqual([]); + expect(readFileSync(calls, 'utf8').split('\n').filter(line => /^(ps|rm|network rm|volume rm|container rm|container inspect) /.test(line))).toEqual([]); }); diff --git a/test/runner-workspace.test.ts b/test/runner-workspace.test.ts index 493c1e11..7c6bbcd2 100644 --- a/test/runner-workspace.test.ts +++ b/test/runner-workspace.test.ts @@ -259,7 +259,7 @@ describe('task storage path identity (#162)', () => { removeRunnerObjects(mine, 'test'); removeRunnerObjects(theirs, 'test'); } - }, 13 * 60_000); + }, 14 * 60_000); it('runs only the image the trusted builder produced', async () => { await expect(probeTaskStoragePathIdentity(`sha256:${'0'.repeat(64)}`, RUNNER_OWNER)).rejects.toThrow('Agent image was not produced by the trusted validated builder.'); await expect(probeTaskStoragePathIdentity('alpine', RUNNER_OWNER)).rejects.toThrow('requires the immutable built image ID'); From a9162f78ac6e02a046ee001dab9cb1465a78646a Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 19:47:47 -0700 Subject: [PATCH 14/75] Sweep both runners of the recovery test; keep the first failure (#189) Review round 13: the recovery test sweeps its two random runner owners after its undo steps, so what a step could not remove or never recorded goes too; the foreign-storage test's cleanup no longer replaces the body's failure; doc wording. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 10 +++++++--- test/fixtures/docker-sweep.ts | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index e2546c78..2e8240c3 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -2418,9 +2418,12 @@ describe('real Docker agent isolation', () => { for (const step of undo.reverse()) { try { await step(); } catch (error) { failures.push(error); } } + // Both runners are this test's own random owners, so no sweep of this run finds what an undo step could not + // remove or never recorded (#189). Each sweep may take up to 5.5 minutes, which the test's budget allows for. + for (const runner of runners) removeRunnerObjects(runner, 'test'); if (failures.length) throw new AggregateError(failures, 'Recovery test cleanup failed.'); } - }, 180_000); + }, 14 * 60_000); it('exports through a recovery handle after a restart, then removes the storage by that handle', async () => { const data = fixture({ hostile: agentBase }); @@ -2507,8 +2510,9 @@ describe('real Docker agent isolation', () => { await expect(createContainerProfile({ ...await governed(invocation(data.clone, 'planning')), filesystems: foreign, inputDirectory: data.input, codexAuthFile: data.fakeAuth, imageId })).rejects.toThrow('another runner'); } finally { - try { if (foreign) removeTaskFilesystems(foreign); } - finally { removeLabelledObjects('io.codeboost.allocation', allocationId, 'test'); } + // Never throws, so a failure above is the one reported; the sweep removes whatever the removal left. + try { if (foreign) removeTaskFilesystems(foreign); } catch { /* removed by allocation below */ } + removeLabelledObjects('io.codeboost.allocation', allocationId, 'test'); } }, 7 * 60_000); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index f5bd900b..13eec27c 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -9,8 +9,8 @@ export const TEST_RUN_LABEL = 'io.codeboost.test-run'; /** * Removes the containers, networks and volumes labelled with one runner owner, and every container that holds one of - * those networks or volumes, whatever its labels (see `removeLabelledObjects`). A test file's afterAll calls it, after - * its other cleanup. + * those networks or volumes, whatever its labels (see `removeLabelledObjects`). A test file's afterAll calls it after + * its other cleanup, and a test that makes objects under its own runner owner calls it in its finally. */ export function removeRunnerObjects(runnerOwner: string, when = 'afterAll'): void { removeLabelledObjects(RUNNER_LABEL, runnerOwner, when); From 3818b53cae95e02667c5cb93fa6d77c6feb69d3a Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 19:55:37 -0700 Subject: [PATCH 15/75] Check the traps are live in the holder test; report every cleanup failure (#189) Review round 14: the holder test first asserts Docker's prefix match and its name-before-prefix resolution, so its bystander checks test something; afterAll collects every cleanup step's failure; comment states which ending the it.fails pair tests. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 2e8240c3..cf03bb27 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -137,18 +137,18 @@ afterEach(async () => { afterAll(async () => { try { for (const container of containers) spawnSync('docker', ['rm', '--force', container], { stdio: 'ignore' }); - // Each one, even when an earlier one fails: storage left here is found by the sweep below only by its label. + // Each step, even when an earlier one fails: storage left here is found by the sweep below only by its label, and + // every failure is reported. const failures: unknown[] = []; - for (const filesystems of taskFilesystems.reverse()) { - try { removeTaskFilesystems(filesystems); } catch (error) { failures.push(error); } - } - for (const profile of profiles) await disposeContainerProfile(profile); - for (const network of vendorNetworks.splice(0).reverse()) await removeVendorNetwork(network); - for (const root of roots.reverse()) { + const attempt = async (step: () => unknown) => { try { await step(); } catch (error) { failures.push(error); } }; + for (const filesystems of taskFilesystems.reverse()) await attempt(() => removeTaskFilesystems(filesystems)); + for (const profile of profiles) await attempt(() => disposeContainerProfile(profile)); + for (const network of vendorNetworks.splice(0).reverse()) await attempt(() => removeVendorNetwork(network)); + for (const root of roots.reverse()) await attempt(() => { chmodSync(join(root, 'input'), 0o700); rmSync(root, { recursive: true, force: true, maxRetries: 3, retryDelay: 50 }); - } - if (failures.length) throw new AggregateError(failures, 'Task storage cleanup did not settle.'); + }); + if (failures.length) throw new AggregateError(failures, 'Test cleanup failed.'); } finally { // The runner owner is this run's own, so nothing else would find what this run leaves behind. Objects a test // makes under another owner or without the runner label are not found here, unless they hold one of this run's @@ -676,8 +676,9 @@ describe('real Docker agent isolation', () => { // creates is genuinely mislabelled and only the owner-label checks can catch it. Production cleanup may refuse such // an object (task storage has no captured IDs), and afterAll's runner sweep finds it only while it holds one of this // run's volumes or networks, so the wrapper labels it with a test-run ID instead (#189). Whatever has that ID is - // removed when the test finishes, passed, failed or timed out, and again in afterAll, for a create that lands after a - // timed-out test's removal (its body runs on). + // removed when the test finishes (Vitest runs `onTestFinished` after a pass, a failure or a timeout; the `it.fails` + // pair below tests the failure), and again in afterAll, for a create that lands after a timed-out test's removal (its + // body runs on). const withoutRunnerLabel = async (prefixes: string[][], run: () => Promise | T): Promise => { if (!prefixes.every(prefix => prefix.includes('create'))) throw new Error('Only create commands can be unlabelled.'); const testRun = randomUUID(); @@ -772,6 +773,12 @@ describe('real Docker agent isolation', () => { const holderLookalike = make('container', holders[0]!.slice(0, 12), 'create', '--entrypoint', 'true', imageId); const labelled = make('container', named('swept'), 'create', '--label', label, '--entrypoint', 'true', imageId); const labelledLookalike = make('container', labelled.slice(0, 12), 'create', '--entrypoint', 'true', imageId); + // The traps are live on this daemon, so the checks below test something: the network filter matches the start of + // a network's ID, and a name is resolved before an ID prefix, for networks and containers alike. + expect(docker('ps', '--all', '--quiet', '--no-trunc', '--filter', `network=${bystanderNetwork.slice(0, 12)}`).split('\n')) + .toContain(bystanders[0]); + expect(docker('container', 'inspect', '--format', '{{.Id}}', holders[0]!.slice(0, 12))).toBe(holderLookalike); + expect(docker('network', 'inspect', '--format', '{{.Id}}', network.slice(0, 12))).toBe(lookalike); // The holders' inspect also names a container that does not exist, as one gone since the listing would be: Docker // 27.4 prints the others and exits 1, and the sweep must still remove them. const shim = mkdtempSync(join(tmpdir(), 'codeboost-sweep-docker-')); roots.push(shim); From 9019dce6004e47cf610ab9e28c4b6ec0e447c0f4 Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 20:04:18 -0700 Subject: [PATCH 16/75] Keep the first failure in the replaced-keeper test; exact sweep wording (#189) Review round 15 nits. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index cf03bb27..a965bd6c 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -674,8 +674,9 @@ describe('real Docker agent isolation', () => { // A docker wrapper that drops our runner label from commands starting with one of `prefixes`, so the object it // creates is genuinely mislabelled and only the owner-label checks can catch it. Production cleanup may refuse such - // an object (task storage has no captured IDs), and afterAll's runner sweep finds it only while it holds one of this - // run's volumes or networks, so the wrapper labels it with a test-run ID instead (#189). Whatever has that ID is + // an object (task storage has no captured IDs), and afterAll's runner sweep finds such a container only while it + // holds one of this run's volumes or networks, and such a volume or network never, so the wrapper labels it with a + // test-run ID instead (#189). Whatever has that ID is // removed when the test finishes (Vitest runs `onTestFinished` after a pass, a failure or a timeout; the `it.fails` // pair below tests the failure), and again in afterAll, for a create that lands after a timed-out test's removal (its // body runs on). @@ -2501,7 +2502,9 @@ describe('real Docker agent isolation', () => { expect(spawnSync('docker', ['volume', 'inspect', filesystems.workVolume]).status).not.toBe(0); } finally { spawnSync('docker', ['rm', '--force', filesystems.keeper], { stdio: 'ignore' }); - removeTaskFilesystems(filesystems); + // Never throws, so a failure above is the one reported: the volumes carry the runner label, so afterAll's sweep + // removes what this leaves. + try { removeTaskFilesystems(filesystems); } catch { /* removed by afterAll's runner sweep */ } } }, 60_000); From 699ea70e3d5720fd6e594853367f455544ee7361 Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 20:12:54 -0700 Subject: [PATCH 17/75] Check the partial-inspect answer; sweep the replaced keeper; budget margin (#189) Review round 16. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index a965bd6c..300f6186 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -780,6 +780,11 @@ describe('real Docker agent isolation', () => { .toContain(bystanders[0]); expect(docker('container', 'inspect', '--format', '{{.Id}}', holders[0]!.slice(0, 12))).toBe(holderLookalike); expect(docker('network', 'inspect', '--format', '{{.Id}}', network.slice(0, 12))).toBe(lookalike); + // And an inspect naming a missing container exits non-zero yet prints the others, the answer the shim below gives. + const partial = spawnSync('docker', ['container', 'inspect', '--format', '{{.Id}}', holders[0]!, `codeboost-gone-${randomUUID()}`], + { encoding: 'utf8' }); + expect(partial.status).not.toBe(0); + expect(partial.stdout).toContain(holders[0]); // The holders' inspect also names a container that does not exist, as one gone since the listing would be: Docker // 27.4 prints the others and exits 1, and the sweep must still remove them. const shim = mkdtempSync(join(tmpdir(), 'codeboost-sweep-docker-')); roots.push(shim); @@ -2431,7 +2436,7 @@ describe('real Docker agent isolation', () => { for (const runner of runners) removeRunnerObjects(runner, 'test'); if (failures.length) throw new AggregateError(failures, 'Recovery test cleanup failed.'); } - }, 14 * 60_000); + }, 15 * 60_000); it('exports through a recovery handle after a restart, then removes the storage by that handle', async () => { const data = fixture({ hostile: agentBase }); @@ -2503,8 +2508,9 @@ describe('real Docker agent isolation', () => { } finally { spawnSync('docker', ['rm', '--force', filesystems.keeper], { stdio: 'ignore' }); // Never throws, so a failure above is the one reported: the volumes carry the runner label, so afterAll's sweep - // removes what this leaves. + // removes what this leaves, and the keeper with another runner's label goes by its allocation (#189). try { removeTaskFilesystems(filesystems); } catch { /* removed by afterAll's runner sweep */ } + removeLabelledObjects('io.codeboost.allocation', owner.allocationId, 'test'); } }, 60_000); From 31a5614656f2fdfebdc74908cc409f038f6802eb Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 20:20:40 -0700 Subject: [PATCH 18/75] Budget the replaced-keeper sweep; say what the holder test's budget covers (#189) Review round 17. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 6 ++++-- test/docker-sweep.test.ts | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 300f6186..9f337e9b 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -809,6 +809,7 @@ describe('real Docker agent isolation', () => { for (const { kind, ref } of made.filter(object => object.kind !== 'container')) spawnSync('docker', [kind, 'rm', ...(kind === 'volume' ? ['--force'] : []), ref], { stdio: 'ignore' }); } + // The budget covers the sweep's worst case and ordinary Docker calls, not every call at its own 60 s limit. }, 8 * 60_000); // #189: these two run in order (run alone, the second fails: it checks what the first left). The first stops after @@ -2508,11 +2509,12 @@ describe('real Docker agent isolation', () => { } finally { spawnSync('docker', ['rm', '--force', filesystems.keeper], { stdio: 'ignore' }); // Never throws, so a failure above is the one reported: the volumes carry the runner label, so afterAll's sweep - // removes what this leaves, and the keeper with another runner's label goes by its allocation (#189). + // removes what this leaves, and the keeper with another runner's label goes by its allocation (#189). The sweep may + // take up to 5.5 minutes, which the test's budget allows for. try { removeTaskFilesystems(filesystems); } catch { /* removed by afterAll's runner sweep */ } removeLabelledObjects('io.codeboost.allocation', owner.allocationId, 'test'); } - }, 60_000); + }, 7 * 60_000); it('refuses task storage that belongs to another runner', async () => { const data = fixture(), allocationId = randomUUID(); diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index b7f82bec..15cdb048 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -2,7 +2,7 @@ import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, expect, it, vi } from 'vitest'; -import { removeLabelledObjects } from './fixtures/docker-sweep.ts'; +import { removeLabelledObjects, TEST_RUN_LABEL } from './fixtures/docker-sweep.ts'; // The test sweep's handling of output it cannot read (#189). The `docker` on PATH answers as no real daemon does: for // `network ls` it prints a line that is not JSON, a JSON null and one network without a name, and it answers every @@ -22,7 +22,7 @@ it('skips output that is not a network, reports what is not JSON, and never thro { mode: 0o755 }); process.env.PATH = `${root}:${savedPath}`; const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); - expect(() => removeLabelledObjects('io.codeboost.test-run', 'x', 'test')).not.toThrow(); + expect(() => removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test')).not.toThrow(); expect(warn.mock.calls.map(call => String(call[0]))).toContain('test: docker printed a line that is not JSON:'); // Nothing it could not read is removed or looked up as a holder. expect(readFileSync(calls, 'utf8').split('\n').filter(line => /^(ps|rm|network rm|volume rm|container rm|container inspect) /.test(line))).toEqual([]); From d74abcaf03c7f8451d4ae10c6001dead0db0441f Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 20:31:24 -0700 Subject: [PATCH 19/75] One per-file sweep in afterAll; real docker for the sweep; timed-out leftovers (#189) Review round 18: objects made without the runner label also carry this file's run ID, so afterAll removes them in one sweep within its budget; the sweep runs the real docker found at load, never a shim left on PATH; agent-network's label-less objects and the rogue volume are also removed in afterAll should their test time out; the volume test checks the wrapper's labels; cleanup warnings are logged; the fake's note names its answers. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 65 +++++++++++++++++++---------------- test/agent-network.test.ts | 13 +++++++ test/docker-sweep.test.ts | 13 ++++--- test/fixtures/docker-sweep.ts | 16 +++++++-- test/runner-workspace.test.ts | 7 ++-- 5 files changed, 71 insertions(+), 43 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 9f337e9b..65707387 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -1,5 +1,5 @@ import { execFileSync, spawnSync } from 'node:child_process'; -import { removeLabelledObjects, removeRunnerObjects, TEST_RUN_LABEL } from './fixtures/docker-sweep.ts'; +import { removeLabelledObjects, removeRunnerObjects, TEST_FILE_LABEL, TEST_RUN_LABEL } from './fixtures/docker-sweep.ts'; import { fixtureGit } from './fixtures/git.ts'; import { createHash, randomBytes, randomUUID } from 'node:crypto'; import { chmodSync, existsSync, mkdtempSync, mkdirSync, readdirSync, readFileSync, realpathSync, renameSync, rmSync, statSync, @@ -47,8 +47,10 @@ const testOwner = (attemptId = 'fixture') => ({ runnerOwner: TEST_RUNNER_OWNER, const roots: string[] = []; const taskFilesystems: ReturnType[] = []; const containers = new Set(); -/** The test-run IDs `withoutRunnerLabel` put on objects it made without the runner label (#189). */ -const testRuns: string[] = []; +/** Put, beside its test-run ID, on every object this file makes without the runner label (#189): afterAll sweeps it. */ +const TEST_FILE_RUN = randomUUID(); +/** Volumes a test makes without labels, removed in afterAll if the test did not remove them (it may have timed out). */ +const volumes = new Set(); const profiles: Awaited>[] = []; let imageId = ''; const vendorNetworks: VendorNetwork[] = []; @@ -137,6 +139,7 @@ afterEach(async () => { afterAll(async () => { try { for (const container of containers) spawnSync('docker', ['rm', '--force', container], { stdio: 'ignore' }); + for (const volume of volumes) spawnSync('docker', ['volume', 'rm', '--force', volume], { stdio: 'ignore' }); // Each step, even when an earlier one fails: storage left here is found by the sweep below only by its label, and // every failure is reported. const failures: unknown[] = []; @@ -152,13 +155,13 @@ afterAll(async () => { } finally { // The runner owner is this run's own, so nothing else would find what this run leaves behind. Objects a test // makes under another owner or without the runner label are not found here, unless they hold one of this run's - // volumes or networks: those made through `withoutRunnerLabel` are found by their test-run IDs, and the other tests - // remove their own. It may not run before a hook timeout, and its budget does not cover every Docker call taking - // its full 30 s (eleven per sweep). - for (const testRun of testRuns) removeLabelledObjects(TEST_RUN_LABEL, testRun, 'afterAll'); + // volumes or networks: those this file labels with its own run ID are found by it, and the other tests remove their + // own. It may not run before a hook timeout. Its budget covers two sweeps of up to 5.5 minutes each and the cleanup + // before them. + removeLabelledObjects(TEST_FILE_LABEL, TEST_FILE_RUN, 'afterAll'); removeRunnerObjects(TEST_RUNNER_OWNER); } -}, 10 * 60_000); +}, 13 * 60_000); describe('real Docker agent isolation', () => { it.each([ @@ -454,7 +457,9 @@ describe('real Docker agent isolation', () => { const rogue = `codeboost-work-${randomUUID()}`; try { - // Inside the try: the volume has no labels, so only the finally below removes it (#189). + // Inside the try, and tracked for afterAll should the test time out: the volume has no labels, so nothing else + // finds it (#189). + volumes.add(rogue); docker('volume', 'create', rogue); const rogueArgs = valid.args.map(value => value.replace(data.filesystems.workVolume, rogue)); docker(...rogueArgs); containers.add(valid.name); @@ -676,14 +681,12 @@ describe('real Docker agent isolation', () => { // creates is genuinely mislabelled and only the owner-label checks can catch it. Production cleanup may refuse such // an object (task storage has no captured IDs), and afterAll's runner sweep finds such a container only while it // holds one of this run's volumes or networks, and such a volume or network never, so the wrapper labels it with a - // test-run ID instead (#189). Whatever has that ID is - // removed when the test finishes (Vitest runs `onTestFinished` after a pass, a failure or a timeout; the `it.fails` - // pair below tests the failure), and again in afterAll, for a create that lands after a timed-out test's removal (its - // body runs on). + // test-run ID and this file's run ID instead (#189). Whatever has that ID is removed when the test finishes (Vitest + // runs `onTestFinished` after a pass, a failure or a timeout; the `it.fails` pair below tests the failure), and again + // in afterAll, for a create that lands after a timed-out test's removal (its body runs on). const withoutRunnerLabel = async (prefixes: string[][], run: () => Promise | T): Promise => { if (!prefixes.every(prefix => prefix.includes('create'))) throw new Error('Only create commands can be unlabelled.'); const testRun = randomUUID(); - testRuns.push(testRun); // Up to eleven synchronous Docker calls of 30 s each, which a timer cannot cut short: the budget covers them, so a // slow removal is not reported as a hook timeout (10 s by default). onTestFinished(() => removeLabelledObjects(TEST_RUN_LABEL, testRun, 'onTestFinished'), 6 * 60_000); @@ -696,7 +699,7 @@ describe('real Docker agent isolation', () => { `const strip = ${JSON.stringify(prefixes)}.some(prefix => prefix.every((word, i) => args[i] === word));`, 'const out = [];', "for (let i = 0; i < args.length; i++) { if (strip && args[i] === '--label' && String(args[i + 1]).startsWith('io.codeboost.runner=')) { i++; continue; } out.push(args[i]); }", - `if (strip) out.splice(out.indexOf('create') + 1, 0, '--label', ${JSON.stringify(`${TEST_RUN_LABEL}=${testRun}`)});`, + `if (strip) out.splice(out.indexOf('create') + 1, 0, '--label', ${JSON.stringify(`${TEST_RUN_LABEL}=${testRun}`)}, '--label', ${JSON.stringify(`${TEST_FILE_LABEL}=${TEST_FILE_RUN}`)});`, `const result = spawnSync(${JSON.stringify(realDocker)}, out, { stdio: 'inherit' });`, 'process.exit(result.status ?? 1);'].join('\n'), { mode: 0o755 }); const path = process.env.PATH; @@ -724,6 +727,11 @@ describe('real Docker agent isolation', () => { it('rejects task volumes without their runner label before the agent starts', async () => { const data = await withoutRunnerLabel([['volume', 'create']], () => fixture()); + // The wrapper labelled the volume for its removal in place of the runner label it dropped. + const labels = JSON.parse(docker('volume', 'inspect', '--format', '{{json .Labels}}', data.filesystems.workVolume)) as Record; + expect(labels).not.toHaveProperty(['io.codeboost.runner']); + expect(labels).toHaveProperty([TEST_RUN_LABEL]); + expect(labels).toHaveProperty([TEST_FILE_LABEL], TEST_FILE_RUN); const live = await profile(data, 'planning', 'must-not-run'); await expect(createValidatedContainer(live)).rejects.toThrow('bounded tmpfs allocation'); refuseThenRemove(data.filesystems, [['volume', 'rm', '--force', data.filesystems.workVolume, @@ -747,10 +755,9 @@ describe('real Docker agent isolation', () => { // #189: the sweep removes what holds a swept network or volume, even a container created and never started, and // nothing on a network or volume it does not sweep, even one whose ID starts with a swept network's name. it('removes the containers that hold a swept network or volume, and nothing else', () => { - // The test-run ID goes to afterAll too, and each object is recorded by name before it is made, so a failure here - // leaves nothing behind (#189). - const run = randomUUID(), label = `${TEST_RUN_LABEL}=${run}`, made: { kind: 'container' | 'network' | 'volume'; ref: string }[] = []; - testRuns.push(run); + // The swept objects carry this file's run ID too, for afterAll, and each object is recorded by name before it is + // made, so a failure here leaves nothing behind (#189). + const run = randomUUID(), label = `${TEST_RUN_LABEL}=${run}`, fileLabel = `${TEST_FILE_LABEL}=${TEST_FILE_RUN}`, made: { kind: 'container' | 'network' | 'volume'; ref: string }[] = []; const make = (kind: 'container' | 'network' | 'volume', name: string, ...args: string[]) => { made.push({ kind, ref: name }); return docker(...(kind === 'container' ? [args[0]!, '--name', name, ...args.slice(1)] : [kind, args[0]!, ...args.slice(1), name])); @@ -761,10 +768,10 @@ describe('real Docker agent isolation', () => { const bystanderVolume = make('volume', named('bystander'), 'create'); const bystanders = [make('container', named('bystander'), 'run', '--detach', '--network', bystanderNetwork, '--entrypoint', 'sleep', imageId, '300'), make('container', named('bystander'), 'create', '--mount', `type=volume,source=${bystanderVolume},target=/x`, '--entrypoint', 'true', imageId)]; - const network = make('network', named('swept'), 'create', '--label', label); + const network = make('network', named('swept'), 'create', '--label', label, '--label', fileLabel); // Named like the start of the bystander network's ID: Docker's network filter also matches that prefix. - make('network', bystanderNetwork.slice(0, 12), 'create', '--label', label); - const volume = make('volume', named('swept'), 'create', '--label', label); + make('network', bystanderNetwork.slice(0, 12), 'create', '--label', label, '--label', fileLabel); + const volume = make('volume', named('swept'), 'create', '--label', label, '--label', fileLabel); // And a bystander named like the start of the swept network's ID: removing by a short ID would remove it. const lookalike = make('network', network.slice(0, 12), 'create'); const holders = [make('container', named('holder'), 'create', '--network', network, '--entrypoint', 'true', imageId), @@ -772,7 +779,7 @@ describe('real Docker agent isolation', () => { make('container', named('holder'), 'create', '--mount', `type=volume,source=${volume},target=/x`, '--entrypoint', 'true', imageId)]; // The same for containers: Docker resolves a name before an ID prefix. A holder and a swept container each have one. const holderLookalike = make('container', holders[0]!.slice(0, 12), 'create', '--entrypoint', 'true', imageId); - const labelled = make('container', named('swept'), 'create', '--label', label, '--entrypoint', 'true', imageId); + const labelled = make('container', named('swept'), 'create', '--label', label, '--label', fileLabel, '--entrypoint', 'true', imageId); const labelledLookalike = make('container', labelled.slice(0, 12), 'create', '--entrypoint', 'true', imageId); // The traps are live on this daemon, so the checks below test something: the network filter matches the start of // a network's ID, and a name is resolved before an ID prefix, for networks and containers alike. @@ -793,9 +800,7 @@ describe('real Docker agent isolation', () => { writeFileSync(join(shim, 'docker'), ['#!/bin/sh', `if [ "$1 $2" = "container inspect" ]; then exec '${realDocker}' "$@" codeboost-gone-${randomUUID()}; fi`, `exec '${realDocker}' "$@"`].join('\n'), { mode: 0o755 }); - const path = process.env.PATH; - process.env.PATH = `${shim}:${path}`; - try { removeLabelledObjects(TEST_RUN_LABEL, run, 'test'); } finally { process.env.PATH = path; } + removeLabelledObjects(TEST_RUN_LABEL, run, 'test', join(shim, 'docker')); const exists = (kind: string, ref: string) => spawnSync('docker', [kind, 'inspect', ref], { stdio: 'ignore' }).status === 0; for (const holder of holders) expect(exists('container', holder)).toBe(false); expect([exists('network', network), exists('volume', volume)]).toEqual([false, false]); @@ -2508,10 +2513,10 @@ describe('real Docker agent isolation', () => { expect(spawnSync('docker', ['volume', 'inspect', filesystems.workVolume]).status).not.toBe(0); } finally { spawnSync('docker', ['rm', '--force', filesystems.keeper], { stdio: 'ignore' }); - // Never throws, so a failure above is the one reported: the volumes carry the runner label, so afterAll's sweep - // removes what this leaves, and the keeper with another runner's label goes by its allocation (#189). The sweep may + // Never throws, so a failure above is the one reported: what the removal leaves, the allocation sweep below removes, + // the keeper with another runner's label included (#189). The sweep may // take up to 5.5 minutes, which the test's budget allows for. - try { removeTaskFilesystems(filesystems); } catch { /* removed by afterAll's runner sweep */ } + try { removeTaskFilesystems(filesystems); } catch (error) { console.warn('cleanup: removing the storage failed:', error); } removeLabelledObjects('io.codeboost.allocation', owner.allocationId, 'test'); } }, 7 * 60_000); @@ -2529,7 +2534,7 @@ describe('real Docker agent isolation', () => { inputDirectory: data.input, codexAuthFile: data.fakeAuth, imageId })).rejects.toThrow('another runner'); } finally { // Never throws, so a failure above is the one reported; the sweep removes whatever the removal left. - try { if (foreign) removeTaskFilesystems(foreign); } catch { /* removed by allocation below */ } + try { if (foreign) removeTaskFilesystems(foreign); } catch (error) { console.warn('cleanup: removing the storage failed:', error); } removeLabelledObjects('io.codeboost.allocation', allocationId, 'test'); } }, 7 * 60_000); diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index a686d5f2..33e0a43f 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -33,8 +33,17 @@ beforeAll(async () => { }); network = await createVendorNetwork(invocation, imageId, randomUUID()); }, 10 * 60_000); +/** + * Objects a test makes without labels, by name or by the file its shim records the name in, so afterAll removes them + * should the test time out before its own removal runs (#189). + */ +const unlabelled: { kind: 'container' | 'network'; name?: string; file?: string }[] = []; // If setup failed there is no network, and a teardown error would hide the setup failure. afterAll(async () => { + for (const { kind, name, file } of unlabelled) { + const ref = name ?? (file && existsSync(file) ? readFileSync(file, 'utf8') : ''); + if (ref) spawnSync('docker', kind === 'container' ? ['rm', '--force', ref] : ['network', 'rm', ref], { stdio: 'ignore' }); + } try { if (network) await removeVendorNetwork(network); } finally { // Then remove whatever this run still owns, even when a test failed or the removal above threw. Objects a test @@ -87,6 +96,7 @@ describe('vendor-only egress', () => { it('leaves a same-named network alone when the daemon refuses the create', async () => { const shim = mkdtempSync(join(tmpdir(), 'docker-shim-')), recorded = join(shim, 'foreign'); + unlabelled.push({ kind: 'network', file: recorded }); const realDocker = execFileSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).trim(); // The name is taken by someone else's network just before our create, so the daemon refuses it. writeFileSync(join(shim, 'docker'), ['#!/bin/sh', @@ -143,6 +153,7 @@ describe('vendor-only egress', () => { it('neither reports nor removes a same-named proxy when the daemon refuses the create', async () => { const shim = mkdtempSync(join(tmpdir(), 'docker-shim-')), recorded = join(shim, 'foreign'); + unlabelled.push({ kind: 'container', file: recorded }); const realDocker = execFileSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).trim(); // Someone else's container takes the proxy name just before our create, so the daemon refuses it. writeFileSync(join(shim, 'docker'), ['#!/bin/sh', @@ -192,6 +203,7 @@ describe('vendor-only egress', () => { it('does not delete a same-named stand-in when setup fails after the proxy exists', async () => { const shim = mkdtempSync(join(tmpdir(), 'docker-shim-')), recorded = join(shim, 'impostor'); + unlabelled.push({ kind: 'container', file: recorded }); const realDocker = execFileSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).trim(); // The readiness exec swaps the proxy for a same-named, same-labelled stand-in, then fails setup. writeFileSync(join(shim, 'docker'), ['#!/bin/sh', 'if [ "$1" = exec ]; then', @@ -227,6 +239,7 @@ describe('vendor-only egress', () => { const partial = captureInvocation({ ...invocation, attemptId: `partial-removal-${randomUUID()}`, deadline: Date.now() + 60_000 }); const created = await createVendorNetwork(partial, imageId, randomUUID()), blocker = `codeboost-blocker-${randomUUID()}`; + unlabelled.push({ kind: 'container', name: blocker }); const [proxy, net] = vendorNetworkResources(created); expect(proxy).toMatchObject({ kind: 'container', name: created.proxyContainer }); expect(net).toMatchObject({ kind: 'network', name: created.name }); diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index 15cdb048..0d75f89e 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -4,12 +4,12 @@ import { join } from 'node:path'; import { afterEach, expect, it, vi } from 'vitest'; import { removeLabelledObjects, TEST_RUN_LABEL } from './fixtures/docker-sweep.ts'; -// The test sweep's handling of output it cannot read (#189). The `docker` on PATH answers as no real daemon does: for -// `network ls` it prints a line that is not JSON, a JSON null and one network without a name, and it answers every -// other call with nothing and exit 0. What Docker 27.4 really prints is covered against the daemon in agent-container. -const roots: string[] = [], savedPath = process.env.PATH; +// The test sweep's handling of output it cannot read (#189). The fake `docker` answers as no real daemon does for +// `network ls`: a line that is not JSON, a JSON null and one network without a name. To every other call it answers +// nothing and exit 0, as Docker 27.4 does for `ls` with no match; the sweep reads only stdout and the exit status, and +// stderr only to report a failure. What Docker 27.4 really prints is covered against the daemon in agent-container. +const roots: string[] = []; afterEach(() => { - process.env.PATH = savedPath; for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); vi.restoreAllMocks(); }); @@ -20,9 +20,8 @@ it('skips output that is not a network, reports what is not JSON, and never thro writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${calls}'`, `if [ "$1 $2" = "network ls" ]; then printf '%s\\n' 'not json' 'null' '{"ID":"abc"}'; fi`, 'exit 0'].join('\n'), { mode: 0o755 }); - process.env.PATH = `${root}:${savedPath}`; const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); - expect(() => removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test')).not.toThrow(); + expect(() => removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test', join(root, 'docker'))).not.toThrow(); expect(warn.mock.calls.map(call => String(call[0]))).toContain('test: docker printed a line that is not JSON:'); // Nothing it could not read is removed or looked up as a holder. expect(readFileSync(calls, 'utf8').split('\n').filter(line => /^(ps|rm|network rm|volume rm|container rm|container inspect) /.test(line))).toEqual([]); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index 13eec27c..39af2b09 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -1,11 +1,20 @@ import { spawnSync } from 'node:child_process'; import { RUNNER_LABEL } from '../../agents/labels.ts'; +/** + * The real `docker`, found once when this module loads, before any test puts a shim on PATH. A sweep that runs while a + * shim is still on PATH (after a test timed out inside one) would otherwise wait on the shim's own child, which the + * call's SIGKILL does not reach. + */ +export const REAL_DOCKER = spawnSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).stdout?.trim() || 'docker'; + /** * The label a test's Docker shim adds to an object it makes without the runner label (#189), so the test can still * find and remove that object however the test ends. No codeboost code writes or reads it. */ export const TEST_RUN_LABEL = 'io.codeboost.test-run'; +/** Beside it, one value per test file run, so afterAll removes them all with one sweep. */ +export const TEST_FILE_LABEL = 'io.codeboost.test-file'; /** * Removes the containers, networks and volumes labelled with one runner owner, and every container that holds one of @@ -22,11 +31,12 @@ export function removeRunnerObjects(runnerOwner: string, when = 'afterAll'): voi * or not: a test can make a network or volume with one owner and the container that holds it with another, or without * one. Containers and volumes go by force; a network another container joins meanwhile stays. It reports what it finds * and any Docker call that fails, prefixed with `when`, and never throws, so cleanup can call it without hiding an - * earlier error. Each call has a 30 s limit, so all eleven can take up to 5.5 minutes. + * earlier error. Each call has a 30 s limit, so all eleven can take up to 5.5 minutes. It runs `docker`, the real one by + * default (`REAL_DOCKER`). */ -export function removeLabelledObjects(label: string, value: string, when: string): void { +export function removeLabelledObjects(label: string, value: string, when: string, docker = REAL_DOCKER): void { const run = (args: string[]) => { - const result = spawnSync('docker', args, { encoding: 'utf8', timeout: 30_000, killSignal: 'SIGKILL' }); + const result = spawnSync(docker, args, { encoding: 'utf8', timeout: 30_000, killSignal: 'SIGKILL' }); if (result.status !== 0) console.warn(`${when}: docker ${args.slice(0, 2).join(' ')} failed:`, result.error?.message ?? result.stderr.trim()); // An inspect of several objects exits 1 if one is gone, yet prints the rest (Docker 27.4), so the output of a call diff --git a/test/runner-workspace.test.ts b/test/runner-workspace.test.ts index 7c6bbcd2..4fb25a32 100644 --- a/test/runner-workspace.test.ts +++ b/test/runner-workspace.test.ts @@ -36,9 +36,10 @@ afterAll(() => { for (const store of stores) store.close(); for (const root of roots) rmSync(root, { recursive: true, force: true }); } finally { - // Then remove the task storage and path identity probes this run still owns, even when a test failed or the cleanup - // above threw. It may not run before a hook timeout. The hook's 7 minutes leave a minute and a half over this - // sweep's worst case (eleven calls of 30 s), for the cleanup before it. + // Then remove the task storage and path identity probes this run still owns, and every container that holds one of + // their volumes or networks, even when a test failed or the cleanup above threw. It may not run before a hook + // timeout. The hook's 7 minutes leave a minute and a half over this sweep's worst case (eleven calls of 30 s), for + // the cleanup before it. removeRunnerObjects(RUNNER_OWNER); } }, 7 * 60_000); From 3c2c64f0a2ceee0516cc8d83ce6365f6358c0814 Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 20:39:51 -0700 Subject: [PATCH 20/75] Track the race and legacy volumes; cleanup loops inside try with real docker (#189) Review round 19. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 18 +++++++++++++----- test/agent-gate.test.ts | 3 ++- test/agent-network.test.ts | 13 ++++++++----- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 65707387..84e585d6 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -1,5 +1,5 @@ import { execFileSync, spawnSync } from 'node:child_process'; -import { removeLabelledObjects, removeRunnerObjects, TEST_FILE_LABEL, TEST_RUN_LABEL } from './fixtures/docker-sweep.ts'; +import { REAL_DOCKER, removeLabelledObjects, removeRunnerObjects, TEST_FILE_LABEL, TEST_RUN_LABEL } from './fixtures/docker-sweep.ts'; import { fixtureGit } from './fixtures/git.ts'; import { createHash, randomBytes, randomUUID } from 'node:crypto'; import { chmodSync, existsSync, mkdtempSync, mkdirSync, readdirSync, readFileSync, realpathSync, renameSync, rmSync, statSync, @@ -47,7 +47,10 @@ const testOwner = (attemptId = 'fixture') => ({ runnerOwner: TEST_RUNNER_OWNER, const roots: string[] = []; const taskFilesystems: ReturnType[] = []; const containers = new Set(); -/** Put, beside its test-run ID, on every object this file makes without the runner label (#189): afterAll sweeps it. */ +/** + * Put, beside its test-run ID, on every object the `withoutRunnerLabel` wrapper makes and on the holder test's swept + * objects (#189): afterAll sweeps it. Other label-less objects are tracked by name (`containers`, `volumes`). + */ const TEST_FILE_RUN = randomUUID(); /** Volumes a test makes without labels, removed in afterAll if the test did not remove them (it may have timed out). */ const volumes = new Set(); @@ -139,7 +142,8 @@ afterEach(async () => { afterAll(async () => { try { for (const container of containers) spawnSync('docker', ['rm', '--force', container], { stdio: 'ignore' }); - for (const volume of volumes) spawnSync('docker', ['volume', 'rm', '--force', volume], { stdio: 'ignore' }); + for (const volume of volumes) + spawnSync(REAL_DOCKER, ['volume', 'rm', '--force', volume], { stdio: 'ignore', timeout: 30_000, killSignal: 'SIGKILL' }); // Each step, even when an earlier one fails: storage left here is found by the sweep below only by its label, and // every failure is reported. const failures: unknown[] = []; @@ -835,8 +839,8 @@ describe('real Docker agent isolation', () => { it('removed the keeper that a test without its runner label left when it stopped', () => { expect(stoppedEarly, 'the test before did not reach its deliberate stop').toBeDefined(); const left = stoppedEarly!.filesystems, allocationId = taskFilesystemOwner(left).allocationId; - // First, so afterAll does not refuse a keeper this test finds left: its test-run sweep removes that keeper, then its - // runner sweep the volumes. + // First, so afterAll does not refuse a keeper this test finds left: its file-label sweep removes that keeper, then + // its runner sweep the volumes. taskFilesystems.splice(taskFilesystems.indexOf(left), 1); expect(spawnSync('docker', ['container', 'inspect', left.keeper], { stdio: 'ignore' }).status).not.toBe(0); // Its volumes carry the runner label, so ordinary cleanup can remove them now that no keeper holds them. @@ -891,6 +895,8 @@ describe('real Docker agent isolation', () => { it('backs out of a vendor network when a concurrent process claims the same allocation ID', async () => { const data = fixture(), allocationId = randomUUID(), rival = `codeboost-race-${randomUUID()}`; + // Tracked for afterAll should the test time out: the rival carries another runner's label (#189). + volumes.add(rival); try { await withDockerShim(['network', 'create'], racer(rival), () => expect(createVendorNetwork( invocation(data.clone, 'planning'), imageId, allocationId)).rejects.toThrow('still labels a Docker object')); @@ -2402,6 +2408,8 @@ describe('real Docker agent isolation', () => { const mine = crashed(runners[0]), theirs = await live(runners[1]); // An object from a build before runner labels. const legacy = `codeboost-work-legacy-${randomUUID()}`; + // Tracked for afterAll too, should the test time out: it has no runner label (#189). + volumes.add(legacy); undo.push(quietly('volume', 'rm', '--force', legacy)); docker('volume', 'create', '--label', `io.codeboost.allocation=${randomUUID()}`, legacy); // A runner with live storage in this process is refused: its agents would be removed under it. diff --git a/test/agent-gate.test.ts b/test/agent-gate.test.ts index 5315b851..eae872f0 100644 --- a/test/agent-gate.test.ts +++ b/test/agent-gate.test.ts @@ -51,7 +51,8 @@ const writableMetadata = tmpfs('/work/.git', 'size=16m,nr_inodes=512'); const boundedScratch = tmpfs('/tmp', 'size=32m,nr_inodes=4096'); beforeAll(() => { imageId = buildAgentImage(); }, 10 * 60_000); -// The file has no other cleanup: a probe container whose client timed out may still be running. +// The file has no other cleanup: a probe container whose client timed out may still be running. The budget is the +// sweep's worst case (eleven calls of 30 s) and half a minute. afterAll(() => removeRunnerObjects(TEST_RUNNER_OWNER), 6 * 60_000); describe('isolation gate detects breaches', () => { diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index 33e0a43f..653447b8 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -9,7 +9,7 @@ import { assertVendorNetwork, createVendorNetwork, removeVendorNetwork, VENDOR_H VendorNetworkCreationCleanupError, type VendorNetwork } from '../agents/network/network.ts'; import { captureInvocation, type InvocationInput } from '../agents/contract.ts'; -import { removeRunnerObjects } from './fixtures/docker-sweep.ts'; +import { REAL_DOCKER, removeRunnerObjects } from './fixtures/docker-sweep.ts'; // Each run has its own runner owner, so the sweep after this file removes only what this run left behind. Network and // proxy names are random, so the fixed attempt IDs below reach only labels, where this owner tells runs apart. const TEST_RUNNER_OWNER = randomBytes(16).toString('hex'); @@ -40,11 +40,14 @@ beforeAll(async () => { const unlabelled: { kind: 'container' | 'network'; name?: string; file?: string }[] = []; // If setup failed there is no network, and a teardown error would hide the setup failure. afterAll(async () => { - for (const { kind, name, file } of unlabelled) { - const ref = name ?? (file && existsSync(file) ? readFileSync(file, 'utf8') : ''); - if (ref) spawnSync('docker', kind === 'container' ? ['rm', '--force', ref] : ['network', 'rm', ref], { stdio: 'ignore' }); + try { + for (const { kind, name, file } of unlabelled) { + const ref = name ?? (file && existsSync(file) ? readFileSync(file, 'utf8') : ''); + if (ref) spawnSync(REAL_DOCKER, kind === 'container' ? ['rm', '--force', ref] : ['network', 'rm', ref], + { stdio: 'ignore', timeout: 30_000, killSignal: 'SIGKILL' }); + } + if (network) await removeVendorNetwork(network); } - try { if (network) await removeVendorNetwork(network); } finally { // Then remove whatever this run still owns, even when a test failed or the removal above threw. Objects a test // makes without the runner label are found here only while they hold one of this run's networks or volumes; those From 6c4c2eee1320fb9f087060e88797b2feb684e273 Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 20:49:40 -0700 Subject: [PATCH 21/75] Sweep other runners' objects when the test finishes, timeouts included (#189) Review round 20: async tests that make objects under another runner's owner or their own random one register their sweep with onTestFinished, which runs after a timeout too, so test budgets return to their old values; the shims and afterAll use the real docker with a timeout; comment wording. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 153 +++++++++++++++++----------------- test/agent-network.test.ts | 4 +- test/fixtures/docker-sweep.ts | 3 +- test/runner-workspace.test.ts | 44 +++++----- 4 files changed, 101 insertions(+), 103 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 84e585d6..dfa9102e 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -15,7 +15,7 @@ import { assertContainerProfile, createContainerProfile, disposeContainerProfile import { createValidatedContainer, disposeValidatedContainer, prepareTaskFilesystems, removeTaskFilesystems, runContainer, startValidatedContainer, hasExactOptions, keepsNoDaemonLog, validateContainer } from '../agents/container/run.ts'; import { createTaskClone } from '../git/clone.ts'; -import { hasOwnerLabels } from '../agents/labels.ts'; +import { hasOwnerLabels, RUNNER_LABEL } from '../agents/labels.ts'; import { exportTaskDiff, exportTaskPaths, importTaskPaths, isRecoveredTaskStorage, prepareTaskFilesystemsAsync, taskFilesystemOwner, EXPORT_SCRIPT, UnusableRepositoryError } from '../agents/container/storage.ts'; import { checkConflictTree, checkTaskTree, commitTaskChanges, inspectTaskChanges, manifestDigest, MAXIMUM_CHANGES, MAXIMUM_NAME_BYTES, snapshotDeclaredLinks, @@ -52,8 +52,18 @@ const containers = new Set(); * objects (#189): afterAll sweeps it. Other label-less objects are tracked by name (`containers`, `volumes`). */ const TEST_FILE_RUN = randomUUID(); -/** Volumes a test makes without labels, removed in afterAll if the test did not remove them (it may have timed out). */ +/** + * Volumes a test makes without this run's runner label, removed in afterAll if the test did not remove them (it may + * have timed out). + */ const volumes = new Set(); +/** + * For an async test that makes objects under another runner's owner, or its own random one, which no afterAll sweep + * finds: removes what carries `label=value` when the test finishes, passed, failed or timed out (a timed-out body runs + * on, and its finally may never run), with the sweep's own 6-minute budget (#189). + */ +const sweepWhenFinished = (label: string, value: string) => + onTestFinished(() => removeLabelledObjects(label, value, 'onTestFinished'), 6 * 60_000); const profiles: Awaited>[] = []; let imageId = ''; const vendorNetworks: VendorNetwork[] = []; @@ -141,7 +151,8 @@ afterEach(async () => { }, 120_000); afterAll(async () => { try { - for (const container of containers) spawnSync('docker', ['rm', '--force', container], { stdio: 'ignore' }); + for (const container of containers) + spawnSync(REAL_DOCKER, ['rm', '--force', container], { stdio: 'ignore', timeout: 30_000, killSignal: 'SIGKILL' }); for (const volume of volumes) spawnSync(REAL_DOCKER, ['volume', 'rm', '--force', volume], { stdio: 'ignore', timeout: 30_000, killSignal: 'SIGKILL' }); // Each step, even when an earlier one fails: storage left here is found by the sweep below only by its label, and @@ -563,7 +574,7 @@ describe('real Docker agent isolation', () => { it('releases a killed create once its settle window passes with no container', async () => { const data = fixture(), unsettled = await profile(data, 'planning', 'noop'); const shim = join(data.root, 'docker-shim'); mkdirSync(shim); - const realDocker = execFileSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).trim(); + const realDocker = REAL_DOCKER; // The create client hangs until its deadline kills it, so the daemon outcome stays unknown. writeFileSync(join(shim, 'docker'), ['#!/bin/sh', 'if [ "$1" = create ]; then exec sleep 30; fi', `exec '${realDocker}' "$@"`].join('\n'), { mode: 0o755 }); @@ -582,7 +593,7 @@ describe('real Docker agent isolation', () => { const data = fixture(), unsettled = await profile(data, 'planning', 'noop'); const shim = join(data.root, 'docker-shim'); mkdirSync(shim); const created = join(shim, 'created'), failed = join(shim, 'failed'); - const realDocker = execFileSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).trim(); + const realDocker = REAL_DOCKER; // The create client hangs until killed, and the first inspect after it fails for an unrelated reason, so the // create path gives up early, inside the settle window. writeFileSync(join(shim, 'docker'), ['#!/bin/sh', @@ -696,7 +707,7 @@ describe('real Docker agent isolation', () => { onTestFinished(() => removeLabelledObjects(TEST_RUN_LABEL, testRun, 'onTestFinished'), 6 * 60_000); const shim = mkdtempSync(join(tmpdir(), 'codeboost-unlabelled-docker-')); roots.push(shim); mkdirSync(join(shim, 'input')); // afterAll resets this path's mode - const realDocker = execFileSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).trim(); + const realDocker = REAL_DOCKER; writeFileSync(join(shim, 'docker'), [`#!${process.execPath}`, "const { spawnSync } = require('node:child_process');", 'const args = process.argv.slice(2);', @@ -800,7 +811,7 @@ describe('real Docker agent isolation', () => { // 27.4 prints the others and exits 1, and the sweep must still remove them. const shim = mkdtempSync(join(tmpdir(), 'codeboost-sweep-docker-')); roots.push(shim); mkdirSync(join(shim, 'input')); // afterAll resets this path's mode - const realDocker = execFileSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).trim(); + const realDocker = REAL_DOCKER; writeFileSync(join(shim, 'docker'), ['#!/bin/sh', `if [ "$1 $2" = "container inspect" ]; then exec '${realDocker}' "$@" codeboost-gone-${randomUUID()}; fi`, `exec '${realDocker}' "$@"`].join('\n'), { mode: 0o755 }); @@ -861,7 +872,7 @@ describe('real Docker agent isolation', () => { const withDockerShim = async (prefix: string[], script: string, run: () => Promise | T): Promise => { const shim = mkdtempSync(join(tmpdir(), 'codeboost-shim-docker-')); roots.push(shim); mkdirSync(join(shim, 'input')); // afterAll resets this path's mode - const realDocker = execFileSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).trim(); + const realDocker = REAL_DOCKER; writeFileSync(join(shim, 'docker'), [`#!${process.execPath}`, "const { spawnSync } = require('node:child_process');", 'const args = process.argv.slice(2);', @@ -906,6 +917,8 @@ describe('real Docker agent isolation', () => { it('never removes a network found by name after a killed create when its runner label differs', async () => { const data = fixture(), allocationId = randomUUID(); + // The network carries another runner's label, so no sweep of this run finds it: removed by its allocation (#189). + sweepWhenFinished('io.codeboost.allocation', allocationId); // The create lands with another runner's label, and its client is killed before it reports the ID. const foreign = [ `const at = args.findIndex(arg => arg.startsWith('io.codeboost.runner='));`, @@ -916,20 +929,14 @@ describe('real Docker agent isolation', () => { const listed = () => spawnSync('docker', ['network', 'ls', '--quiet', '--no-trunc', '--filter', `label=io.codeboost.allocation=${allocationId}`], { encoding: 'utf8' }).stdout?.trim() ?? ''; let left = ''; - try { - const created = await withDockerShim(['network', 'create'], foreign, () => - createVendorNetwork(invocation(data.clone, 'planning'), imageId, allocationId).then(() => undefined, error => error)); - left = listed(); - expect(created).toBeInstanceOf(VendorNetworkCreationCleanupError); - const cleanup = (created as VendorNetworkCreationCleanupError).errors[1] as AggregateError; - expect(cleanup.errors.map(error => (error as Error).message)).toEqual(['Refused to remove unowned vendor network.']); - expect(left).not.toBe(''); - } finally { - // The network carries another runner's label, so no sweep of this run finds it: removed by its allocation (#189). - // The sweep may take up to 5.5 minutes, which the test's budget allows for. - removeLabelledObjects('io.codeboost.allocation', allocationId, 'test'); - } - }, 7 * 60_000); + const created = await withDockerShim(['network', 'create'], foreign, () => + createVendorNetwork(invocation(data.clone, 'planning'), imageId, allocationId).then(() => undefined, error => error)); + left = listed(); + expect(created).toBeInstanceOf(VendorNetworkCreationCleanupError); + const cleanup = (created as VendorNetworkCreationCleanupError).errors[1] as AggregateError; + expect(cleanup.errors.map(error => (error as Error).message)).toEqual(['Refused to remove unowned vendor network.']); + expect(left).not.toBe(''); + }, 60_000); it('never removes an agent container found by name after a killed create when a label is missing', async () => { const data = fixture(), live = await profile(data, 'planning', 'must-not-run'); @@ -2317,25 +2324,24 @@ describe('real Docker agent isolation', () => { it('keeps a failed allocation whose cleanup did not settle live, so recovery in this process refuses its runner', async () => { const data = fixture(), runnerOwner = randomBytes(16).toString('hex'); const owner = { runnerOwner, attemptId: 'unsettled-allocation', allocationId: randomUUID() }; + // Another runner's objects, so no sweep of this run finds them: removed by allocation when the test finishes (#189). + sweepWhenFinished('io.codeboost.allocation', owner.allocationId); // The seeder is refused, and every volume removal fails, so the allocation's cleanup does not settle. const failing = [ "if (args[0] === 'run' && args[1] === '--rm') result = { status: 1, stdout: '', stderr: 'refused' };", "else if (args[0] === 'volume' && args[1] === 'rm') result = { status: 1, stdout: '', stderr: 'volume is in use' };", 'else result = run(args);', ].join('\n'); - try { - await withDockerShim([], failing, () => expect(() => prepareTaskFilesystems(data.clone, storageLimits, imageId, - owner)).toThrow('cleanup did not settle')); - await expect(recoverLeftovers(runnerOwner)).rejects.toThrow('still holds task storage'); - } finally { - // Another runner's objects, so no sweep of this run finds them: removed by allocation, by full ID, never throwing - // (#189). The sweep may take up to 5.5 minutes, which the test's budget allows for. - removeLabelledObjects('io.codeboost.allocation', owner.allocationId, 'test'); - } - }, 7 * 60_000); + await withDockerShim([], failing, () => expect(() => prepareTaskFilesystems(data.clone, storageLimits, imageId, + owner)).toThrow('cleanup did not settle')); + await expect(recoverLeftovers(runnerOwner)).rejects.toThrow('still holds task storage'); + }, 60_000); it('recovers only its own runner after a restart, keeping its storage for removal by handle', async () => { const data = fixture(), runners = [randomBytes(16).toString('hex'), randomBytes(16).toString('hex')] as const; + // Both runners are this test's own random owners, so no sweep of this run finds what an undo step could not remove + // or never recorded: each is swept when the test finishes (#189). + for (const runner of runners) sweepWhenFinished(RUNNER_LABEL, runner); // Every object is registered for cleanup as soon as it exists, so a failure anywhere in setup leaks nothing. // Cleanup runs in reverse: containers go before the networks and volumes they use. const undo: (() => unknown)[] = []; @@ -2445,16 +2451,15 @@ describe('real Docker agent isolation', () => { for (const step of undo.reverse()) { try { await step(); } catch (error) { failures.push(error); } } - // Both runners are this test's own random owners, so no sweep of this run finds what an undo step could not - // remove or never recorded (#189). Each sweep may take up to 5.5 minutes, which the test's budget allows for. - for (const runner of runners) removeRunnerObjects(runner, 'test'); if (failures.length) throw new AggregateError(failures, 'Recovery test cleanup failed.'); } - }, 15 * 60_000); + }, 180_000); it('exports through a recovery handle after a restart, then removes the storage by that handle', async () => { const data = fixture({ hostile: agentBase }); const owner = { runnerOwner: randomBytes(16).toString('hex'), attemptId: `restart-${randomUUID()}`, allocationId: randomUUID() }; + // Its own random runner, so no sweep of this run finds what it leaves: removed by allocation when it finishes (#189). + sweepWhenFinished('io.codeboost.allocation', owner.allocationId); const staging = join(data.root, 'restart-staging'); mkdirSync(staging); // An earlier process clones and allocates task storage, then exits without releasing it, as a crash does: this // process never held the allocator value. @@ -2472,40 +2477,34 @@ describe('real Docker agent isolation', () => { "process.stdout.write('\\n' + JSON.stringify(prepareTaskFilesystems(clone, limits, imageId, owner)));", ].join('\n')], { encoding: 'utf8', timeout: 120_000, env: { ...process.env, EARLIER: JSON.stringify({ source: data.source, staging, head: data.clone.head, owner }) } }); - try { - if (earlier.status !== 0) throw new Error(`The earlier process failed (${earlier.status}): ${earlier.stderr}`); - const left = JSON.parse(earlier.stdout.trim().split('\n').pop()!) as ReturnType; - agentChanges(left); - const report = await recoverLeftovers(owner.runnerOwner); - expect(report.removed).toEqual([]); - expect(report.storage).toEqual([{ ...owner, workVolume: left.workVolume, metadataVolume: left.metadataVolume, - keeper: left.keeper }]); - const handle = report.storage[0]!; - // D keeps no baseline across a restart: the export needs the one F recorded, and checks the metadata against it. - await expect(exportTaskDiff(handle, { base: data.clone.head, imageId })).rejects.toThrow('needs the metadataBaseline'); - await expect(exportTaskDiff(handle, { base: data.clone.head, imageId, metadataBaseline: 'c'.repeat(64) })) - .rejects.toThrow('metadata changed'); - const before = storageSnapshot(left); - const exported = await exportTaskDiff(handle, { base: data.clone.head, imageId, metadataBaseline: left.metadataBaseline }); - expect(storageSnapshot(left)).toBe(before); - const text = exported.diff.toString('utf8'); - expect(exported.truncated).toBe(false); - for (const expected of ['+committed', '+changed', '+staged only', '+brand new', 'GIT binary patch']) - expect(text).toContain(expected); - // Every export, refused or not, removed its container: only the keeper is left before release. - expect(docker('ps', '--all', '--quiet', '--no-trunc', '--filter', `label=io.codeboost.allocation=${owner.allocationId}`)) - .toBe(docker('container', 'inspect', '--format', '{{.Id}}', left.keeper)); - // The handle still releases the storage after the export, once. - removeTaskFilesystems(handle); - expect(isRecoveredTaskStorage(handle)).toBe(false); - for (const [kind, ref] of [['container', left.keeper], ['volume', left.workVolume], ['volume', left.metadataVolume]] as const) - expect(spawnSync('docker', [kind, 'inspect', ref], { stdio: 'ignore' }).status).not.toBe(0); - } finally { - // Never throws, so a failure above is the one reported. By full ID: Docker resolves a name before an ID prefix. The - // sweep may take up to 5.5 minutes, which the test's budget allows for. - removeLabelledObjects('io.codeboost.allocation', owner.allocationId, 'test'); - } - }, 9 * 60_000); + if (earlier.status !== 0) throw new Error(`The earlier process failed (${earlier.status}): ${earlier.stderr}`); + const left = JSON.parse(earlier.stdout.trim().split('\n').pop()!) as ReturnType; + agentChanges(left); + const report = await recoverLeftovers(owner.runnerOwner); + expect(report.removed).toEqual([]); + expect(report.storage).toEqual([{ ...owner, workVolume: left.workVolume, metadataVolume: left.metadataVolume, + keeper: left.keeper }]); + const handle = report.storage[0]!; + // D keeps no baseline across a restart: the export needs the one F recorded, and checks the metadata against it. + await expect(exportTaskDiff(handle, { base: data.clone.head, imageId })).rejects.toThrow('needs the metadataBaseline'); + await expect(exportTaskDiff(handle, { base: data.clone.head, imageId, metadataBaseline: 'c'.repeat(64) })) + .rejects.toThrow('metadata changed'); + const before = storageSnapshot(left); + const exported = await exportTaskDiff(handle, { base: data.clone.head, imageId, metadataBaseline: left.metadataBaseline }); + expect(storageSnapshot(left)).toBe(before); + const text = exported.diff.toString('utf8'); + expect(exported.truncated).toBe(false); + for (const expected of ['+committed', '+changed', '+staged only', '+brand new', 'GIT binary patch']) + expect(text).toContain(expected); + // Every export, refused or not, removed its container: only the keeper is left before release. + expect(docker('ps', '--all', '--quiet', '--no-trunc', '--filter', `label=io.codeboost.allocation=${owner.allocationId}`)) + .toBe(docker('container', 'inspect', '--format', '{{.Id}}', left.keeper)); + // The handle still releases the storage after the export, once. + removeTaskFilesystems(handle); + expect(isRecoveredTaskStorage(handle)).toBe(false); + for (const [kind, ref] of [['container', left.keeper], ['volume', left.workVolume], ['volume', left.metadataVolume]] as const) + expect(spawnSync('docker', [kind, 'inspect', ref], { stdio: 'ignore' }).status).not.toBe(0); + }, 180_000); it('never removes a task keeper replaced by another runner, but still removes the owned volumes', () => { const data = fixture(), filesystems = data.filesystems, owner = taskFilesystemOwner(filesystems); @@ -2522,8 +2521,8 @@ describe('real Docker agent isolation', () => { } finally { spawnSync('docker', ['rm', '--force', filesystems.keeper], { stdio: 'ignore' }); // Never throws, so a failure above is the one reported: what the removal leaves, the allocation sweep below removes, - // the keeper with another runner's label included (#189). The sweep may - // take up to 5.5 minutes, which the test's budget allows for. + // the keeper with another runner's label included (#189). The sweep may take up to 5.5 minutes, which the test's + // budget allows for. try { removeTaskFilesystems(filesystems); } catch (error) { console.warn('cleanup: removing the storage failed:', error); } removeLabelledObjects('io.codeboost.allocation', owner.allocationId, 'test'); } @@ -2532,8 +2531,9 @@ describe('real Docker agent isolation', () => { it('refuses task storage that belongs to another runner', async () => { const data = fixture(), allocationId = randomUUID(); let foreign: ReturnType | undefined; - // Removed here: it carries another runner's label, so no sweep of this run finds it (#189). Then whatever still - // carries its allocation, should its creation or removal not settle. + // It carries another runner's label, so no sweep of this run finds it: removed here, and whatever still carries its + // allocation when the test finishes, should its creation or removal not settle or the test time out (#189). + sweepWhenFinished('io.codeboost.allocation', allocationId); try { foreign = prepareTaskFilesystems(data.clone, { workBytes: 16 * 1024 * 1024, workInodes: 512, metadataBytes: 16 * 1024 * 1024, metadataInodes: 512 }, imageId, @@ -2543,9 +2543,8 @@ describe('real Docker agent isolation', () => { } finally { // Never throws, so a failure above is the one reported; the sweep removes whatever the removal left. try { if (foreign) removeTaskFilesystems(foreign); } catch (error) { console.warn('cleanup: removing the storage failed:', error); } - removeLabelledObjects('io.codeboost.allocation', allocationId, 'test'); } - }, 7 * 60_000); + }, 60_000); it('refuses to seed a clone whose staging directory was replaced after creation', async () => { const data = fixture(); @@ -2585,7 +2584,7 @@ describe('real Docker agent isolation', () => { '--filter', `label=io.codeboost.runner=${TEST_RUNNER_OWNER}`).split('\n').filter(Boolean)); const before = keepers(); const shim = join(data.root, 'docker-shim'); mkdirSync(shim); - const realDocker = execFileSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).trim(); + const realDocker = REAL_DOCKER; // The keeper's create client hangs until killed, and the real create lands in the daemon afterwards. writeFileSync(join(shim, 'docker'), ['#!/bin/sh', `if [ "$1" = create ] && [ "$2" = --name ]; then ( sleep 10; exec '${realDocker}' "$@" ) >/dev/null 2>&1 { network = await createVendorNetwork(invocation, imageId, randomUUID()); }, 10 * 60_000); /** - * Objects a test makes without labels, by name or by the file its shim records the name in, so afterAll removes them - * should the test time out before its own removal runs (#189). + * Objects a test makes without this run's runner label, by name or by the file its shim records the name in, so + * afterAll removes them should the test time out before its own removal runs (#189). */ const unlabelled: { kind: 'container' | 'network'; name?: string; file?: string }[] = []; // If setup failed there is no network, and a teardown error would hide the setup failure. diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index 39af2b09..14f04043 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -19,7 +19,8 @@ export const TEST_FILE_LABEL = 'io.codeboost.test-file'; /** * Removes the containers, networks and volumes labelled with one runner owner, and every container that holds one of * those networks or volumes, whatever its labels (see `removeLabelledObjects`). A test file's afterAll calls it after - * its other cleanup, and a test that makes objects under its own runner owner calls it in its finally. + * its other cleanup, and a test that makes objects under its own runner owner sweeps them (by owner or by allocation) + * when it finishes. */ export function removeRunnerObjects(runnerOwner: string, when = 'afterAll'): void { removeLabelledObjects(RUNNER_LABEL, runnerOwner, when); diff --git a/test/runner-workspace.test.ts b/test/runner-workspace.test.ts index 4fb25a32..ccd949bc 100644 --- a/test/runner-workspace.test.ts +++ b/test/runner-workspace.test.ts @@ -4,7 +4,7 @@ import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSy import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { promisify } from 'node:util'; -import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { afterAll, beforeAll, describe, expect, it, onTestFinished } from 'vitest'; import { buildAgentImage } from '../agents/container/image.ts'; import { pathIdentityProbeArgs, probeTaskStoragePathIdentity, type TaskFilesystems } from '../agents/container/storage.ts'; import { recoverLeftovers } from '../agents/recovery.ts'; @@ -239,28 +239,26 @@ describe('task storage path identity (#162)', () => { it('removes a probe a killed runner left, with its volume, in the next startup\'s recovery, and no other runner\'s (#181)', async () => { // Tokens of their own: recovery removes every agent container of its runner, and this file's may have some. const mine = randomBytes(16).toString('hex'), theirs = randomBytes(16).toString('hex'); - try { - const left = leftoverProbe(mine), other = leftoverProbe(theirs); - // Both parts carry the runner's token and the probe kind. - for (const labels of [docker('container', 'inspect', '--format', '{{json .Config.Labels}}', left.name), - docker('volume', 'inspect', '--format', '{{json .Labels}}', left.volume)]) - expect(JSON.parse(labels)).toMatchObject({ 'io.codeboost.runner': mine, 'io.codeboost.probe': 'path-identity' }); - // A container of this runner, named like a probe, without the probe kind label: reported, never removed. - const unmarked = `codeboost-path-identity-${randomUUID()}`; - docker('create', '--name', unmarked, '--label', `io.codeboost.runner=${mine}`, '--entrypoint', 'true', imageId); - const report = await recoverLeftovers(mine, 120_000, { unowned: false }); - expect(report.removed.map(resource => resource.name).sort()).toEqual([left.name, left.volume].sort()); - expect(() => docker('container', 'inspect', left.name)).toThrow(/No such container/); - expect(() => docker('volume', 'inspect', left.volume)).toThrow(/no such volume/i); - expect(report.unowned).toEqual([expect.objectContaining({ name: unmarked, reason: 'unknown-kind' })]); - for (const ref of [other.name, unmarked]) expect(docker('container', 'inspect', '--format', '{{.State.Status}}', ref)).toBe('created'); - expect(docker('volume', 'inspect', '--format', '{{.Name}}', other.volume)).toBe(other.volume); - } finally { - // Each sweep may take up to 5.5 minutes, which the test's budget allows for. - removeRunnerObjects(mine, 'test'); - removeRunnerObjects(theirs, 'test'); - } - }, 14 * 60_000); + // This test's own random runners, so no sweep of this run finds them: each is swept when the test finishes, however + // it ends (a timed-out body runs on, and its finally may never run), with the sweep's own budget (#189). + for (const runner of [mine, theirs]) + onTestFinished(() => removeRunnerObjects(runner, 'onTestFinished'), 6 * 60_000); + const left = leftoverProbe(mine), other = leftoverProbe(theirs); + // Both parts carry the runner's token and the probe kind. + for (const labels of [docker('container', 'inspect', '--format', '{{json .Config.Labels}}', left.name), + docker('volume', 'inspect', '--format', '{{json .Labels}}', left.volume)]) + expect(JSON.parse(labels)).toMatchObject({ 'io.codeboost.runner': mine, 'io.codeboost.probe': 'path-identity' }); + // A container of this runner, named like a probe, without the probe kind label: reported, never removed. + const unmarked = `codeboost-path-identity-${randomUUID()}`; + docker('create', '--name', unmarked, '--label', `io.codeboost.runner=${mine}`, '--entrypoint', 'true', imageId); + const report = await recoverLeftovers(mine, 120_000, { unowned: false }); + expect(report.removed.map(resource => resource.name).sort()).toEqual([left.name, left.volume].sort()); + expect(() => docker('container', 'inspect', left.name)).toThrow(/No such container/); + expect(() => docker('volume', 'inspect', left.volume)).toThrow(/no such volume/i); + expect(report.unowned).toEqual([expect.objectContaining({ name: unmarked, reason: 'unknown-kind' })]); + for (const ref of [other.name, unmarked]) expect(docker('container', 'inspect', '--format', '{{.State.Status}}', ref)).toBe('created'); + expect(docker('volume', 'inspect', '--format', '{{.Name}}', other.volume)).toBe(other.volume); + }, 120_000); it('runs only the image the trusted builder produced', async () => { await expect(probeTaskStoragePathIdentity(`sha256:${'0'.repeat(64)}`, RUNNER_OWNER)).rejects.toThrow('Agent image was not produced by the trusted validated builder.'); await expect(probeTaskStoragePathIdentity('alpine', RUNNER_OWNER)).rejects.toThrow('requires the immutable built image ID'); From 18ba72b5e11fe76344d8e5a9fd1c97d562fa41cb Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 20:58:47 -0700 Subject: [PATCH 22/75] Real docker in every shim; foreign-name containers swept on finish; killed-call test (#189) Review round 21: every shim calls the real docker found at load and refuses when there is none; agent-supervisor's two foreign-name containers are removed by name when their test finishes, timeouts included; a unit test shows a killed call's output is not read; one more race volume tracked; comments name every removal afterAll makes and what the finish-time sweep cannot cover. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 27 +++++++++++++++------------ test/agent-network.test.ts | 22 +++++++++++----------- test/agent-supervisor.test.ts | 10 +++++++--- test/docker-sweep.test.ts | 12 ++++++++++++ test/fixtures/docker-sweep.ts | 7 ++++++- test/runner-workspace.test.ts | 4 ++-- 6 files changed, 53 insertions(+), 29 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index dfa9102e..0d09df4f 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -1,5 +1,5 @@ import { execFileSync, spawnSync } from 'node:child_process'; -import { REAL_DOCKER, removeLabelledObjects, removeRunnerObjects, TEST_FILE_LABEL, TEST_RUN_LABEL } from './fixtures/docker-sweep.ts'; +import { REAL_DOCKER, removeLabelledObjects, removeRunnerObjects, requireRealDocker, TEST_FILE_LABEL, TEST_RUN_LABEL } from './fixtures/docker-sweep.ts'; import { fixtureGit } from './fixtures/git.ts'; import { createHash, randomBytes, randomUUID } from 'node:crypto'; import { chmodSync, existsSync, mkdtempSync, mkdirSync, readdirSync, readFileSync, realpathSync, renameSync, rmSync, statSync, @@ -60,7 +60,8 @@ const volumes = new Set(); /** * For an async test that makes objects under another runner's owner, or its own random one, which no afterAll sweep * finds: removes what carries `label=value` when the test finishes, passed, failed or timed out (a timed-out body runs - * on, and its finally may never run), with the sweep's own 6-minute budget (#189). + * on, and its finally may never run), with the sweep's own 6-minute budget (#189). Not covered: an object a timed-out + * body makes after this sweep has run. */ const sweepWhenFinished = (label: string, value: string) => onTestFinished(() => removeLabelledObjects(label, value, 'onTestFinished'), 6 * 60_000); @@ -170,8 +171,9 @@ afterAll(async () => { } finally { // The runner owner is this run's own, so nothing else would find what this run leaves behind. Objects a test // makes under another owner or without the runner label are not found here, unless they hold one of this run's - // volumes or networks: those this file labels with its own run ID are found by it, and the other tests remove their - // own. It may not run before a hook timeout. Its budget covers two sweeps of up to 5.5 minutes each and the cleanup + // volumes or networks: those this file labels with its own run ID are found by it, the try above removes those + // tracked by name (`containers`, `volumes`), and the other tests remove their own. It may not run before a hook + // timeout. Its budget covers two sweeps of up to 5.5 minutes each and the cleanup // before them. removeLabelledObjects(TEST_FILE_LABEL, TEST_FILE_RUN, 'afterAll'); removeRunnerObjects(TEST_RUNNER_OWNER); @@ -574,7 +576,7 @@ describe('real Docker agent isolation', () => { it('releases a killed create once its settle window passes with no container', async () => { const data = fixture(), unsettled = await profile(data, 'planning', 'noop'); const shim = join(data.root, 'docker-shim'); mkdirSync(shim); - const realDocker = REAL_DOCKER; + const realDocker = requireRealDocker(); // The create client hangs until its deadline kills it, so the daemon outcome stays unknown. writeFileSync(join(shim, 'docker'), ['#!/bin/sh', 'if [ "$1" = create ]; then exec sleep 30; fi', `exec '${realDocker}' "$@"`].join('\n'), { mode: 0o755 }); @@ -593,7 +595,7 @@ describe('real Docker agent isolation', () => { const data = fixture(), unsettled = await profile(data, 'planning', 'noop'); const shim = join(data.root, 'docker-shim'); mkdirSync(shim); const created = join(shim, 'created'), failed = join(shim, 'failed'); - const realDocker = REAL_DOCKER; + const realDocker = requireRealDocker(); // The create client hangs until killed, and the first inspect after it fails for an unrelated reason, so the // create path gives up early, inside the settle window. writeFileSync(join(shim, 'docker'), ['#!/bin/sh', @@ -707,7 +709,7 @@ describe('real Docker agent isolation', () => { onTestFinished(() => removeLabelledObjects(TEST_RUN_LABEL, testRun, 'onTestFinished'), 6 * 60_000); const shim = mkdtempSync(join(tmpdir(), 'codeboost-unlabelled-docker-')); roots.push(shim); mkdirSync(join(shim, 'input')); // afterAll resets this path's mode - const realDocker = REAL_DOCKER; + const realDocker = requireRealDocker(); writeFileSync(join(shim, 'docker'), [`#!${process.execPath}`, "const { spawnSync } = require('node:child_process');", 'const args = process.argv.slice(2);', @@ -811,7 +813,7 @@ describe('real Docker agent isolation', () => { // 27.4 prints the others and exits 1, and the sweep must still remove them. const shim = mkdtempSync(join(tmpdir(), 'codeboost-sweep-docker-')); roots.push(shim); mkdirSync(join(shim, 'input')); // afterAll resets this path's mode - const realDocker = REAL_DOCKER; + const realDocker = requireRealDocker(); writeFileSync(join(shim, 'docker'), ['#!/bin/sh', `if [ "$1 $2" = "container inspect" ]; then exec '${realDocker}' "$@" codeboost-gone-${randomUUID()}; fi`, `exec '${realDocker}' "$@"`].join('\n'), { mode: 0o755 }); @@ -872,7 +874,7 @@ describe('real Docker agent isolation', () => { const withDockerShim = async (prefix: string[], script: string, run: () => Promise | T): Promise => { const shim = mkdtempSync(join(tmpdir(), 'codeboost-shim-docker-')); roots.push(shim); mkdirSync(join(shim, 'input')); // afterAll resets this path's mode - const realDocker = REAL_DOCKER; + const realDocker = requireRealDocker(); writeFileSync(join(shim, 'docker'), [`#!${process.execPath}`, "const { spawnSync } = require('node:child_process');", 'const args = process.argv.slice(2);', @@ -896,6 +898,8 @@ describe('real Docker agent isolation', () => { it('backs out of task storage when a concurrent process claims the same allocation ID', async () => { const data = fixture(), owner = testOwner('storage-race'), rival = `codeboost-race-${randomUUID()}`; + // Tracked for afterAll should the test time out: the rival carries another runner's label (#189). + volumes.add(rival); try { await withDockerShim(['volume', 'create'], racer(rival), () => expect(() => prepareTaskFilesystems(data.clone, { workBytes: 16 * 1024 * 1024, workInodes: 512, metadataBytes: 16 * 1024 * 1024, metadataInodes: 512 }, @@ -928,10 +932,9 @@ describe('real Docker agent isolation', () => { ].join('\n'); const listed = () => spawnSync('docker', ['network', 'ls', '--quiet', '--no-trunc', '--filter', `label=io.codeboost.allocation=${allocationId}`], { encoding: 'utf8' }).stdout?.trim() ?? ''; - let left = ''; const created = await withDockerShim(['network', 'create'], foreign, () => createVendorNetwork(invocation(data.clone, 'planning'), imageId, allocationId).then(() => undefined, error => error)); - left = listed(); + const left = listed(); expect(created).toBeInstanceOf(VendorNetworkCreationCleanupError); const cleanup = (created as VendorNetworkCreationCleanupError).errors[1] as AggregateError; expect(cleanup.errors.map(error => (error as Error).message)).toEqual(['Refused to remove unowned vendor network.']); @@ -2584,7 +2587,7 @@ describe('real Docker agent isolation', () => { '--filter', `label=io.codeboost.runner=${TEST_RUNNER_OWNER}`).split('\n').filter(Boolean)); const before = keepers(); const shim = join(data.root, 'docker-shim'); mkdirSync(shim); - const realDocker = REAL_DOCKER; + const realDocker = requireRealDocker(); // The keeper's create client hangs until killed, and the real create lands in the daemon afterwards. writeFileSync(join(shim, 'docker'), ['#!/bin/sh', `if [ "$1" = create ] && [ "$2" = --name ]; then ( sleep 10; exec '${realDocker}' "$@" ) >/dev/null 2>&1 { } finally { // Then remove whatever this run still owns, even when a test failed or the removal above threw. Objects a test - // makes without the runner label are found here only while they hold one of this run's networks or volumes; those - // tests remove their own. It may not run before a hook timeout. - // The hook's 7 minutes leave a minute and a half over this sweep's worst case (eleven calls of 30 s), for the - // cleanup before it. + // makes without the runner label are found here only while they hold one of this run's networks or volumes; the + // tests remove their own, and the try above removes those registered in `unlabelled`. It may not run before a hook + // timeout. The hook's 7 minutes leave a minute and a half over this sweep's worst case (eleven calls of 30 s), for + // the cleanup before it. removeRunnerObjects(TEST_RUNNER_OWNER); } }, 7 * 60_000); @@ -61,7 +61,7 @@ afterAll(async () => { describe('vendor-only egress', () => { it('keeps failed allocation and its cleanup inside the caller deadline', async () => { const shim = mkdtempSync(join(tmpdir(), 'docker-shim-')); - const realDocker = execFileSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).trim(); + const realDocker = requireRealDocker(); // Every network operation hangs, so both setup and cleanup can only end by deadline. writeFileSync(join(shim, 'docker'), ['#!/bin/sh', 'if [ "$1" = network ]; then exec sleep 30; fi', `exec '${realDocker}' "$@"`].join('\n'), { mode: 0o755 }); @@ -74,7 +74,7 @@ describe('vendor-only egress', () => { it('removes a network that lands in the daemon after its create client was killed', async () => { const shim = mkdtempSync(join(tmpdir(), 'docker-shim-')), requested = join(shim, 'network-name'); - const realDocker = execFileSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).trim(); + const realDocker = requireRealDocker(); // The create client hangs until killed, and the real create lands after that, inside the cleanup reserve. // The shim records this test's network name, since other suites may create egress networks concurrently. writeFileSync(join(shim, 'docker'), ['#!/bin/sh', @@ -100,7 +100,7 @@ describe('vendor-only egress', () => { it('leaves a same-named network alone when the daemon refuses the create', async () => { const shim = mkdtempSync(join(tmpdir(), 'docker-shim-')), recorded = join(shim, 'foreign'); unlabelled.push({ kind: 'network', file: recorded }); - const realDocker = execFileSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).trim(); + const realDocker = requireRealDocker(); // The name is taken by someone else's network just before our create, so the daemon refuses it. writeFileSync(join(shim, 'docker'), ['#!/bin/sh', `if [ "$1" = network ] && [ "$2" = create ]; then for arg; do last="$arg"; done; ` @@ -133,7 +133,7 @@ describe('vendor-only egress', () => { it('removes a proxy that was created even though its start failed', async () => { const shim = mkdtempSync(join(tmpdir(), 'docker-shim-')), recorded = join(shim, 'proxy'); - const realDocker = execFileSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).trim(); + const realDocker = requireRealDocker(); // The proxy container is created, then its start fails: a container this allocation owns by ID. writeFileSync(join(shim, 'docker'), ['#!/bin/sh', `if [ "$1" = start ] && [ "$2" != --attach ]; then printf %s "$2" > '${recorded}'; ` @@ -157,7 +157,7 @@ describe('vendor-only egress', () => { it('neither reports nor removes a same-named proxy when the daemon refuses the create', async () => { const shim = mkdtempSync(join(tmpdir(), 'docker-shim-')), recorded = join(shim, 'foreign'); unlabelled.push({ kind: 'container', file: recorded }); - const realDocker = execFileSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).trim(); + const realDocker = requireRealDocker(); // Someone else's container takes the proxy name just before our create, so the daemon refuses it. writeFileSync(join(shim, 'docker'), ['#!/bin/sh', `if [ "$1" = create ] && [ "$2" = --name ] && case "$3" in codeboost-proxy-*) true;; *) false;; esac; then ` @@ -207,7 +207,7 @@ describe('vendor-only egress', () => { it('does not delete a same-named stand-in when setup fails after the proxy exists', async () => { const shim = mkdtempSync(join(tmpdir(), 'docker-shim-')), recorded = join(shim, 'impostor'); unlabelled.push({ kind: 'container', file: recorded }); - const realDocker = execFileSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).trim(); + const realDocker = requireRealDocker(); // The readiness exec swaps the proxy for a same-named, same-labelled stand-in, then fails setup. writeFileSync(join(shim, 'docker'), ['#!/bin/sh', 'if [ "$1" = exec ]; then', ` name=$('${realDocker}' inspect -f '{{.Name}}' "$2" | sed 's#^/##')`, diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 6d94e551..5071026a 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -1,11 +1,11 @@ import { randomBytes, randomUUID } from 'node:crypto'; import { execFileSync, spawnSync } from 'node:child_process'; -import { removeRunnerObjects } from './fixtures/docker-sweep.ts'; +import { REAL_DOCKER, removeRunnerObjects, requireRealDocker } from './fixtures/docker-sweep.ts'; import { fixtureGit } from './fixtures/git.ts'; import { chmodSync, mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; -import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { afterAll, beforeAll, describe, expect, it, onTestFinished } from 'vitest'; import { startClaudeInvocation } from '../agents/adapters/claude.ts'; import { readCodexOutput } from '../agents/adapters/codex.ts'; import { ACKNOWLEDGEMENT_SCRIPT, isInvocationActive, readBoundedContainerFile, retainSetupCleanup, @@ -332,7 +332,7 @@ describe('container invocation supervisor', () => { const hangingDocker = (match: string) => { const shim = mkdtempSync(join(tmpdir(), 'codeboost-slow-docker-')); roots.push(shim); mkdirSync(join(shim, 'input')); // afterAll resets this path's mode - const realDocker = execFileSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).trim(); + const realDocker = requireRealDocker(); writeFileSync(join(shim, 'docker'), ['#!/bin/sh', `case "$*" in ${match}) exec sleep 60;; esac`, `exec '${realDocker}' "$@"`].join('\n'), { mode: 0o755 }); return `${shim}:${process.env.PATH}`; @@ -387,6 +387,8 @@ describe('container invocation supervisor', () => { it('releases its own resources, and leaves a same-named container another invocation owns', async () => { const current = await profile(fixture(), 'finite-output', 'foreign-name'); + // Removed by name when the test finishes, timeouts included: its label value is shared across runs (#189). + onTestFinished(() => { spawnSync(REAL_DOCKER, ['rm', '--force', current.name], { stdio: 'ignore', timeout: 30_000, killSignal: 'SIGKILL' }); }); try { // Inside the try, so its removal runs even if the create's client fails after the daemon made it (#189). execFileSync('docker', ['create', '--name', current.name, '--label', 'io.codeboost.invocation=foreign', @@ -440,6 +442,8 @@ describe('container invocation supervisor', () => { it('releases only the profile when rejecting before creation, even if its name is held elsewhere', async () => { const current = await profile(fixture(), 'noop'); // A foreign container occupies the deterministic name, so container-level cleanup could never settle. + // Removed by name when the test finishes, timeouts included: its label value is shared across runs (#189). + onTestFinished(() => { spawnSync(REAL_DOCKER, ['rm', '--force', current.name], { stdio: 'ignore', timeout: 30_000, killSignal: 'SIGKILL' }); }); try { execFileSync('docker', ['create', '--name', current.name, '--label', 'io.codeboost.invocation=someone-else', '--entrypoint', 'true', imageId], { stdio: 'ignore' }); diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index 0d75f89e..6d7dcc26 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -26,3 +26,15 @@ it('skips output that is not a network, reports what is not JSON, and never thro // Nothing it could not read is removed or looked up as a holder. expect(readFileSync(calls, 'utf8').split('\n').filter(line => /^(ps|rm|network rm|volume rm|container rm|container inspect) /.test(line))).toEqual([]); }); + +it('reads nothing from a docker call that was killed, whose output may be cut short', () => { + const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); + const calls = join(root, 'calls'); + // A well-formed network line, then the client is killed before it exits by itself (as at the 30 s limit). + writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${calls}'`, + `if [ "$1 $2" = "network ls" ]; then printf '%s\\n' '{"ID":"${'a'.repeat(64)}","Name":"swept"}'; kill -9 $$; fi`, 'exit 0'].join('\n'), + { mode: 0o755 }); + vi.spyOn(console, 'warn').mockImplementation(() => undefined); + removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test', join(root, 'docker')); + expect(readFileSync(calls, 'utf8').split('\n').filter(line => /^(ps|network rm) /.test(line))).toEqual([]); +}); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index 14f04043..bdd4e0e1 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -7,6 +7,11 @@ import { RUNNER_LABEL } from '../../agents/labels.ts'; * call's SIGKILL does not reach. */ export const REAL_DOCKER = spawnSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).stdout?.trim() || 'docker'; +/** The real `docker` for a shim to call; refuses when none was found, since a shim calling `docker` would call itself. */ +export function requireRealDocker(): string { + if (!REAL_DOCKER.startsWith('/')) throw new Error('No docker on PATH when the test files loaded.'); + return REAL_DOCKER; +} /** * The label a test's Docker shim adds to an object it makes without the runner label (#189), so the test can still @@ -70,7 +75,7 @@ export function removeLabelledObjects(label: string, value: string, when: string const holders = kind === 'volume' || !candidates.length ? candidates : run(['container', 'inspect', '--format', '{{json .Id}} {{json .NetworkSettings.Networks}}', ...candidates]).flatMap(line => { const at = line.indexOf(' '), id = parsed(line.slice(0, at)), on = parsed>(line.slice(at + 1)); - return typeof id[0] === 'string' && on.length ? [{ id: id[0], on: Object.keys(on[0] ?? {}) }] : []; + return typeof id[0] === 'string' && typeof on[0] === 'object' ? [{ id: id[0], on: Object.keys(on[0] ?? {}) }] : []; }).filter(holder => holder.on.some(name => names.includes(name))).map(holder => holder.id); if (holders.length) { console.warn(`${when} removes the containers that hold them: ${holders.join(' ')}`); diff --git a/test/runner-workspace.test.ts b/test/runner-workspace.test.ts index ccd949bc..ad55ca11 100644 --- a/test/runner-workspace.test.ts +++ b/test/runner-workspace.test.ts @@ -15,7 +15,7 @@ import { attemptRef, openRunnerRepository } from '../runner/runner-repository.ts import { ReviewService } from '../runner/review.ts'; import { Store } from '../runner/store.ts'; import { createTaskWorkspace } from '../runner/workspace.ts'; -import { removeRunnerObjects } from './fixtures/docker-sweep.ts'; +import { removeRunnerObjects, requireRealDocker } from './fixtures/docker-sweep.ts'; import { fixtureGit as git } from './fixtures/git.ts'; // F2b against the real lane D (#87): each item runs in fresh task storage cloned from the runner-owned repository, an @@ -165,7 +165,7 @@ describe('task storage path identity (#162)', () => { // run from another checkout) or a container a dead runner left behind does not count. async function watchedProbe() { const shim = mkdtempSync(join(tmpdir(), 'docker-shim-')); roots.push(shim); - const names = join(shim, 'names'), realDocker = execFileSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).trim(); + const names = join(shim, 'names'), realDocker = requireRealDocker(); // Not a fake: it records the argument after each separate `--name` (the form the probe uses; if that changed, no name // is recorded and the test fails) and runs the real docker with the same arguments, so every exit status, stdout // and stderr the probe reads is Docker's own. From 1c1f004f697c3f71e5e9db1478b633e18b6c366e Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 21:07:07 -0700 Subject: [PATCH 23/75] Budget the supervisor finish hooks; name a signal kill; exact comments (#189) Review round 22. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 11 +++++------ test/agent-supervisor.test.ts | 4 ++-- test/docker-sweep.test.ts | 2 +- test/fixtures/docker-sweep.ts | 2 +- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 0d09df4f..cf10b8ba 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -169,12 +169,11 @@ afterAll(async () => { }); if (failures.length) throw new AggregateError(failures, 'Test cleanup failed.'); } finally { - // The runner owner is this run's own, so nothing else would find what this run leaves behind. Objects a test - // makes under another owner or without the runner label are not found here, unless they hold one of this run's - // volumes or networks: those this file labels with its own run ID are found by it, the try above removes those - // tracked by name (`containers`, `volumes`), and the other tests remove their own. It may not run before a hook - // timeout. Its budget covers two sweeps of up to 5.5 minutes each and the cleanup - // before them. + // The runner owner is this run's own, so nothing else would find what this run leaves behind. Objects a test makes + // under another owner or without the runner label are not found here, unless they hold one of this run's volumes or + // networks: those this file labels with its own run ID are found by the file-label sweep, the try above removes + // those tracked by name (`containers`, `volumes`), and the other tests remove their own. It may not run before a + // hook timeout. Its budget covers two sweeps of up to 5.5 minutes each and the cleanup before them. removeLabelledObjects(TEST_FILE_LABEL, TEST_FILE_RUN, 'afterAll'); removeRunnerObjects(TEST_RUNNER_OWNER); } diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 5071026a..7c13dd2a 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -388,7 +388,7 @@ describe('container invocation supervisor', () => { it('releases its own resources, and leaves a same-named container another invocation owns', async () => { const current = await profile(fixture(), 'finite-output', 'foreign-name'); // Removed by name when the test finishes, timeouts included: its label value is shared across runs (#189). - onTestFinished(() => { spawnSync(REAL_DOCKER, ['rm', '--force', current.name], { stdio: 'ignore', timeout: 30_000, killSignal: 'SIGKILL' }); }); + onTestFinished(() => { spawnSync(REAL_DOCKER, ['rm', '--force', current.name], { stdio: 'ignore', timeout: 30_000, killSignal: 'SIGKILL' }); }, 60_000); try { // Inside the try, so its removal runs even if the create's client fails after the daemon made it (#189). execFileSync('docker', ['create', '--name', current.name, '--label', 'io.codeboost.invocation=foreign', @@ -443,7 +443,7 @@ describe('container invocation supervisor', () => { const current = await profile(fixture(), 'noop'); // A foreign container occupies the deterministic name, so container-level cleanup could never settle. // Removed by name when the test finishes, timeouts included: its label value is shared across runs (#189). - onTestFinished(() => { spawnSync(REAL_DOCKER, ['rm', '--force', current.name], { stdio: 'ignore', timeout: 30_000, killSignal: 'SIGKILL' }); }); + onTestFinished(() => { spawnSync(REAL_DOCKER, ['rm', '--force', current.name], { stdio: 'ignore', timeout: 30_000, killSignal: 'SIGKILL' }); }, 60_000); try { execFileSync('docker', ['create', '--name', current.name, '--label', 'io.codeboost.invocation=someone-else', '--entrypoint', 'true', imageId], { stdio: 'ignore' }); diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index 6d7dcc26..e4fc7c05 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -30,7 +30,7 @@ it('skips output that is not a network, reports what is not JSON, and never thro it('reads nothing from a docker call that was killed, whose output may be cut short', () => { const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); const calls = join(root, 'calls'); - // A well-formed network line, then the client is killed before it exits by itself (as at the 30 s limit). + // A well-formed network line, then the client is killed by a signal before it exits by itself. writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${calls}'`, `if [ "$1 $2" = "network ls" ]; then printf '%s\\n' '{"ID":"${'a'.repeat(64)}","Name":"swept"}'; kill -9 $$; fi`, 'exit 0'].join('\n'), { mode: 0o755 }); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index bdd4e0e1..48b9d7f4 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -44,7 +44,7 @@ export function removeLabelledObjects(label: string, value: string, when: string const run = (args: string[]) => { const result = spawnSync(docker, args, { encoding: 'utf8', timeout: 30_000, killSignal: 'SIGKILL' }); if (result.status !== 0) console.warn(`${when}: docker ${args.slice(0, 2).join(' ')} failed:`, - result.error?.message ?? result.stderr.trim()); + result.error?.message ?? (result.signal ? `killed by ${result.signal}` : result.stderr.trim())); // An inspect of several objects exits 1 if one is gone, yet prints the rest (Docker 27.4), so the output of a call // that exited by itself is read whatever its status. A killed call's output may be cut short, so it is not. return result.error || result.status === null ? [] : result.stdout.split('\n').filter(Boolean); From 078b189171d4b63c4b4f6dadffe7b03a6c9ac906 Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 21:16:33 -0700 Subject: [PATCH 24/75] Register late-landing agent-network objects by name; exact comments (#189) Review round 23: the three agent-network tests whose foreign object may land after their own removal register it by name for afterAll, since the record goes with the shim; the late-network test says why its network is swept; comment wording. Co-Authored-By: Claude Opus 5.5 --- test/agent-network.test.ts | 8 ++++++++ test/agent-supervisor.test.ts | 2 +- test/docker-sweep.test.ts | 4 ++-- test/fixtures/docker-sweep.ts | 4 ++-- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index d70feb50..27e3fa8a 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -91,6 +91,8 @@ describe('vendor-only egress', () => { name = readFileSync(requested, 'utf8'); } finally { process.env.PATH = path; rmSync(shim, { recursive: true, force: true }); } execFileSync('sleep', ['3']); + // A network landing later still carries this run's runner label (the shim passes production's arguments on), so + // afterAll's runner sweep removes it. const orphaned = spawnSync('docker', ['network', 'inspect', name], { stdio: 'ignore' }).status === 0; if (orphaned) docker('network', 'rm', name); expect(name).toMatch(/^codeboost-egress-/); @@ -118,6 +120,8 @@ describe('vendor-only egress', () => { process.env.PATH = path; // Recorded before it is made, and removed here: the network has no labels, so no sweep finds it (#189). foreign = existsSync(recorded) ? readFileSync(recorded, 'utf8') : ''; + // By name too: the record goes with the shim below, and the object may land after this removal (#189). + if (foreign) unlabelled.push({ kind: 'network', name: foreign }); if (foreign) { survived = spawnSync('docker', ['network', 'inspect', foreign], { stdio: 'ignore' }).status === 0; spawnSync('docker', ['network', 'rm', foreign], { stdio: 'ignore' }); @@ -174,6 +178,8 @@ describe('vendor-only egress', () => { process.env.PATH = path; // Recorded before it is made, and removed here: the container has no labels, so no sweep finds it (#189). foreign = existsSync(recorded) ? readFileSync(recorded, 'utf8') : ''; + // By name too: the record goes with the shim below, and the object may land after this removal (#189). + if (foreign) unlabelled.push({ kind: 'container', name: foreign }); if (foreign) { survived = spawnSync('docker', ['container', 'inspect', foreign], { stdio: 'ignore' }).status === 0; spawnSync('docker', ['rm', '--force', foreign], { stdio: 'ignore' }); @@ -228,6 +234,8 @@ describe('vendor-only egress', () => { process.env.PATH = path; // The stand-in has no runner label, so no sweep finds it: remove it here, however the test ends (#189). impostor = existsSync(recorded) ? readFileSync(recorded, 'utf8') : ''; + // By name too: the record goes with the shim below, and the object may land after this removal (#189). + if (impostor) unlabelled.push({ kind: 'container', name: impostor }); if (impostor) { survived = spawnSync('docker', ['container', 'inspect', impostor], { stdio: 'ignore' }).status === 0; spawnSync('docker', ['rm', '--force', impostor], { stdio: 'ignore' }); diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 7c13dd2a..f10bcd80 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -403,7 +403,7 @@ describe('container invocation supervisor', () => { expect(execFileSync('docker', ['container', 'inspect', '--format', '{{index .Config.Labels "io.codeboost.invocation"}}', current.name], { encoding: 'utf8' }).trim()).toBe('foreign'); } finally { - // Never throws: the create may have failed before Docker made anything. + // Never throws, whatever exit status this Docker version gives for a missing name. spawnSync('docker', ['rm', '--force', current.name], { stdio: 'ignore' }); } }, 3 * 60_000); diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index e4fc7c05..66a18465 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -6,8 +6,8 @@ import { removeLabelledObjects, TEST_RUN_LABEL } from './fixtures/docker-sweep.t // The test sweep's handling of output it cannot read (#189). The fake `docker` answers as no real daemon does for // `network ls`: a line that is not JSON, a JSON null and one network without a name. To every other call it answers -// nothing and exit 0, as Docker 27.4 does for `ls` with no match; the sweep reads only stdout and the exit status, and -// stderr only to report a failure. What Docker 27.4 really prints is covered against the daemon in agent-container. +// nothing and exit 0, as Docker 27.4 does for `ls` with no match; the sweep reads only stdout, the exit status and +// whether a signal killed the call, and stderr only to report a failure. What Docker 27.4 really prints is covered against the daemon in agent-container. const roots: string[] = []; afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index 48b9d7f4..9a641a6e 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -24,8 +24,8 @@ export const TEST_FILE_LABEL = 'io.codeboost.test-file'; /** * Removes the containers, networks and volumes labelled with one runner owner, and every container that holds one of * those networks or volumes, whatever its labels (see `removeLabelledObjects`). A test file's afterAll calls it after - * its other cleanup, and a test that makes objects under its own runner owner sweeps them (by owner or by allocation) - * when it finishes. + * its other cleanup, and a test that makes objects under its own runner owner sweeps them, by owner here or by + * allocation with `removeLabelledObjects`. */ export function removeRunnerObjects(runnerOwner: string, when = 'afterAll'): void { removeLabelledObjects(RUNNER_LABEL, runnerOwner, when); From 052848b4939d6dbb86ffd37e7e1fc0885eb902f9 Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 21:27:50 -0700 Subject: [PATCH 25/75] Track the holder test's objects for afterAll, removed by exact name (#189) Review round 24: the holder test's unlabelled objects and the swapped keeper are tracked for afterAll; afterAll removes tracked names only when an object of exactly that name exists, by its full ID, since some are named like ID prefixes; withoutRunnerLabel uses sweepWhenFinished; the finish-time sweep states what lands too late for it. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 33 +++++++++++++++++++-------------- test/agent-network.test.ts | 5 ++--- test/fixtures/docker-sweep.ts | 14 ++++++++++++++ 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index cf10b8ba..b7abb6e0 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -1,5 +1,5 @@ import { execFileSync, spawnSync } from 'node:child_process'; -import { REAL_DOCKER, removeLabelledObjects, removeRunnerObjects, requireRealDocker, TEST_FILE_LABEL, TEST_RUN_LABEL } from './fixtures/docker-sweep.ts'; +import { REAL_DOCKER, removeByName, removeLabelledObjects, removeRunnerObjects, requireRealDocker, TEST_FILE_LABEL, TEST_RUN_LABEL } from './fixtures/docker-sweep.ts'; import { fixtureGit } from './fixtures/git.ts'; import { createHash, randomBytes, randomUUID } from 'node:crypto'; import { chmodSync, existsSync, mkdtempSync, mkdirSync, readdirSync, readFileSync, realpathSync, renameSync, rmSync, statSync, @@ -52,6 +52,8 @@ const containers = new Set(); * objects (#189): afterAll sweeps it. Other label-less objects are tracked by name (`containers`, `volumes`). */ const TEST_FILE_RUN = randomUUID(); +/** Networks a test makes without this run's runner label, removed by name in afterAll as `volumes` are. */ +const networks = new Set(); /** * Volumes a test makes without this run's runner label, removed in afterAll if the test did not remove them (it may * have timed out). @@ -60,8 +62,9 @@ const volumes = new Set(); /** * For an async test that makes objects under another runner's owner, or its own random one, which no afterAll sweep * finds: removes what carries `label=value` when the test finishes, passed, failed or timed out (a timed-out body runs - * on, and its finally may never run), with the sweep's own 6-minute budget (#189). Not covered: an object a timed-out - * body makes after this sweep has run. + * on, and its finally may never run), with the sweep's own budget: up to eleven synchronous Docker calls of 30 s each, + * which a timer cannot cut short, so 6 minutes rather than a hook's default 10 s (#189). Not covered: an object that + * lands after this sweep has run, from a timed-out body or a killed process's Docker client. */ const sweepWhenFinished = (label: string, value: string) => onTestFinished(() => removeLabelledObjects(label, value, 'onTestFinished'), 6 * 60_000); @@ -152,10 +155,10 @@ afterEach(async () => { }, 120_000); afterAll(async () => { try { - for (const container of containers) - spawnSync(REAL_DOCKER, ['rm', '--force', container], { stdio: 'ignore', timeout: 30_000, killSignal: 'SIGKILL' }); - for (const volume of volumes) - spawnSync(REAL_DOCKER, ['volume', 'rm', '--force', volume], { stdio: 'ignore', timeout: 30_000, killSignal: 'SIGKILL' }); + // By exact name: some are named like the start of another object's ID. + for (const container of containers) removeByName('container', container); + for (const network of networks) removeByName('network', network); + for (const volume of volumes) removeByName('volume', volume); // Each step, even when an earlier one fails: storage left here is found by the sweep below only by its label, and // every failure is reported. const failures: unknown[] = []; @@ -703,9 +706,7 @@ describe('real Docker agent isolation', () => { const withoutRunnerLabel = async (prefixes: string[][], run: () => Promise | T): Promise => { if (!prefixes.every(prefix => prefix.includes('create'))) throw new Error('Only create commands can be unlabelled.'); const testRun = randomUUID(); - // Up to eleven synchronous Docker calls of 30 s each, which a timer cannot cut short: the budget covers them, so a - // slow removal is not reported as a hook timeout (10 s by default). - onTestFinished(() => removeLabelledObjects(TEST_RUN_LABEL, testRun, 'onTestFinished'), 6 * 60_000); + sweepWhenFinished(TEST_RUN_LABEL, testRun); const shim = mkdtempSync(join(tmpdir(), 'codeboost-unlabelled-docker-')); roots.push(shim); mkdirSync(join(shim, 'input')); // afterAll resets this path's mode const realDocker = requireRealDocker(); @@ -776,6 +777,8 @@ describe('real Docker agent isolation', () => { const run = randomUUID(), label = `${TEST_RUN_LABEL}=${run}`, fileLabel = `${TEST_FILE_LABEL}=${TEST_FILE_RUN}`, made: { kind: 'container' | 'network' | 'volume'; ref: string }[] = []; const make = (kind: 'container' | 'network' | 'volume', name: string, ...args: string[]) => { made.push({ kind, ref: name }); + // For afterAll too: the daemon can still create an object after its client's 60 s limit and this test's finally. + (kind === 'container' ? containers : kind === 'network' ? networks : volumes).add(name); return docker(...(kind === 'container' ? [args[0]!, '--name', name, ...args.slice(1)] : [kind, args[0]!, ...args.slice(1), name])); }; const named = (what: string) => `codeboost-${what}-${randomUUID()}`; @@ -929,11 +932,10 @@ describe('real Docker agent isolation', () => { 'result = run(args);', "process.kill(process.pid, 'SIGKILL');", ].join('\n'); - const listed = () => spawnSync('docker', ['network', 'ls', '--quiet', '--no-trunc', '--filter', - `label=io.codeboost.allocation=${allocationId}`], { encoding: 'utf8' }).stdout?.trim() ?? ''; const created = await withDockerShim(['network', 'create'], foreign, () => createVendorNetwork(invocation(data.clone, 'planning'), imageId, allocationId).then(() => undefined, error => error)); - const left = listed(); + const left = spawnSync('docker', ['network', 'ls', '--quiet', '--filter', `label=io.codeboost.allocation=${allocationId}`], + { encoding: 'utf8' }).stdout?.trim() ?? ''; expect(created).toBeInstanceOf(VendorNetworkCreationCleanupError); const cleanup = (created as VendorNetworkCreationCleanupError).errors[1] as AggregateError; expect(cleanup.errors.map(error => (error as Error).message)).toEqual(['Refused to remove unowned vendor network.']); @@ -960,6 +962,8 @@ describe('real Docker agent isolation', () => { it('removes a task keeper by the ID it inspected, never a same-named replacement created after the inspect', async () => { const data = fixture(), filesystems = data.filesystems, swapped = join(data.root, 'swapped'); taskFilesystems.splice(taskFilesystems.indexOf(filesystems), 1); + // Tracked before the shim runs: the stand-in has no labels, so afterAll's removal by name is the only other one (#189). + containers.add(filesystems.keeper); // Right after cleanup inspects the keeper, its name is taken over by another container. const swap = [ 'result = run(args);', @@ -2567,7 +2571,8 @@ describe('real Docker agent isolation', () => { }, 60_000); it('startup probe refuses to exec when seccomp filtering is disabled', async () => { - // Named and tracked before it is made: it has no labels, so afterAll's removal by name is the only one (#189). + // Named and tracked before it is made: it has no labels, so if `--rm` never applies (its client killed before the + // start), afterAll's removal by name is the only one (#189). const name = `codeboost-seccomp-probe-${randomUUID()}`; containers.add(name); const result = spawnSync('docker', ['run', '--rm', '--name', name, '--read-only', '--user', '10001:10001', '--cap-drop=ALL', diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index 27e3fa8a..5d16e825 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -9,7 +9,7 @@ import { assertVendorNetwork, createVendorNetwork, removeVendorNetwork, VENDOR_H VendorNetworkCreationCleanupError, type VendorNetwork } from '../agents/network/network.ts'; import { captureInvocation, type InvocationInput } from '../agents/contract.ts'; -import { REAL_DOCKER, removeRunnerObjects, requireRealDocker } from './fixtures/docker-sweep.ts'; +import { removeByName, removeRunnerObjects, requireRealDocker } from './fixtures/docker-sweep.ts'; // Each run has its own runner owner, so the sweep after this file removes only what this run left behind. Network and // proxy names are random, so the fixed attempt IDs below reach only labels, where this owner tells runs apart. const TEST_RUNNER_OWNER = randomBytes(16).toString('hex'); @@ -43,8 +43,7 @@ afterAll(async () => { try { for (const { kind, name, file } of unlabelled) { const ref = name ?? (file && existsSync(file) ? readFileSync(file, 'utf8') : ''); - if (ref) spawnSync(REAL_DOCKER, kind === 'container' ? ['rm', '--force', ref] : ['network', 'rm', ref], - { stdio: 'ignore', timeout: 30_000, killSignal: 'SIGKILL' }); + if (ref) removeByName(kind, ref); } if (network) await removeVendorNetwork(network); } diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index 9a641a6e..d54a4e98 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -21,6 +21,20 @@ export const TEST_RUN_LABEL = 'io.codeboost.test-run'; /** Beside it, one value per test file run, so afterAll removes them all with one sweep. */ export const TEST_FILE_LABEL = 'io.codeboost.test-file'; +/** + * Removes the container, network or volume named exactly `name`, by its full ID (a volume by its name, which is its + * ID). Docker would read a missing name as an ID prefix, so an object found under another name is left alone. Never + * throws; each call has a 30 s limit. + */ +export function removeByName(kind: 'container' | 'network' | 'volume', name: string, docker = REAL_DOCKER): void { + const options = { encoding: 'utf8' as const, timeout: 30_000, killSignal: 'SIGKILL' as const }; + const found = spawnSync(docker, [kind, 'inspect', '--format', kind === 'volume' ? '{{.Name}}' : '{{.Id}} {{.Name}}', name], options); + if (found.status !== 0 || found.error) return; + const [id, actual] = kind === 'volume' ? [name, found.stdout.trim()] : found.stdout.trim().split(' '); + if (actual?.replace(/^\//, '') !== name || !id) return; + spawnSync(docker, kind === 'container' ? ['rm', '--force', id] : [kind, 'rm', ...(kind === 'volume' ? ['--force'] : []), id], options); +} + /** * Removes the containers, networks and volumes labelled with one runner owner, and every container that holds one of * those networks or volumes, whatever its labels (see `removeLabelledObjects`). A test file's afterAll calls it after From f8e9749938b1d30ca5869e1b1a62448a0e69c781 Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 21:37:54 -0700 Subject: [PATCH 26/75] Report removeByName failures; records keep every name; shim scoped (#189) Review round 25: removeByName reads JSON (names with spaces), reports a failed call other than no such object; agent-network's shims append one name per line and every name is removed; the foreign-label shim rewrites only its own allocation's create; the holder test checks the prefix-named network goes; the killed-call unit test first checks its fake and asserts the warning; comments name the networks set. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 14 ++++++----- test/agent-network.test.ts | 44 +++++++++++++++++------------------ test/docker-sweep.test.ts | 13 ++++++++--- test/fixtures/docker-sweep.ts | 25 +++++++++++++------- 4 files changed, 56 insertions(+), 40 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index b7abb6e0..08f68a66 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -49,7 +49,7 @@ const taskFilesystems: ReturnType[] = []; const containers = new Set(); /** * Put, beside its test-run ID, on every object the `withoutRunnerLabel` wrapper makes and on the holder test's swept - * objects (#189): afterAll sweeps it. Other label-less objects are tracked by name (`containers`, `volumes`). + * objects (#189): afterAll sweeps it. Other label-less objects are tracked by name (`containers`, `networks`, `volumes`). */ const TEST_FILE_RUN = randomUUID(); /** Networks a test makes without this run's runner label, removed by name in afterAll as `volumes` are. */ @@ -175,7 +175,7 @@ afterAll(async () => { // The runner owner is this run's own, so nothing else would find what this run leaves behind. Objects a test makes // under another owner or without the runner label are not found here, unless they hold one of this run's volumes or // networks: those this file labels with its own run ID are found by the file-label sweep, the try above removes - // those tracked by name (`containers`, `volumes`), and the other tests remove their own. It may not run before a + // those tracked by name (`containers`, `networks`, `volumes`), and the other tests remove their own. It may not run before a // hook timeout. Its budget covers two sweeps of up to 5.5 minutes each and the cleanup before them. removeLabelledObjects(TEST_FILE_LABEL, TEST_FILE_RUN, 'afterAll'); removeRunnerObjects(TEST_RUNNER_OWNER); @@ -789,7 +789,7 @@ describe('real Docker agent isolation', () => { make('container', named('bystander'), 'create', '--mount', `type=volume,source=${bystanderVolume},target=/x`, '--entrypoint', 'true', imageId)]; const network = make('network', named('swept'), 'create', '--label', label, '--label', fileLabel); // Named like the start of the bystander network's ID: Docker's network filter also matches that prefix. - make('network', bystanderNetwork.slice(0, 12), 'create', '--label', label, '--label', fileLabel); + const prefixNamed = make('network', bystanderNetwork.slice(0, 12), 'create', '--label', label, '--label', fileLabel); const volume = make('volume', named('swept'), 'create', '--label', label, '--label', fileLabel); // And a bystander named like the start of the swept network's ID: removing by a short ID would remove it. const lookalike = make('network', network.slice(0, 12), 'create'); @@ -822,7 +822,7 @@ describe('real Docker agent isolation', () => { removeLabelledObjects(TEST_RUN_LABEL, run, 'test', join(shim, 'docker')); const exists = (kind: string, ref: string) => spawnSync('docker', [kind, 'inspect', ref], { stdio: 'ignore' }).status === 0; for (const holder of holders) expect(exists('container', holder)).toBe(false); - expect([exists('network', network), exists('volume', volume)]).toEqual([false, false]); + expect([exists('network', network), exists('network', prefixNamed), exists('volume', volume)]).toEqual([false, false, false]); expect(exists('container', labelled)).toBe(false); for (const bystander of [...bystanders, holderLookalike, labelledLookalike]) expect(exists('container', bystander)).toBe(true); expect([exists('network', bystanderNetwork), exists('volume', bystanderVolume), exists('network', lookalike)]) @@ -925,12 +925,14 @@ describe('real Docker agent isolation', () => { const data = fixture(), allocationId = randomUUID(); // The network carries another runner's label, so no sweep of this run finds it: removed by its allocation (#189). sweepWhenFinished('io.codeboost.allocation', allocationId); - // The create lands with another runner's label, and its client is killed before it reports the ID. + // The create lands with another runner's label, and its client is killed before it reports the ID. Only this + // allocation's: should the test time out with the shim still on PATH, other tests' creates pass through. const foreign = [ + `if (!args.includes('io.codeboost.allocation=${allocationId}')) result = run(args); else {`, `const at = args.findIndex(arg => arg.startsWith('io.codeboost.runner='));`, `args[at] = 'io.codeboost.runner=${'f'.repeat(32)}';`, 'result = run(args);', - "process.kill(process.pid, 'SIGKILL');", + "process.kill(process.pid, 'SIGKILL'); }", ].join('\n'); const created = await withDockerShim(['network', 'create'], foreign, () => createVendorNetwork(invocation(data.clone, 'planning'), imageId, allocationId).then(() => undefined, error => error)); diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index 5d16e825..a6121a46 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -42,8 +42,9 @@ const unlabelled: { kind: 'container' | 'network'; name?: string; file?: string afterAll(async () => { try { for (const { kind, name, file } of unlabelled) { - const ref = name ?? (file && existsSync(file) ? readFileSync(file, 'utf8') : ''); - if (ref) removeByName(kind, ref); + // A record holds one name per line: should a test time out with its shim still on PATH, later creates add theirs. + for (const ref of name ? [name] : file && existsSync(file) ? readFileSync(file, 'utf8').split('\n').filter(Boolean) : []) + removeByName(kind, ref); } if (network) await removeVendorNetwork(network); } @@ -105,7 +106,7 @@ describe('vendor-only egress', () => { // The name is taken by someone else's network just before our create, so the daemon refuses it. writeFileSync(join(shim, 'docker'), ['#!/bin/sh', `if [ "$1" = network ] && [ "$2" = create ]; then for arg; do last="$arg"; done; ` - + `printf %s "$last" > '${recorded}'; '${realDocker}' network create "$last" >/dev/null; ` + + `printf '%s\\n' "$last" >> '${recorded}'; '${realDocker}' network create "$last" >/dev/null; ` + `echo "Error response from daemon: network with name $last already exists" >&2; exit 1; fi`, `exec '${realDocker}' "$@"`].join('\n'), { mode: 0o755 }); const refused = captureInvocation({ ...invocation, attemptId: `refused-network-${randomUUID()}`, @@ -118,13 +119,12 @@ describe('vendor-only egress', () => { } finally { process.env.PATH = path; // Recorded before it is made, and removed here: the network has no labels, so no sweep finds it (#189). - foreign = existsSync(recorded) ? readFileSync(recorded, 'utf8') : ''; + const made = existsSync(recorded) ? readFileSync(recorded, 'utf8').split('\n').filter(Boolean) : []; + foreign = made[0] ?? ''; // By name too: the record goes with the shim below, and the object may land after this removal (#189). - if (foreign) unlabelled.push({ kind: 'network', name: foreign }); - if (foreign) { - survived = spawnSync('docker', ['network', 'inspect', foreign], { stdio: 'ignore' }).status === 0; - spawnSync('docker', ['network', 'rm', foreign], { stdio: 'ignore' }); - } + for (const name of made) unlabelled.push({ kind: 'network', name }); + if (foreign) survived = spawnSync('docker', ['network', 'inspect', foreign], { stdio: 'ignore' }).status === 0; + for (const name of made) removeByName('network', name); rmSync(shim, { recursive: true, force: true }); } // The refused create is reported as itself, with no cleanup left to retry, and the foreign network survives. @@ -164,7 +164,7 @@ describe('vendor-only egress', () => { // Someone else's container takes the proxy name just before our create, so the daemon refuses it. writeFileSync(join(shim, 'docker'), ['#!/bin/sh', `if [ "$1" = create ] && [ "$2" = --name ] && case "$3" in codeboost-proxy-*) true;; *) false;; esac; then ` - + `printf %s "$3" > '${recorded}'; '${realDocker}' create --name "$3" --entrypoint true ${imageId} >/dev/null; fi`, + + `printf '%s\\n' "$3" >> '${recorded}'; '${realDocker}' create --name "$3" --entrypoint true ${imageId} >/dev/null; fi`, `exec '${realDocker}' "$@"`].join('\n'), { mode: 0o755 }); const refused = captureInvocation({ ...invocation, attemptId: `refused-proxy-${randomUUID()}`, deadline: Date.now() + 60_000 }); @@ -176,13 +176,12 @@ describe('vendor-only egress', () => { } finally { process.env.PATH = path; // Recorded before it is made, and removed here: the container has no labels, so no sweep finds it (#189). - foreign = existsSync(recorded) ? readFileSync(recorded, 'utf8') : ''; + const made = existsSync(recorded) ? readFileSync(recorded, 'utf8').split('\n').filter(Boolean) : []; + foreign = made[0] ?? ''; // By name too: the record goes with the shim below, and the object may land after this removal (#189). - if (foreign) unlabelled.push({ kind: 'container', name: foreign }); - if (foreign) { - survived = spawnSync('docker', ['container', 'inspect', foreign], { stdio: 'ignore' }).status === 0; - spawnSync('docker', ['rm', '--force', foreign], { stdio: 'ignore' }); - } + for (const name of made) unlabelled.push({ kind: 'container', name }); + if (foreign) survived = spawnSync('docker', ['container', 'inspect', foreign], { stdio: 'ignore' }).status === 0; + for (const name of made) removeByName('container', name); rmSync(shim, { recursive: true, force: true }); } // The refused create is reported as itself: our network was removed, nothing is left to retry or report. @@ -219,7 +218,7 @@ describe('vendor-only egress', () => { ` label=$('${realDocker}' inspect -f '{{index .Config.Labels "io.codeboost.egress"}}' "$2")`, ` '${realDocker}' rm --force "$2" >/dev/null`, // Recorded before the stand-in exists, so the test can remove it even if the run is cut short. - ` printf %s "$name" > '${recorded}'`, + ` printf '%s\\n' "$name" >> '${recorded}'`, ` '${realDocker}' run --detach --name "$name" --label "io.codeboost.egress=$label" --entrypoint sleep ${imageId} 300 >/dev/null`, ' exit 1', 'fi', `exec '${realDocker}' "$@"`].join('\n'), { mode: 0o755 }); const failing = captureInvocation({ ...invocation, attemptId: `failed-setup-${randomUUID()}`, @@ -232,13 +231,12 @@ describe('vendor-only egress', () => { } finally { process.env.PATH = path; // The stand-in has no runner label, so no sweep finds it: remove it here, however the test ends (#189). - impostor = existsSync(recorded) ? readFileSync(recorded, 'utf8') : ''; + const made = existsSync(recorded) ? readFileSync(recorded, 'utf8').split('\n').filter(Boolean) : []; + impostor = made[0] ?? ''; // By name too: the record goes with the shim below, and the object may land after this removal (#189). - if (impostor) unlabelled.push({ kind: 'container', name: impostor }); - if (impostor) { - survived = spawnSync('docker', ['container', 'inspect', impostor], { stdio: 'ignore' }).status === 0; - spawnSync('docker', ['rm', '--force', impostor], { stdio: 'ignore' }); - } + for (const name of made) unlabelled.push({ kind: 'container', name }); + if (impostor) survived = spawnSync('docker', ['container', 'inspect', impostor], { stdio: 'ignore' }).status === 0; + for (const name of made) removeByName('container', name); rmSync(shim, { recursive: true, force: true }); } expect(impostor).toMatch(/^codeboost-proxy-/); diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index 66a18465..e3b0ff96 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -1,13 +1,15 @@ import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; import { afterEach, expect, it, vi } from 'vitest'; import { removeLabelledObjects, TEST_RUN_LABEL } from './fixtures/docker-sweep.ts'; // The test sweep's handling of output it cannot read (#189). The fake `docker` answers as no real daemon does for // `network ls`: a line that is not JSON, a JSON null and one network without a name. To every other call it answers -// nothing and exit 0, as Docker 27.4 does for `ls` with no match; the sweep reads only stdout, the exit status and -// whether a signal killed the call, and stderr only to report a failure. What Docker 27.4 really prints is covered against the daemon in agent-container. +// nothing and exit 0, as Docker 27.4 does for `ls` with no match; the sweep reads only stdout, the exit status, +// whether a signal killed the call and whether it could start, and stderr only to report a failure. What Docker 27.4 +// really prints is covered against the daemon in agent-container. const roots: string[] = []; afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); @@ -34,7 +36,12 @@ it('reads nothing from a docker call that was killed, whose output may be cut sh writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${calls}'`, `if [ "$1 $2" = "network ls" ]; then printf '%s\\n' '{"ID":"${'a'.repeat(64)}","Name":"swept"}'; kill -9 $$; fi`, 'exit 0'].join('\n'), { mode: 0o755 }); - vi.spyOn(console, 'warn').mockImplementation(() => undefined); + // The fake does what this test needs: a network line on stdout, then death by a signal. + const direct = spawnSync(join(root, 'docker'), ['network', 'ls'], { encoding: 'utf8' }); + expect([direct.status, direct.signal, direct.stdout]).toEqual([null, 'SIGKILL', `{"ID":"${'a'.repeat(64)}","Name":"swept"}\n`]); + rmSync(calls); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test', join(root, 'docker')); expect(readFileSync(calls, 'utf8').split('\n').filter(line => /^(ps|network rm) /.test(line))).toEqual([]); + expect(warn.mock.calls).toContainEqual(['test: docker network ls failed:', 'killed by SIGKILL']); }); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index d54a4e98..8cd613a6 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -1,4 +1,4 @@ -import { spawnSync } from 'node:child_process'; +import { spawnSync, type SpawnSyncReturns } from 'node:child_process'; import { RUNNER_LABEL } from '../../agents/labels.ts'; /** @@ -23,16 +23,25 @@ export const TEST_FILE_LABEL = 'io.codeboost.test-file'; /** * Removes the container, network or volume named exactly `name`, by its full ID (a volume by its name, which is its - * ID). Docker would read a missing name as an ID prefix, so an object found under another name is left alone. Never - * throws; each call has a 30 s limit. + * ID). Docker would read a missing name as an ID prefix, so an object found under another name is left alone. It + * reports a failed call, other than an inspect that finds no such object, and never throws; each call has a 30 s limit. */ export function removeByName(kind: 'container' | 'network' | 'volume', name: string, docker = REAL_DOCKER): void { const options = { encoding: 'utf8' as const, timeout: 30_000, killSignal: 'SIGKILL' as const }; - const found = spawnSync(docker, [kind, 'inspect', '--format', kind === 'volume' ? '{{.Name}}' : '{{.Id}} {{.Name}}', name], options); - if (found.status !== 0 || found.error) return; - const [id, actual] = kind === 'volume' ? [name, found.stdout.trim()] : found.stdout.trim().split(' '); - if (actual?.replace(/^\//, '') !== name || !id) return; - spawnSync(docker, kind === 'container' ? ['rm', '--force', id] : [kind, 'rm', ...(kind === 'volume' ? ['--force'] : []), id], options); + const failed = (result: SpawnSyncReturns, what: string) => console.warn(`removing ${kind} ${name}: docker ${what} failed:`, + result.error?.message ?? (result.signal ? `killed by ${result.signal}` : result.stderr.trim())); + // As JSON: a network's name may hold spaces. + const found = spawnSync(docker, [kind, 'inspect', '--format', kind === 'volume' ? '{{json .Name}}' : '{{json .Id}} {{json .Name}}', name], options); + // Docker 27.4's answers for a missing object: "No such container", "network … not found", "no such volume". + if (found.error || found.status !== 0) { if (found.status === null || !/no such|not found/i.test(found.stderr)) failed(found, 'inspect'); return; } + let id: unknown, actual: unknown; + try { + const at = found.stdout.indexOf(' '); + [id, actual] = kind === 'volume' ? [name, JSON.parse(found.stdout)] : [JSON.parse(found.stdout.slice(0, at)), JSON.parse(found.stdout.slice(at + 1))]; + } catch { console.warn(`removing ${kind} ${name}: docker printed what is not JSON:`, found.stdout); return; } + if (typeof id !== 'string' || typeof actual !== 'string' || actual.replace(/^\//, '') !== name) return; + const removed = spawnSync(docker, kind === 'container' ? ['rm', '--force', id] : [kind, 'rm', ...(kind === 'volume' ? ['--force'] : []), id], options); + if (removed.error || removed.status !== 0) failed(removed, `${kind} rm`); } /** From f0c8278dd62998de28f779e6e537f357ef873348 Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 21:47:12 -0700 Subject: [PATCH 27/75] Test removeByName's exact-name guard and reports; records outside shims (#189) Review round 26: the holder test checks removeByName leaves an object whose ID starts with a now-missing tracked name; a unit test covers its no-such answer, an ID-prefix match, a refused removal and a killed inspect; only a container's name loses its leading slash; agent-network keeps its records outside the shims so afterAll reads names a late client adds; sweepWhenFinished's doc covers withoutRunnerLabel. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 9 ++++++++- test/agent-network.test.ts | 20 ++++++++++---------- test/docker-sweep.test.ts | 26 +++++++++++++++++++++++++- test/fixtures/docker-sweep.ts | 6 +++--- 4 files changed, 46 insertions(+), 15 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 08f68a66..cd76f4e2 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -61,7 +61,8 @@ const networks = new Set(); const volumes = new Set(); /** * For an async test that makes objects under another runner's owner, or its own random one, which no afterAll sweep - * finds: removes what carries `label=value` when the test finishes, passed, failed or timed out (a timed-out body runs + * finds, and for `withoutRunnerLabel`'s test-run ID, so they go before the next test rather than at afterAll: removes + * what carries `label=value` when the test finishes, passed, failed or timed out (a timed-out body runs * on, and its finally may never run), with the sweep's own budget: up to eleven synchronous Docker calls of 30 s each, * which a timer cannot cut short, so 6 minutes rather than a hook's default 10 s (#189). Not covered: an object that * lands after this sweep has run, from a timed-out body or a killed process's Docker client. @@ -820,6 +821,12 @@ describe('real Docker agent isolation', () => { `if [ "$1 $2" = "container inspect" ]; then exec '${realDocker}' "$@" codeboost-gone-${randomUUID()}; fi`, `exec '${realDocker}' "$@"`].join('\n'), { mode: 0o755 }); removeLabelledObjects(TEST_RUN_LABEL, run, 'test', join(shim, 'docker')); + // afterAll removes these tracked names with removeByName. One now gone is the start of a live object's ID, which + // Docker resolves it to; removeByName must leave that object alone. + expect(docker('network', 'inspect', '--format', '{{.Id}}', bystanderNetwork.slice(0, 12))).toBe(bystanderNetwork); + removeByName('network', bystanderNetwork.slice(0, 12)); + expect(docker('container', 'inspect', '--format', '{{.Id}}', bystanders[0]!.slice(0, 12))).toBe(bystanders[0]); + removeByName('container', bystanders[0]!.slice(0, 12)); const exists = (kind: string, ref: string) => spawnSync('docker', [kind, 'inspect', ref], { stdio: 'ignore' }).status === 0; for (const holder of holders) expect(exists('container', holder)).toBe(false); expect([exists('network', network), exists('network', prefixNamed), exists('volume', volume)]).toEqual([false, false, false]); diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index a6121a46..d3495179 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -35,9 +35,11 @@ beforeAll(async () => { }, 10 * 60_000); /** * Objects a test makes without this run's runner label, by name or by the file its shim records the name in, so - * afterAll removes them should the test time out before its own removal runs (#189). + * afterAll removes them should the test time out before its own removal runs (#189). The record files live in + * `RECORDS`, not in the shims, so a name a late client adds after the test's own removal is still read here. */ const unlabelled: { kind: 'container' | 'network'; name?: string; file?: string }[] = []; +const RECORDS = mkdtempSync(join(tmpdir(), 'codeboost-records-')); // If setup failed there is no network, and a teardown error would hide the setup failure. afterAll(async () => { try { @@ -55,6 +57,7 @@ afterAll(async () => { // timeout. The hook's 7 minutes leave a minute and a half over this sweep's worst case (eleven calls of 30 s), for // the cleanup before it. removeRunnerObjects(TEST_RUNNER_OWNER); + rmSync(RECORDS, { recursive: true, force: true }); } }, 7 * 60_000); @@ -100,7 +103,7 @@ describe('vendor-only egress', () => { }, 60_000); it('leaves a same-named network alone when the daemon refuses the create', async () => { - const shim = mkdtempSync(join(tmpdir(), 'docker-shim-')), recorded = join(shim, 'foreign'); + const shim = mkdtempSync(join(tmpdir(), 'docker-shim-')), recorded = join(RECORDS, randomUUID()); unlabelled.push({ kind: 'network', file: recorded }); const realDocker = requireRealDocker(); // The name is taken by someone else's network just before our create, so the daemon refuses it. @@ -121,8 +124,7 @@ describe('vendor-only egress', () => { // Recorded before it is made, and removed here: the network has no labels, so no sweep finds it (#189). const made = existsSync(recorded) ? readFileSync(recorded, 'utf8').split('\n').filter(Boolean) : []; foreign = made[0] ?? ''; - // By name too: the record goes with the shim below, and the object may land after this removal (#189). - for (const name of made) unlabelled.push({ kind: 'network', name }); + // afterAll reads the record again: the object may land after this removal (#189). if (foreign) survived = spawnSync('docker', ['network', 'inspect', foreign], { stdio: 'ignore' }).status === 0; for (const name of made) removeByName('network', name); rmSync(shim, { recursive: true, force: true }); @@ -158,7 +160,7 @@ describe('vendor-only egress', () => { }, 60_000); it('neither reports nor removes a same-named proxy when the daemon refuses the create', async () => { - const shim = mkdtempSync(join(tmpdir(), 'docker-shim-')), recorded = join(shim, 'foreign'); + const shim = mkdtempSync(join(tmpdir(), 'docker-shim-')), recorded = join(RECORDS, randomUUID()); unlabelled.push({ kind: 'container', file: recorded }); const realDocker = requireRealDocker(); // Someone else's container takes the proxy name just before our create, so the daemon refuses it. @@ -178,8 +180,7 @@ describe('vendor-only egress', () => { // Recorded before it is made, and removed here: the container has no labels, so no sweep finds it (#189). const made = existsSync(recorded) ? readFileSync(recorded, 'utf8').split('\n').filter(Boolean) : []; foreign = made[0] ?? ''; - // By name too: the record goes with the shim below, and the object may land after this removal (#189). - for (const name of made) unlabelled.push({ kind: 'container', name }); + // afterAll reads the record again: the object may land after this removal (#189). if (foreign) survived = spawnSync('docker', ['container', 'inspect', foreign], { stdio: 'ignore' }).status === 0; for (const name of made) removeByName('container', name); rmSync(shim, { recursive: true, force: true }); @@ -209,7 +210,7 @@ describe('vendor-only egress', () => { }, 60_000); it('does not delete a same-named stand-in when setup fails after the proxy exists', async () => { - const shim = mkdtempSync(join(tmpdir(), 'docker-shim-')), recorded = join(shim, 'impostor'); + const shim = mkdtempSync(join(tmpdir(), 'docker-shim-')), recorded = join(RECORDS, randomUUID()); unlabelled.push({ kind: 'container', file: recorded }); const realDocker = requireRealDocker(); // The readiness exec swaps the proxy for a same-named, same-labelled stand-in, then fails setup. @@ -233,8 +234,7 @@ describe('vendor-only egress', () => { // The stand-in has no runner label, so no sweep finds it: remove it here, however the test ends (#189). const made = existsSync(recorded) ? readFileSync(recorded, 'utf8').split('\n').filter(Boolean) : []; impostor = made[0] ?? ''; - // By name too: the record goes with the shim below, and the object may land after this removal (#189). - for (const name of made) unlabelled.push({ kind: 'container', name }); + // afterAll reads the record again: the object may land after this removal (#189). if (impostor) survived = spawnSync('docker', ['container', 'inspect', impostor], { stdio: 'ignore' }).status === 0; for (const name of made) removeByName('container', name); rmSync(shim, { recursive: true, force: true }); diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index e3b0ff96..0591e278 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { spawnSync } from 'node:child_process'; import { afterEach, expect, it, vi } from 'vitest'; -import { removeLabelledObjects, TEST_RUN_LABEL } from './fixtures/docker-sweep.ts'; +import { removeByName, removeLabelledObjects, TEST_RUN_LABEL } from './fixtures/docker-sweep.ts'; // The test sweep's handling of output it cannot read (#189). The fake `docker` answers as no real daemon does for // `network ls`: a line that is not JSON, a JSON null and one network without a name. To every other call it answers @@ -45,3 +45,27 @@ it('reads nothing from a docker call that was killed, whose output may be cut sh expect(readFileSync(calls, 'utf8').split('\n').filter(line => /^(ps|network rm) /.test(line))).toEqual([]); expect(warn.mock.calls).toContainEqual(['test: docker network ls failed:', 'killed by SIGKILL']); }); + +it('removes by name only an object of exactly that name, and reports failures other than no such object', () => { + const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); + const calls = join(root, 'calls'), id = 'b'.repeat(64); + // Answers as Docker 27.4 does, read for `--format '{{json .Id}} {{json .Name}}'`: a missing name, another object found + // by an ID prefix, one whose removal is refused, and an inspect killed by a signal. + writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${calls}'`, + 'for arg; do name="$arg"; done', + 'if [ "$2" = inspect ]; then case "$name" in', + " gone) echo 'Error response from daemon: No such container: gone' >&2; exit 1;;", + ` prefix) printf '%s\\n' '"${id}" "/someone-else"';;`, + ` busy) printf '%s\\n' '"${id}" "/busy"';;`, + ' killed) kill -9 $$;;', 'esac; exit 0; fi', + "if [ \"$1\" = rm ]; then echo 'Error response from daemon: conflict' >&2; exit 1; fi", 'exit 0'].join('\n'), + { mode: 0o755 }); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + for (const name of ['gone', 'prefix', 'busy', 'killed']) removeByName('container', name, join(root, 'docker')); + // Only the exact name is removed, and by its full ID. + expect(readFileSync(calls, 'utf8').split('\n').filter(line => line.startsWith('rm '))).toEqual([`rm --force ${id}`]); + expect(warn.mock.calls).toEqual([ + ['removing container busy: docker container rm failed:', 'Error response from daemon: conflict'], + ['removing container killed: docker inspect failed:', 'killed by SIGKILL'], + ]); +}); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index 8cd613a6..a8ef3692 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -30,7 +30,7 @@ export function removeByName(kind: 'container' | 'network' | 'volume', name: str const options = { encoding: 'utf8' as const, timeout: 30_000, killSignal: 'SIGKILL' as const }; const failed = (result: SpawnSyncReturns, what: string) => console.warn(`removing ${kind} ${name}: docker ${what} failed:`, result.error?.message ?? (result.signal ? `killed by ${result.signal}` : result.stderr.trim())); - // As JSON: a network's name may hold spaces. + // As JSON: a network's name may hold spaces. Only a container's name comes with a leading slash. const found = spawnSync(docker, [kind, 'inspect', '--format', kind === 'volume' ? '{{json .Name}}' : '{{json .Id}} {{json .Name}}', name], options); // Docker 27.4's answers for a missing object: "No such container", "network … not found", "no such volume". if (found.error || found.status !== 0) { if (found.status === null || !/no such|not found/i.test(found.stderr)) failed(found, 'inspect'); return; } @@ -39,7 +39,7 @@ export function removeByName(kind: 'container' | 'network' | 'volume', name: str const at = found.stdout.indexOf(' '); [id, actual] = kind === 'volume' ? [name, JSON.parse(found.stdout)] : [JSON.parse(found.stdout.slice(0, at)), JSON.parse(found.stdout.slice(at + 1))]; } catch { console.warn(`removing ${kind} ${name}: docker printed what is not JSON:`, found.stdout); return; } - if (typeof id !== 'string' || typeof actual !== 'string' || actual.replace(/^\//, '') !== name) return; + if (typeof id !== 'string' || typeof actual !== 'string' || (kind === 'container' ? actual.replace(/^\//, '') : actual) !== name) return; const removed = spawnSync(docker, kind === 'container' ? ['rm', '--force', id] : [kind, 'rm', ...(kind === 'volume' ? ['--force'] : []), id], options); if (removed.error || removed.status !== 0) failed(removed, `${kind} rm`); } @@ -79,7 +79,7 @@ export function removeLabelledObjects(label: string, value: string, when: string for (const kind of ['container', 'network', 'volume'] as const) { // Networks with their names: `ps --filter network=` finds a created, never started container by the name only // (Docker 27.4). Each is removed by its full ID, which no other network's name can be mistaken for. - // As JSON: a network's name may hold spaces. + // As JSON: a network's name may hold spaces. Only a container's name comes with a leading slash. const listed = run([kind, 'ls', ...(kind === 'container' ? ['--all'] : []), ...(kind === 'network' ? ['--no-trunc', '--format', '{{json .}}'] : kind === 'container' ? ['--quiet', '--no-trunc'] : ['--quiet']), '--filter', `label=${label}=${value}`]) From fcf4257091542aa1d42a4632e20d5712132d79a8 Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 21:56:47 -0700 Subject: [PATCH 28/75] removeByName treats only Docker's own missing answers as missing; tests every kind (#189) Review round 27: a failure before the daemon (an unknown context) is reported; the fake covers containers, networks and volumes, found, missing, under another name and refused; the unlabelled-create shim acts only on its own container; agent-supervisor's finish-time removals use removeByName; doc and import fixes. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 11 +++++--- test/agent-supervisor.test.ts | 6 ++--- test/docker-sweep.test.ts | 49 ++++++++++++++++++++++++----------- test/fixtures/docker-sweep.ts | 6 +++-- 4 files changed, 48 insertions(+), 24 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index cd76f4e2..68740076 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -1,5 +1,5 @@ import { execFileSync, spawnSync } from 'node:child_process'; -import { REAL_DOCKER, removeByName, removeLabelledObjects, removeRunnerObjects, requireRealDocker, TEST_FILE_LABEL, TEST_RUN_LABEL } from './fixtures/docker-sweep.ts'; +import { removeByName, removeLabelledObjects, removeRunnerObjects, requireRealDocker, TEST_FILE_LABEL, TEST_RUN_LABEL } from './fixtures/docker-sweep.ts'; import { fixtureGit } from './fixtures/git.ts'; import { createHash, randomBytes, randomUUID } from 'node:crypto'; import { chmodSync, existsSync, mkdtempSync, mkdirSync, readdirSync, readFileSync, realpathSync, renameSync, rmSync, statSync, @@ -49,7 +49,8 @@ const taskFilesystems: ReturnType[] = []; const containers = new Set(); /** * Put, beside its test-run ID, on every object the `withoutRunnerLabel` wrapper makes and on the holder test's swept - * objects (#189): afterAll sweeps it. Other label-less objects are tracked by name (`containers`, `networks`, `volumes`). + * objects (#189): afterAll sweeps it. Other label-less objects are tracked by name (`containers`, `networks`, `volumes`), or found as holders of this run's + * networks and volumes. */ const TEST_FILE_RUN = randomUUID(); /** Networks a test makes without this run's runner label, removed by name in afterAll as `volumes` are. */ @@ -953,12 +954,14 @@ describe('real Docker agent isolation', () => { it('never removes an agent container found by name after a killed create when a label is missing', async () => { const data = fixture(), live = await profile(data, 'planning', 'must-not-run'); - // The create lands without the runner label, and its client is killed before it reports the ID. + // The create lands without the runner label, and its client is killed before it reports the ID. Only this + // container's: should the test time out with the shim still on PATH, other tests' creates pass through. const unlabelled = [ + `if (args[2] !== ${JSON.stringify(live.name)}) result = run(args); else {`, "const at = args.findIndex(arg => arg.startsWith('io.codeboost.runner='));", 'args.splice(at - 1, 2);', 'result = run(args);', - "process.kill(process.pid, 'SIGKILL');", + "process.kill(process.pid, 'SIGKILL'); }", ].join('\n'); try { await withDockerShim(['create', '--name'], unlabelled, () => diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index f10bcd80..4da1289a 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -1,6 +1,6 @@ import { randomBytes, randomUUID } from 'node:crypto'; import { execFileSync, spawnSync } from 'node:child_process'; -import { REAL_DOCKER, removeRunnerObjects, requireRealDocker } from './fixtures/docker-sweep.ts'; +import { removeByName, removeRunnerObjects, requireRealDocker } from './fixtures/docker-sweep.ts'; import { fixtureGit } from './fixtures/git.ts'; import { chmodSync, mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; @@ -388,7 +388,7 @@ describe('container invocation supervisor', () => { it('releases its own resources, and leaves a same-named container another invocation owns', async () => { const current = await profile(fixture(), 'finite-output', 'foreign-name'); // Removed by name when the test finishes, timeouts included: its label value is shared across runs (#189). - onTestFinished(() => { spawnSync(REAL_DOCKER, ['rm', '--force', current.name], { stdio: 'ignore', timeout: 30_000, killSignal: 'SIGKILL' }); }, 60_000); + onTestFinished(() => removeByName('container', current.name), 60_000); try { // Inside the try, so its removal runs even if the create's client fails after the daemon made it (#189). execFileSync('docker', ['create', '--name', current.name, '--label', 'io.codeboost.invocation=foreign', @@ -443,7 +443,7 @@ describe('container invocation supervisor', () => { const current = await profile(fixture(), 'noop'); // A foreign container occupies the deterministic name, so container-level cleanup could never settle. // Removed by name when the test finishes, timeouts included: its label value is shared across runs (#189). - onTestFinished(() => { spawnSync(REAL_DOCKER, ['rm', '--force', current.name], { stdio: 'ignore', timeout: 30_000, killSignal: 'SIGKILL' }); }, 60_000); + onTestFinished(() => removeByName('container', current.name), 60_000); try { execFileSync('docker', ['create', '--name', current.name, '--label', 'io.codeboost.invocation=someone-else', '--entrypoint', 'true', imageId], { stdio: 'ignore' }); diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index 0591e278..4008685c 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -8,8 +8,9 @@ import { removeByName, removeLabelledObjects, TEST_RUN_LABEL } from './fixtures/ // The test sweep's handling of output it cannot read (#189). The fake `docker` answers as no real daemon does for // `network ls`: a line that is not JSON, a JSON null and one network without a name. To every other call it answers // nothing and exit 0, as Docker 27.4 does for `ls` with no match; the sweep reads only stdout, the exit status, -// whether a signal killed the call and whether it could start, and stderr only to report a failure. What Docker 27.4 -// really prints is covered against the daemon in agent-container. +// whether a signal killed the call and whether it could start, and stderr only to report a failure. `removeByName` +// also reads an inspect's stderr to tell a missing object from a failure. What Docker 27.4 really prints is covered +// against the daemon in agent-container. const roots: string[] = []; afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); @@ -48,24 +49,42 @@ it('reads nothing from a docker call that was killed, whose output may be cut sh it('removes by name only an object of exactly that name, and reports failures other than no such object', () => { const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); - const calls = join(root, 'calls'), id = 'b'.repeat(64); - // Answers as Docker 27.4 does, read for `--format '{{json .Id}} {{json .Name}}'`: a missing name, another object found - // by an ID prefix, one whose removal is refused, and an inspect killed by a signal. + const calls = join(root, 'calls'), id = 'b'.repeat(64), other = 'c'.repeat(64); + // Answers as Docker 27.4 does for `inspect --format '{{json .Id}} {{json .Name}}'` (a volume: '{{json .Name}}') and + // `rm`, in the parts removeByName reads: stdout of an inspect that succeeded, stderr and the exit status. A missing + // object, an object found under another name (by an ID prefix), a refused removal, an inspect killed by a signal, and a + // failure before the daemon (an unknown context). The stdout of a failed call, a bare newline in 27.4, is not read. writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${calls}'`, 'for arg; do name="$arg"; done', - 'if [ "$2" = inspect ]; then case "$name" in', - " gone) echo 'Error response from daemon: No such container: gone' >&2; exit 1;;", - ` prefix) printf '%s\\n' '"${id}" "/someone-else"';;`, - ` busy) printf '%s\\n' '"${id}" "/busy"';;`, - ' killed) kill -9 $$;;', 'esac; exit 0; fi', - "if [ \"$1\" = rm ]; then echo 'Error response from daemon: conflict' >&2; exit 1; fi", 'exit 0'].join('\n'), + 'if [ "$2" = inspect ]; then case "$1 $name" in', + " 'container gone') echo; echo 'Error response from daemon: No such container: gone' >&2; exit 1;;", + " 'network gone') echo; echo 'Error response from daemon: network gone not found' >&2; exit 1;;", + " 'volume gone') echo; echo 'Error response from daemon: get gone: no such volume' >&2; exit 1;;", + ` 'container prefix') printf '%s\\n' '"${id}" "/someone-else"';;`, + ` 'network prefix') printf '%s\\n' '"${other}" "someone-else"';;`, + ` 'container busy') printf '%s\\n' '"${id}" "/busy"';;`, + ` 'network busy') printf '%s\\n' '"${other}" "busy"';;`, + ` 'volume busy') printf '%s\\n' '"busy"';;`, + " 'container killed') kill -9 $$;;", + " 'container lost') echo 'context \"x\": context not found: open /x/meta.json: no such file or directory' >&2; exit 1;;", + 'esac; exit 0; fi', + "if [ \"$1\" = rm ]; then echo 'Error response from daemon: removal of container is already in progress' >&2; exit 1; fi", + "if [ \"$1 $2\" = 'network rm' ]; then echo 'Error response from daemon: error while removing network: network busy has active endpoints' >&2; exit 1; fi", + "if [ \"$1 $2\" = 'volume rm' ]; then echo 'Error response from daemon: remove busy: volume is in use' >&2; exit 1; fi", + 'exit 0'].join('\n'), { mode: 0o755 }); const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); - for (const name of ['gone', 'prefix', 'busy', 'killed']) removeByName('container', name, join(root, 'docker')); - // Only the exact name is removed, and by its full ID. - expect(readFileSync(calls, 'utf8').split('\n').filter(line => line.startsWith('rm '))).toEqual([`rm --force ${id}`]); + const objects = [['container', 'gone'], ['network', 'gone'], ['volume', 'gone'], ['container', 'prefix'], ['network', 'prefix'], + ['container', 'busy'], ['network', 'busy'], ['volume', 'busy'], ['container', 'killed'], ['container', 'lost']] as const; + for (const [kind, name] of objects) removeByName(kind, name, join(root, 'docker')); + // Only the exact names are removed, by full ID (a volume's name is its ID). + expect(readFileSync(calls, 'utf8').split('\n').filter(line => / rm |^rm /.test(line))) + .toEqual([`rm --force ${id}`, `network rm ${other}`, 'volume rm --force busy']); expect(warn.mock.calls).toEqual([ - ['removing container busy: docker container rm failed:', 'Error response from daemon: conflict'], + ['removing container busy: docker container rm failed:', 'Error response from daemon: removal of container is already in progress'], + ['removing network busy: docker network rm failed:', 'Error response from daemon: error while removing network: network busy has active endpoints'], + ['removing volume busy: docker volume rm failed:', 'Error response from daemon: remove busy: volume is in use'], ['removing container killed: docker inspect failed:', 'killed by SIGKILL'], + ['removing container lost: docker inspect failed:', 'context "x": context not found: open /x/meta.json: no such file or directory'], ]); }); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index a8ef3692..c0a1f0e1 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -32,8 +32,10 @@ export function removeByName(kind: 'container' | 'network' | 'volume', name: str result.error?.message ?? (result.signal ? `killed by ${result.signal}` : result.stderr.trim())); // As JSON: a network's name may hold spaces. Only a container's name comes with a leading slash. const found = spawnSync(docker, [kind, 'inspect', '--format', kind === 'volume' ? '{{json .Name}}' : '{{json .Id}} {{json .Name}}', name], options); - // Docker 27.4's answers for a missing object: "No such container", "network … not found", "no such volume". - if (found.error || found.status !== 0) { if (found.status === null || !/no such|not found/i.test(found.stderr)) failed(found, 'inspect'); return; } + // Docker 27.4's answers for a missing object, each naming it, so a failure before the daemon (an unknown context's + // "context not found: … no such file or directory") is still reported. + const missing = { container: `No such container: ${name}`, network: `network ${name} not found`, volume: `get ${name}: no such volume` }[kind]; + if (found.error || found.status !== 0) { if (found.status === null || !found.stderr.includes(missing)) failed(found, 'inspect'); return; } let id: unknown, actual: unknown; try { const at = found.stdout.indexOf(' '); From 0183f5aab8317a01352a1b5fb0093b93a30ffe60 Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 22:07:56 -0700 Subject: [PATCH 29/75] Real-Docker guard check on a free network; real error text in the fake; budgets (#189) Review round 28: the holder test's removeByName check uses a network with no endpoints, so a broken guard would remove it; the fake's stderr is Docker 27.4's own; the supervisor's finish-time removals get 90 s for two 30 s calls; agent-network makes its records directory on first use; comment fixes. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 18 ++++++++++-------- test/agent-network.test.ts | 12 +++++++----- test/agent-supervisor.test.ts | 4 ++-- test/docker-sweep.test.ts | 31 +++++++++++++++++-------------- 4 files changed, 36 insertions(+), 29 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 68740076..dd9a2cbb 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -49,8 +49,8 @@ const taskFilesystems: ReturnType[] = []; const containers = new Set(); /** * Put, beside its test-run ID, on every object the `withoutRunnerLabel` wrapper makes and on the holder test's swept - * objects (#189): afterAll sweeps it. Other label-less objects are tracked by name (`containers`, `networks`, `volumes`), or found as holders of this run's - * networks and volumes. + * objects (#189): afterAll sweeps it. Other label-less objects are tracked by name (`containers`, `networks`, + * `volumes`), or found as holders of this run's networks and volumes. */ const TEST_FILE_RUN = randomUUID(); /** Networks a test makes without this run's runner label, removed by name in afterAll as `volumes` are. */ @@ -177,8 +177,8 @@ afterAll(async () => { // The runner owner is this run's own, so nothing else would find what this run leaves behind. Objects a test makes // under another owner or without the runner label are not found here, unless they hold one of this run's volumes or // networks: those this file labels with its own run ID are found by the file-label sweep, the try above removes - // those tracked by name (`containers`, `networks`, `volumes`), and the other tests remove their own. It may not run before a - // hook timeout. Its budget covers two sweeps of up to 5.5 minutes each and the cleanup before them. + // those tracked by name (`containers`, `networks`, `volumes`), and the other tests remove their own. It may not run + // before a hook timeout. Its budget covers two sweeps of up to 5.5 minutes each and the cleanup before them. removeLabelledObjects(TEST_FILE_LABEL, TEST_FILE_RUN, 'afterAll'); removeRunnerObjects(TEST_RUNNER_OWNER); } @@ -823,9 +823,10 @@ describe('real Docker agent isolation', () => { `exec '${realDocker}' "$@"`].join('\n'), { mode: 0o755 }); removeLabelledObjects(TEST_RUN_LABEL, run, 'test', join(shim, 'docker')); // afterAll removes these tracked names with removeByName. One now gone is the start of a live object's ID, which - // Docker resolves it to; removeByName must leave that object alone. - expect(docker('network', 'inspect', '--format', '{{.Id}}', bystanderNetwork.slice(0, 12))).toBe(bystanderNetwork); - removeByName('network', bystanderNetwork.slice(0, 12)); + // Docker resolves it to; removeByName must leave that object alone. The network has no endpoints, so Docker would + // remove it if asked. + expect(docker('network', 'inspect', '--format', '{{.Id}}', lookalike.slice(0, 12))).toBe(lookalike); + removeByName('network', lookalike.slice(0, 12)); expect(docker('container', 'inspect', '--format', '{{.Id}}', bystanders[0]!.slice(0, 12))).toBe(bystanders[0]); removeByName('container', bystanders[0]!.slice(0, 12)); const exists = (kind: string, ref: string) => spawnSync('docker', [kind, 'inspect', ref], { stdio: 'ignore' }).status === 0; @@ -841,7 +842,8 @@ describe('real Docker agent isolation', () => { for (const { kind, ref } of made.filter(object => object.kind !== 'container')) spawnSync('docker', [kind, 'rm', ...(kind === 'volume' ? ['--force'] : []), ref], { stdio: 'ignore' }); } - // The budget covers the sweep's worst case and ordinary Docker calls, not every call at its own 60 s limit. + // The budget covers the sweep's worst case (5.5 minutes), two removals by name (up to a minute each) and ordinary + // Docker calls, not every call at its own 60 s limit. }, 8 * 60_000); // #189: these two run in order (run alone, the second fails: it checks what the first left). The first stops after diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index d3495179..2d88d280 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -39,7 +39,9 @@ beforeAll(async () => { * `RECORDS`, not in the shims, so a name a late client adds after the test's own removal is still read here. */ const unlabelled: { kind: 'container' | 'network'; name?: string; file?: string }[] = []; -const RECORDS = mkdtempSync(join(tmpdir(), 'codeboost-records-')); +// Made by the first test that records, so a run that selects none of this file's tests leaves no directory. +let RECORDS = ''; +const recordFile = () => join(RECORDS ||= mkdtempSync(join(tmpdir(), 'codeboost-records-')), randomUUID()); // If setup failed there is no network, and a teardown error would hide the setup failure. afterAll(async () => { try { @@ -57,7 +59,7 @@ afterAll(async () => { // timeout. The hook's 7 minutes leave a minute and a half over this sweep's worst case (eleven calls of 30 s), for // the cleanup before it. removeRunnerObjects(TEST_RUNNER_OWNER); - rmSync(RECORDS, { recursive: true, force: true }); + if (RECORDS) rmSync(RECORDS, { recursive: true, force: true }); } }, 7 * 60_000); @@ -103,7 +105,7 @@ describe('vendor-only egress', () => { }, 60_000); it('leaves a same-named network alone when the daemon refuses the create', async () => { - const shim = mkdtempSync(join(tmpdir(), 'docker-shim-')), recorded = join(RECORDS, randomUUID()); + const shim = mkdtempSync(join(tmpdir(), 'docker-shim-')), recorded = recordFile(); unlabelled.push({ kind: 'network', file: recorded }); const realDocker = requireRealDocker(); // The name is taken by someone else's network just before our create, so the daemon refuses it. @@ -160,7 +162,7 @@ describe('vendor-only egress', () => { }, 60_000); it('neither reports nor removes a same-named proxy when the daemon refuses the create', async () => { - const shim = mkdtempSync(join(tmpdir(), 'docker-shim-')), recorded = join(RECORDS, randomUUID()); + const shim = mkdtempSync(join(tmpdir(), 'docker-shim-')), recorded = recordFile(); unlabelled.push({ kind: 'container', file: recorded }); const realDocker = requireRealDocker(); // Someone else's container takes the proxy name just before our create, so the daemon refuses it. @@ -210,7 +212,7 @@ describe('vendor-only egress', () => { }, 60_000); it('does not delete a same-named stand-in when setup fails after the proxy exists', async () => { - const shim = mkdtempSync(join(tmpdir(), 'docker-shim-')), recorded = join(RECORDS, randomUUID()); + const shim = mkdtempSync(join(tmpdir(), 'docker-shim-')), recorded = recordFile(); unlabelled.push({ kind: 'container', file: recorded }); const realDocker = requireRealDocker(); // The readiness exec swaps the proxy for a same-named, same-labelled stand-in, then fails setup. diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 4da1289a..30479f55 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -388,7 +388,7 @@ describe('container invocation supervisor', () => { it('releases its own resources, and leaves a same-named container another invocation owns', async () => { const current = await profile(fixture(), 'finite-output', 'foreign-name'); // Removed by name when the test finishes, timeouts included: its label value is shared across runs (#189). - onTestFinished(() => removeByName('container', current.name), 60_000); + onTestFinished(() => removeByName('container', current.name), 90_000); // two calls of up to 30 s try { // Inside the try, so its removal runs even if the create's client fails after the daemon made it (#189). execFileSync('docker', ['create', '--name', current.name, '--label', 'io.codeboost.invocation=foreign', @@ -443,7 +443,7 @@ describe('container invocation supervisor', () => { const current = await profile(fixture(), 'noop'); // A foreign container occupies the deterministic name, so container-level cleanup could never settle. // Removed by name when the test finishes, timeouts included: its label value is shared across runs (#189). - onTestFinished(() => removeByName('container', current.name), 60_000); + onTestFinished(() => removeByName('container', current.name), 90_000); // two calls of up to 30 s try { execFileSync('docker', ['create', '--name', current.name, '--label', 'io.codeboost.invocation=someone-else', '--entrypoint', 'true', imageId], { stdio: 'ignore' }); diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index 4008685c..8f5e4e67 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -5,18 +5,18 @@ import { spawnSync } from 'node:child_process'; import { afterEach, expect, it, vi } from 'vitest'; import { removeByName, removeLabelledObjects, TEST_RUN_LABEL } from './fixtures/docker-sweep.ts'; -// The test sweep's handling of output it cannot read (#189). The fake `docker` answers as no real daemon does for -// `network ls`: a line that is not JSON, a JSON null and one network without a name. To every other call it answers -// nothing and exit 0, as Docker 27.4 does for `ls` with no match; the sweep reads only stdout, the exit status, -// whether a signal killed the call and whether it could start, and stderr only to report a failure. `removeByName` -// also reads an inspect's stderr to tell a missing object from a failure. What Docker 27.4 really prints is covered -// against the daemon in agent-container. +// The test sweep's and `removeByName`'s handling of answers they cannot use (#189), each test with its own fake +// `docker`, described beside it. The sweep reads only stdout, the exit status, whether a signal killed the call and +// whether it could start, and stderr only to report a failure; `removeByName` also reads an inspect's stderr to tell a +// missing object from a failure. What Docker 27.4 really prints is covered against the daemon in agent-container. const roots: string[] = []; afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); vi.restoreAllMocks(); }); +// The fake answers as no real daemon does for `network ls`: a line that is not JSON, a JSON null and one network +// without a name. To every other call it answers nothing and exit 0, as Docker 27.4 does for `ls` with no match. it('skips output that is not a network, reports what is not JSON, and never throws', () => { const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); const calls = join(root, 'calls'); @@ -66,11 +66,11 @@ it('removes by name only an object of exactly that name, and reports failures ot ` 'network busy') printf '%s\\n' '"${other}" "busy"';;`, ` 'volume busy') printf '%s\\n' '"busy"';;`, " 'container killed') kill -9 $$;;", - " 'container lost') echo 'context \"x\": context not found: open /x/meta.json: no such file or directory' >&2; exit 1;;", + " 'container lost') echo 'Failed to initialize: unable to resolve docker endpoint: context \"x\": context not found: open /x/meta.json: no such file or directory' >&2; exit 1;;", 'esac; exit 0; fi', - "if [ \"$1\" = rm ]; then echo 'Error response from daemon: removal of container is already in progress' >&2; exit 1; fi", - "if [ \"$1 $2\" = 'network rm' ]; then echo 'Error response from daemon: error while removing network: network busy has active endpoints' >&2; exit 1; fi", - "if [ \"$1 $2\" = 'volume rm' ]; then echo 'Error response from daemon: remove busy: volume is in use' >&2; exit 1; fi", + `if [ "$1" = rm ]; then echo 'Error response from daemon: removal of container ${id} is already in progress' >&2; exit 1; fi`, + `if [ "$1 $2" = 'network rm' ]; then echo 'Error response from daemon: error while removing network: network busy id ${other} has active endpoints' >&2; exit 1; fi`, + `if [ "$1 $2" = 'volume rm' ]; then echo 'Error response from daemon: remove busy: volume is in use - [${id}]' >&2; exit 1; fi`, 'exit 0'].join('\n'), { mode: 0o755 }); const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); @@ -81,10 +81,13 @@ it('removes by name only an object of exactly that name, and reports failures ot expect(readFileSync(calls, 'utf8').split('\n').filter(line => / rm |^rm /.test(line))) .toEqual([`rm --force ${id}`, `network rm ${other}`, 'volume rm --force busy']); expect(warn.mock.calls).toEqual([ - ['removing container busy: docker container rm failed:', 'Error response from daemon: removal of container is already in progress'], - ['removing network busy: docker network rm failed:', 'Error response from daemon: error while removing network: network busy has active endpoints'], - ['removing volume busy: docker volume rm failed:', 'Error response from daemon: remove busy: volume is in use'], + ['removing container busy: docker container rm failed:', + `Error response from daemon: removal of container ${id} is already in progress`], + ['removing network busy: docker network rm failed:', + `Error response from daemon: error while removing network: network busy id ${other} has active endpoints`], + ['removing volume busy: docker volume rm failed:', `Error response from daemon: remove busy: volume is in use - [${id}]`], ['removing container killed: docker inspect failed:', 'killed by SIGKILL'], - ['removing container lost: docker inspect failed:', 'context "x": context not found: open /x/meta.json: no such file or directory'], + ['removing container lost: docker inspect failed:', + 'Failed to initialize: unable to resolve docker endpoint: context "x": context not found: open /x/meta.json: no such file or directory'], ]); }); From 5b30289599f722d76a561cf48b2eb41ae530cf8c Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 22:16:54 -0700 Subject: [PATCH 30/75] Holder test checks removeByName's exact removal and silence on a gone name (#189) Review round 29: one free object of each kind is removed by exact name and asserted gone, and a second removal of the gone names reports nothing; the ID-prefix check's comment says what the names are; the test's finally removes by exact name; the unit file's header names what the daemon test covers. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 29 ++++++++++++++++++++--------- test/docker-sweep.test.ts | 3 ++- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index dd9a2cbb..5a0b6961 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -802,6 +802,10 @@ describe('real Docker agent isolation', () => { const holderLookalike = make('container', holders[0]!.slice(0, 12), 'create', '--entrypoint', 'true', imageId); const labelled = make('container', named('swept'), 'create', '--label', label, '--label', fileLabel, '--entrypoint', 'true', imageId); const labelledLookalike = make('container', labelled.slice(0, 12), 'create', '--entrypoint', 'true', imageId); + // Unlabelled and free, for removeByName below: one of each kind, by its exact name. + const byName = [['container', named('by-name')], ['network', named('by-name')], ['volume', named('by-name')]] as const; + for (const [kind, name] of byName) + make(kind, name, 'create', ...(kind === 'container' ? ['--entrypoint', 'true', imageId] : [])); // The traps are live on this daemon, so the checks below test something: the network filter matches the start of // a network's ID, and a name is resolved before an ID prefix, for networks and containers alike. expect(docker('ps', '--all', '--quiet', '--no-trunc', '--filter', `network=${bystanderNetwork.slice(0, 12)}`).split('\n')) @@ -822,9 +826,17 @@ describe('real Docker agent isolation', () => { `if [ "$1 $2" = "container inspect" ]; then exec '${realDocker}' "$@" codeboost-gone-${randomUUID()}; fi`, `exec '${realDocker}' "$@"`].join('\n'), { mode: 0o755 }); removeLabelledObjects(TEST_RUN_LABEL, run, 'test', join(shim, 'docker')); - // afterAll removes these tracked names with removeByName. One now gone is the start of a live object's ID, which - // Docker resolves it to; removeByName must leave that object alone. The network has no endpoints, so Docker would - // remove it if asked. + // afterAll removes tracked names with removeByName, as here: an object of exactly that name goes, and a name that + // is gone says nothing. + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + try { + for (const [kind, name] of byName) removeByName(kind, name); + expect(byName.filter(([kind, name]) => spawnSync('docker', [kind, 'inspect', name], { stdio: 'ignore' }).status === 0)).toEqual([]); + for (const [kind, name] of byName) removeByName(kind, name); + expect(warn).not.toHaveBeenCalled(); + } finally { warn.mockRestore(); } + // A missing name that Docker resolves to a live object's ID, as a tracked name gone since would be: removeByName + // must leave that object alone. The network has no endpoints, so Docker would remove it if asked. expect(docker('network', 'inspect', '--format', '{{.Id}}', lookalike.slice(0, 12))).toBe(lookalike); removeByName('network', lookalike.slice(0, 12)); expect(docker('container', 'inspect', '--format', '{{.Id}}', bystanders[0]!.slice(0, 12))).toBe(bystanders[0]); @@ -837,13 +849,12 @@ describe('real Docker agent isolation', () => { expect([exists('network', bystanderNetwork), exists('volume', bystanderVolume), exists('network', lookalike)]) .toEqual([true, true, true]); } finally { - for (const { ref } of made.filter(object => object.kind === 'container')) - spawnSync('docker', ['rm', '--force', ref], { stdio: 'ignore' }); - for (const { kind, ref } of made.filter(object => object.kind !== 'container')) - spawnSync('docker', [kind, 'rm', ...(kind === 'volume' ? ['--force'] : []), ref], { stdio: 'ignore' }); + // By exact name: some are named like the start of another object's ID. Containers first, as they hold the rest. + for (const { kind, ref } of [...made.filter(object => object.kind === 'container'), ...made.filter(object => object.kind !== 'container')]) + removeByName(kind, ref); } - // The budget covers the sweep's worst case (5.5 minutes), two removals by name (up to a minute each) and ordinary - // Docker calls, not every call at its own 60 s limit. + // The budget covers the sweep's worst case (5.5 minutes) and ordinary Docker calls, removals by name included, not + // every call at its own limit. }, 8 * 60_000); // #189: these two run in order (run alone, the second fails: it checks what the first left). The first stops after diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index 8f5e4e67..3ccdcf8c 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -8,7 +8,8 @@ import { removeByName, removeLabelledObjects, TEST_RUN_LABEL } from './fixtures/ // The test sweep's and `removeByName`'s handling of answers they cannot use (#189), each test with its own fake // `docker`, described beside it. The sweep reads only stdout, the exit status, whether a signal killed the call and // whether it could start, and stderr only to report a failure; `removeByName` also reads an inspect's stderr to tell a -// missing object from a failure. What Docker 27.4 really prints is covered against the daemon in agent-container. +// missing object from a failure. Against the daemon, agent-container's holder test covers the sweep and removeByName's +// removal, its silence on a missing name, and its refusal of an object found by an ID prefix. const roots: string[] = []; afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); From 1227ec13af9311b7f358c4fdd6d94fd0a0a7d561 Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 22:27:20 -0700 Subject: [PATCH 31/75] The stand-in shim acts only on an exec in a proxy (#189) Review round 30: any other exec passes through, so the shim can never run a stand-in with an empty name, which Docker would give a random, unrecorded one. Co-Authored-By: Claude Opus 5.5 --- test/agent-network.test.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index 2d88d280..44b8bfb1 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -215,15 +215,17 @@ describe('vendor-only egress', () => { const shim = mkdtempSync(join(tmpdir(), 'docker-shim-')), recorded = recordFile(); unlabelled.push({ kind: 'container', file: recorded }); const realDocker = requireRealDocker(); - // The readiness exec swaps the proxy for a same-named, same-labelled stand-in, then fails setup. + // The readiness exec swaps the proxy for a same-named, same-labelled stand-in, then fails setup. Only an exec in a + // proxy: any other call passes through, so the shim never makes a stand-in with an empty, random name. writeFileSync(join(shim, 'docker'), ['#!/bin/sh', 'if [ "$1" = exec ]; then', - ` name=$('${realDocker}' inspect -f '{{.Name}}' "$2" | sed 's#^/##')`, + ` name=$('${realDocker}' inspect -f '{{.Name}}' "$2" 2>/dev/null | sed 's#^/##')`, + ' case "$name" in codeboost-proxy-*)', ` label=$('${realDocker}' inspect -f '{{index .Config.Labels "io.codeboost.egress"}}' "$2")`, ` '${realDocker}' rm --force "$2" >/dev/null`, // Recorded before the stand-in exists, so the test can remove it even if the run is cut short. ` printf '%s\\n' "$name" >> '${recorded}'`, ` '${realDocker}' run --detach --name "$name" --label "io.codeboost.egress=$label" --entrypoint sleep ${imageId} 300 >/dev/null`, - ' exit 1', 'fi', `exec '${realDocker}' "$@"`].join('\n'), { mode: 0o755 }); + ' exit 1;;', ' esac', 'fi', `exec '${realDocker}' "$@"`].join('\n'), { mode: 0o755 }); const failing = captureInvocation({ ...invocation, attemptId: `failed-setup-${randomUUID()}`, deadline: Date.now() + 60_000 }); const path = process.env.PATH; From 9cb832470cbb90b6546cba399f3b7facf9c74c6b Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 22:37:02 -0700 Subject: [PATCH 32/75] Doc: a test may sweep its own runner owner with removeLabelledObjects (#189) Review round 31. Co-Authored-By: Claude Opus 5.5 --- test/fixtures/docker-sweep.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index c0a1f0e1..59673b9e 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -49,8 +49,8 @@ export function removeByName(kind: 'container' | 'network' | 'volume', name: str /** * Removes the containers, networks and volumes labelled with one runner owner, and every container that holds one of * those networks or volumes, whatever its labels (see `removeLabelledObjects`). A test file's afterAll calls it after - * its other cleanup, and a test that makes objects under its own runner owner sweeps them, by owner here or by - * allocation with `removeLabelledObjects`. + * its other cleanup, and a test that makes objects under its own runner owner sweeps them, by owner (here, or with + * `removeLabelledObjects(RUNNER_LABEL, owner)`) or by allocation with `removeLabelledObjects`. */ export function removeRunnerObjects(runnerOwner: string, when = 'afterAll'): void { removeLabelledObjects(RUNNER_LABEL, runnerOwner, when); From 08d1a65a0645fa473329d0c144993a8e71515e0d Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 22:45:39 -0700 Subject: [PATCH 33/75] Say what the killed-call fake leaves out (#189) Review round 32. Co-Authored-By: Claude Opus 5.5 --- test/docker-sweep.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index 3ccdcf8c..04a81514 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -34,7 +34,9 @@ it('skips output that is not a network, reports what is not JSON, and never thro it('reads nothing from a docker call that was killed, whose output may be cut short', () => { const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); const calls = join(root, 'calls'); - // A well-formed network line, then the client is killed by a signal before it exits by itself. + // A well-formed network line, then the client is killed by a signal before it exits by itself. The line holds only + // the fields the sweep reads (Docker 27.4 also prints CreatedAt, Driver, IPv6, Internal, Labels and Scope); every + // other call gets no output and exit 0, as Docker 27.4 gives for `ls` with no match. writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${calls}'`, `if [ "$1 $2" = "network ls" ]; then printf '%s\\n' '{"ID":"${'a'.repeat(64)}","Name":"swept"}'; kill -9 $$; fi`, 'exit 0'].join('\n'), { mode: 0o755 }); From 88ba06291c53e98318bf3283a064b8ce475a8f3d Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 22:54:21 -0700 Subject: [PATCH 34/75] Sweeps keep the Docker daemon set when they loaded (#189) Review round 33: a test that times out may leave DOCKER_HOST changed (agent-supervisor points it at a missing socket until its handle settles); every call in docker-sweep.ts now restores DOCKER_HOST and DOCKER_CONTEXT as they were at load, with a unit test that fails without it; the context fake says its path is shortened. Co-Authored-By: Claude Opus 5.5 --- test/docker-sweep.test.ts | 22 +++++++++++++++++++++- test/fixtures/docker-sweep.ts | 15 +++++++++++++-- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index 04a81514..0bdd3402 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -56,7 +56,8 @@ it('removes by name only an object of exactly that name, and reports failures ot // Answers as Docker 27.4 does for `inspect --format '{{json .Id}} {{json .Name}}'` (a volume: '{{json .Name}}') and // `rm`, in the parts removeByName reads: stdout of an inspect that succeeded, stderr and the exit status. A missing // object, an object found under another name (by an ID prefix), a refused removal, an inspect killed by a signal, and a - // failure before the daemon (an unknown context). The stdout of a failed call, a bare newline in 27.4, is not read. + // failure before the daemon (an unknown context; its meta.json path shortened). The stdout of a failed call, a bare + // newline in 27.4, is not read. writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${calls}'`, 'for arg; do name="$arg"; done', 'if [ "$2" = inspect ]; then case "$1 $name" in', @@ -94,3 +95,22 @@ it('removes by name only an object of exactly that name, and reports failures ot 'Failed to initialize: unable to resolve docker endpoint: context "x": context not found: open /x/meta.json: no such file or directory'], ]); }); + +it('talks to the daemon set when it loaded, though a test changed DOCKER_HOST or DOCKER_CONTEXT since', () => { + const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); + const seen = join(root, 'seen'); + // Records the address each call is given, and answers as Docker 27.4 does for a missing container and an empty `ls`. + writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s|%s\\n' "\${DOCKER_HOST-unset}" "\${DOCKER_CONTEXT-unset}" >> '${seen}'`, + 'if [ "$2" = inspect ]; then echo; echo "Error response from daemon: No such container: x" >&2; exit 1; fi', 'exit 0'].join('\n'), + { mode: 0o755 }); + const loaded = `${process.env.DOCKER_HOST ?? 'unset'}|${process.env.DOCKER_CONTEXT ?? 'unset'}`; + vi.stubEnv('DOCKER_HOST', 'unix:///codeboost-missing.sock'); + vi.stubEnv('DOCKER_CONTEXT', 'codeboost-missing'); + try { + removeByName('container', 'x', join(root, 'docker')); + removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test', join(root, 'docker')); + } finally { vi.unstubAllEnvs(); } + const lines = readFileSync(seen, 'utf8').split('\n').filter(Boolean); + expect(lines.length).toBeGreaterThan(1); + expect(new Set(lines)).toEqual(new Set([loaded])); +}); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index 59673b9e..9c87f46f 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -7,6 +7,17 @@ import { RUNNER_LABEL } from '../../agents/labels.ts'; * call's SIGKILL does not reach. */ export const REAL_DOCKER = spawnSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).stdout?.trim() || 'docker'; +/** + * The daemon `docker` talks to when this module loads. A test that times out may leave DOCKER_HOST or DOCKER_CONTEXT + * changed (agent-supervisor points DOCKER_HOST at a missing socket until its handle settles), so every call here + * restores them. + */ +const LOADED_ENV = { DOCKER_HOST: process.env.DOCKER_HOST, DOCKER_CONTEXT: process.env.DOCKER_CONTEXT }; +const dockerEnv = () => { + const env = { ...process.env }; + for (const [name, value] of Object.entries(LOADED_ENV)) if (value === undefined) delete env[name]; else env[name] = value; + return env; +}; /** The real `docker` for a shim to call; refuses when none was found, since a shim calling `docker` would call itself. */ export function requireRealDocker(): string { if (!REAL_DOCKER.startsWith('/')) throw new Error('No docker on PATH when the test files loaded.'); @@ -27,7 +38,7 @@ export const TEST_FILE_LABEL = 'io.codeboost.test-file'; * reports a failed call, other than an inspect that finds no such object, and never throws; each call has a 30 s limit. */ export function removeByName(kind: 'container' | 'network' | 'volume', name: string, docker = REAL_DOCKER): void { - const options = { encoding: 'utf8' as const, timeout: 30_000, killSignal: 'SIGKILL' as const }; + const options = { encoding: 'utf8' as const, timeout: 30_000, killSignal: 'SIGKILL' as const, env: dockerEnv() }; const failed = (result: SpawnSyncReturns, what: string) => console.warn(`removing ${kind} ${name}: docker ${what} failed:`, result.error?.message ?? (result.signal ? `killed by ${result.signal}` : result.stderr.trim())); // As JSON: a network's name may hold spaces. Only a container's name comes with a leading slash. @@ -67,7 +78,7 @@ export function removeRunnerObjects(runnerOwner: string, when = 'afterAll'): voi */ export function removeLabelledObjects(label: string, value: string, when: string, docker = REAL_DOCKER): void { const run = (args: string[]) => { - const result = spawnSync(docker, args, { encoding: 'utf8', timeout: 30_000, killSignal: 'SIGKILL' }); + const result = spawnSync(docker, args, { encoding: 'utf8', timeout: 30_000, killSignal: 'SIGKILL', env: dockerEnv() }); if (result.status !== 0) console.warn(`${when}: docker ${args.slice(0, 2).join(' ')} failed:`, result.error?.message ?? (result.signal ? `killed by ${result.signal}` : result.stderr.trim())); // An inspect of several objects exits 1 if one is gone, yet prints the rest (Docker 27.4), so the output of a call From 26e9103c43fa5beed7e6a708b5caf5b133353747 Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 23:05:43 -0700 Subject: [PATCH 35/75] Comment and naming fixes (#189) Review round 34: what the loaded Docker address means for each call, which stdout a failed call leaves unread, who adds the test-run label; the records directory variable is lower case, as it changes. Co-Authored-By: Claude Opus 5.5 --- test/agent-network.test.ts | 8 ++++---- test/docker-sweep.test.ts | 4 ++-- test/fixtures/docker-sweep.ts | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index 44b8bfb1..82686d92 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -36,12 +36,12 @@ beforeAll(async () => { /** * Objects a test makes without this run's runner label, by name or by the file its shim records the name in, so * afterAll removes them should the test time out before its own removal runs (#189). The record files live in - * `RECORDS`, not in the shims, so a name a late client adds after the test's own removal is still read here. + * `records`, not in the shims, so a name a late client adds after the test's own removal is still read here. */ const unlabelled: { kind: 'container' | 'network'; name?: string; file?: string }[] = []; // Made by the first test that records, so a run that selects none of this file's tests leaves no directory. -let RECORDS = ''; -const recordFile = () => join(RECORDS ||= mkdtempSync(join(tmpdir(), 'codeboost-records-')), randomUUID()); +let records = ''; +const recordFile = () => join(records ||= mkdtempSync(join(tmpdir(), 'codeboost-records-')), randomUUID()); // If setup failed there is no network, and a teardown error would hide the setup failure. afterAll(async () => { try { @@ -59,7 +59,7 @@ afterAll(async () => { // timeout. The hook's 7 minutes leave a minute and a half over this sweep's worst case (eleven calls of 30 s), for // the cleanup before it. removeRunnerObjects(TEST_RUNNER_OWNER); - if (RECORDS) rmSync(RECORDS, { recursive: true, force: true }); + if (records) rmSync(records, { recursive: true, force: true }); } }, 7 * 60_000); diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index 0bdd3402..a3f19cc9 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -56,8 +56,8 @@ it('removes by name only an object of exactly that name, and reports failures ot // Answers as Docker 27.4 does for `inspect --format '{{json .Id}} {{json .Name}}'` (a volume: '{{json .Name}}') and // `rm`, in the parts removeByName reads: stdout of an inspect that succeeded, stderr and the exit status. A missing // object, an object found under another name (by an ID prefix), a refused removal, an inspect killed by a signal, and a - // failure before the daemon (an unknown context; its meta.json path shortened). The stdout of a failed call, a bare - // newline in 27.4, is not read. + // failure before the daemon (an unknown context; its meta.json path shortened). The stdout of a failed call (a bare + // newline for a missing object in 27.4, nothing for the others) is not read. writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${calls}'`, 'for arg; do name="$arg"; done', 'if [ "$2" = inspect ]; then case "$1 $name" in', diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index 9c87f46f..34744189 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -10,7 +10,7 @@ export const REAL_DOCKER = spawnSync('sh', ['-c', 'command -v docker'], { encodi /** * The daemon `docker` talks to when this module loads. A test that times out may leave DOCKER_HOST or DOCKER_CONTEXT * changed (agent-supervisor points DOCKER_HOST at a missing socket until its handle settles), so every call here - * restores them. + * runs with the values they had when this module loaded. */ const LOADED_ENV = { DOCKER_HOST: process.env.DOCKER_HOST, DOCKER_CONTEXT: process.env.DOCKER_CONTEXT }; const dockerEnv = () => { @@ -25,8 +25,8 @@ export function requireRealDocker(): string { } /** - * The label a test's Docker shim adds to an object it makes without the runner label (#189), so the test can still - * find and remove that object however the test ends. No codeboost code writes or reads it. + * The label a test (most often through its Docker shim) adds to an object it makes without the runner label (#189), + * so the test can still find and remove that object however the test ends. No codeboost code writes or reads it. */ export const TEST_RUN_LABEL = 'io.codeboost.test-run'; /** Beside it, one value per test file run, so afterAll removes them all with one sweep. */ From 6497cf7d81242a61ede4d290ba9a8ff997dc719a Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 23:18:28 -0700 Subject: [PATCH 36/75] The racer shim acts only on its own allocation's create (#189) Review round 35: should a race test time out with the shim on PATH, other tests' creates pass through; withoutRunnerLabel's doc says what a wrapper left on PATH does to a later test. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 5a0b6961..4ba9d306 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -704,7 +704,9 @@ describe('real Docker agent isolation', () => { // holds one of this run's volumes or networks, and such a volume or network never, so the wrapper labels it with a // test-run ID and this file's run ID instead (#189). Whatever has that ID is removed when the test finishes (Vitest // runs `onTestFinished` after a pass, a failure or a timeout; the `it.fails` pair below tests the failure), and again - // in afterAll, for a create that lands after a timed-out test's removal (its body runs on). + // in afterAll, for a create that lands after a timed-out test's removal (its body runs on). A timed-out test leaves the + // wrapper on PATH while its body runs on, so a later test's matching create is unlabelled too and that test fails; + // what it makes carries this file's run ID, so afterAll removes it. const withoutRunnerLabel = async (prefixes: string[][], run: () => Promise | T): Promise => { if (!prefixes.every(prefix => prefix.includes('create'))) throw new Error('Only create commands can be unlabelled.'); const testRun = randomUUID(); @@ -913,10 +915,11 @@ describe('real Docker agent isolation', () => { try { return await run(); } finally { process.env.PATH = path; } }; // After our create, another process creates a volume carrying the same allocation ID: both passed the check first. - const racer = (name: string) => [ + // Only for this allocation's create: should the test time out with the shim still on PATH, other creates pass through. + const racer = (name: string, allocationId: string) => [ 'result = run(args);', - "const allocation = args.find(arg => arg.startsWith('io.codeboost.allocation='));", - `run(['volume', 'create', '--label', 'io.codeboost.runner=${'f'.repeat(32)}', '--label', allocation, '${name}']);`, + `if (args.includes('io.codeboost.allocation=${allocationId}'))`, + ` run(['volume', 'create', '--label', 'io.codeboost.runner=${'f'.repeat(32)}', '--label', 'io.codeboost.allocation=${allocationId}', '${name}']);`, ].join('\n'); it('backs out of task storage when a concurrent process claims the same allocation ID', async () => { @@ -924,7 +927,7 @@ describe('real Docker agent isolation', () => { // Tracked for afterAll should the test time out: the rival carries another runner's label (#189). volumes.add(rival); try { - await withDockerShim(['volume', 'create'], racer(rival), () => expect(() => prepareTaskFilesystems(data.clone, + await withDockerShim(['volume', 'create'], racer(rival, owner.allocationId), () => expect(() => prepareTaskFilesystems(data.clone, { workBytes: 16 * 1024 * 1024, workInodes: 512, metadataBytes: 16 * 1024 * 1024, metadataInodes: 512 }, imageId, owner)).toThrow('still labels a Docker object')); expect(byAllocation(owner.allocationId)).toEqual([rival]); @@ -936,7 +939,7 @@ describe('real Docker agent isolation', () => { // Tracked for afterAll should the test time out: the rival carries another runner's label (#189). volumes.add(rival); try { - await withDockerShim(['network', 'create'], racer(rival), () => expect(createVendorNetwork( + await withDockerShim(['network', 'create'], racer(rival, allocationId), () => expect(createVendorNetwork( invocation(data.clone, 'planning'), imageId, allocationId)).rejects.toThrow('still labels a Docker object')); expect(byAllocation(allocationId)).toEqual([rival]); } finally { spawnSync('docker', ['volume', 'rm', '--force', rival], { stdio: 'ignore' }); } From b1176fd7fe525db08e69788b0f40ce01c5de28d5 Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 23:29:41 -0700 Subject: [PATCH 37/75] removeByName's reports name the stage and the command run (#189) Review round 36: removeByName takes a when prefix, as removeLabelledObjects does, and names the command it ran (docker rm, docker container inspect). Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 16 ++++++++-------- test/agent-network.test.ts | 8 ++++---- test/agent-supervisor.test.ts | 4 ++-- test/docker-sweep.test.ts | 14 +++++++------- test/fixtures/docker-sweep.ts | 13 +++++++------ 5 files changed, 28 insertions(+), 27 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 4ba9d306..231eaf9a 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -158,9 +158,9 @@ afterEach(async () => { afterAll(async () => { try { // By exact name: some are named like the start of another object's ID. - for (const container of containers) removeByName('container', container); - for (const network of networks) removeByName('network', network); - for (const volume of volumes) removeByName('volume', volume); + for (const container of containers) removeByName('container', container, 'afterAll'); + for (const network of networks) removeByName('network', network, 'afterAll'); + for (const volume of volumes) removeByName('volume', volume, 'afterAll'); // Each step, even when an earlier one fails: storage left here is found by the sweep below only by its label, and // every failure is reported. const failures: unknown[] = []; @@ -832,17 +832,17 @@ describe('real Docker agent isolation', () => { // is gone says nothing. const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); try { - for (const [kind, name] of byName) removeByName(kind, name); + for (const [kind, name] of byName) removeByName(kind, name, 'test'); expect(byName.filter(([kind, name]) => spawnSync('docker', [kind, 'inspect', name], { stdio: 'ignore' }).status === 0)).toEqual([]); - for (const [kind, name] of byName) removeByName(kind, name); + for (const [kind, name] of byName) removeByName(kind, name, 'test'); expect(warn).not.toHaveBeenCalled(); } finally { warn.mockRestore(); } // A missing name that Docker resolves to a live object's ID, as a tracked name gone since would be: removeByName // must leave that object alone. The network has no endpoints, so Docker would remove it if asked. expect(docker('network', 'inspect', '--format', '{{.Id}}', lookalike.slice(0, 12))).toBe(lookalike); - removeByName('network', lookalike.slice(0, 12)); + removeByName('network', lookalike.slice(0, 12), 'test'); expect(docker('container', 'inspect', '--format', '{{.Id}}', bystanders[0]!.slice(0, 12))).toBe(bystanders[0]); - removeByName('container', bystanders[0]!.slice(0, 12)); + removeByName('container', bystanders[0]!.slice(0, 12), 'test'); const exists = (kind: string, ref: string) => spawnSync('docker', [kind, 'inspect', ref], { stdio: 'ignore' }).status === 0; for (const holder of holders) expect(exists('container', holder)).toBe(false); expect([exists('network', network), exists('network', prefixNamed), exists('volume', volume)]).toEqual([false, false, false]); @@ -853,7 +853,7 @@ describe('real Docker agent isolation', () => { } finally { // By exact name: some are named like the start of another object's ID. Containers first, as they hold the rest. for (const { kind, ref } of [...made.filter(object => object.kind === 'container'), ...made.filter(object => object.kind !== 'container')]) - removeByName(kind, ref); + removeByName(kind, ref, 'finally'); } // The budget covers the sweep's worst case (5.5 minutes) and ordinary Docker calls, removals by name included, not // every call at its own limit. diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index 82686d92..982c2a99 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -48,7 +48,7 @@ afterAll(async () => { for (const { kind, name, file } of unlabelled) { // A record holds one name per line: should a test time out with its shim still on PATH, later creates add theirs. for (const ref of name ? [name] : file && existsSync(file) ? readFileSync(file, 'utf8').split('\n').filter(Boolean) : []) - removeByName(kind, ref); + removeByName(kind, ref, 'afterAll'); } if (network) await removeVendorNetwork(network); } @@ -128,7 +128,7 @@ describe('vendor-only egress', () => { foreign = made[0] ?? ''; // afterAll reads the record again: the object may land after this removal (#189). if (foreign) survived = spawnSync('docker', ['network', 'inspect', foreign], { stdio: 'ignore' }).status === 0; - for (const name of made) removeByName('network', name); + for (const name of made) removeByName('network', name, 'finally'); rmSync(shim, { recursive: true, force: true }); } // The refused create is reported as itself, with no cleanup left to retry, and the foreign network survives. @@ -184,7 +184,7 @@ describe('vendor-only egress', () => { foreign = made[0] ?? ''; // afterAll reads the record again: the object may land after this removal (#189). if (foreign) survived = spawnSync('docker', ['container', 'inspect', foreign], { stdio: 'ignore' }).status === 0; - for (const name of made) removeByName('container', name); + for (const name of made) removeByName('container', name, 'finally'); rmSync(shim, { recursive: true, force: true }); } // The refused create is reported as itself: our network was removed, nothing is left to retry or report. @@ -240,7 +240,7 @@ describe('vendor-only egress', () => { impostor = made[0] ?? ''; // afterAll reads the record again: the object may land after this removal (#189). if (impostor) survived = spawnSync('docker', ['container', 'inspect', impostor], { stdio: 'ignore' }).status === 0; - for (const name of made) removeByName('container', name); + for (const name of made) removeByName('container', name, 'finally'); rmSync(shim, { recursive: true, force: true }); } expect(impostor).toMatch(/^codeboost-proxy-/); diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 30479f55..4828ac58 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -388,7 +388,7 @@ describe('container invocation supervisor', () => { it('releases its own resources, and leaves a same-named container another invocation owns', async () => { const current = await profile(fixture(), 'finite-output', 'foreign-name'); // Removed by name when the test finishes, timeouts included: its label value is shared across runs (#189). - onTestFinished(() => removeByName('container', current.name), 90_000); // two calls of up to 30 s + onTestFinished(() => removeByName('container', current.name, 'onTestFinished'), 90_000); // two calls of up to 30 s try { // Inside the try, so its removal runs even if the create's client fails after the daemon made it (#189). execFileSync('docker', ['create', '--name', current.name, '--label', 'io.codeboost.invocation=foreign', @@ -443,7 +443,7 @@ describe('container invocation supervisor', () => { const current = await profile(fixture(), 'noop'); // A foreign container occupies the deterministic name, so container-level cleanup could never settle. // Removed by name when the test finishes, timeouts included: its label value is shared across runs (#189). - onTestFinished(() => removeByName('container', current.name), 90_000); // two calls of up to 30 s + onTestFinished(() => removeByName('container', current.name, 'onTestFinished'), 90_000); // two calls of up to 30 s try { execFileSync('docker', ['create', '--name', current.name, '--label', 'io.codeboost.invocation=someone-else', '--entrypoint', 'true', imageId], { stdio: 'ignore' }); diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index a3f19cc9..682654ac 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -80,18 +80,18 @@ it('removes by name only an object of exactly that name, and reports failures ot const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); const objects = [['container', 'gone'], ['network', 'gone'], ['volume', 'gone'], ['container', 'prefix'], ['network', 'prefix'], ['container', 'busy'], ['network', 'busy'], ['volume', 'busy'], ['container', 'killed'], ['container', 'lost']] as const; - for (const [kind, name] of objects) removeByName(kind, name, join(root, 'docker')); + for (const [kind, name] of objects) removeByName(kind, name, 'test', join(root, 'docker')); // Only the exact names are removed, by full ID (a volume's name is its ID). expect(readFileSync(calls, 'utf8').split('\n').filter(line => / rm |^rm /.test(line))) .toEqual([`rm --force ${id}`, `network rm ${other}`, 'volume rm --force busy']); expect(warn.mock.calls).toEqual([ - ['removing container busy: docker container rm failed:', + ['test: removing container busy: docker rm failed:', `Error response from daemon: removal of container ${id} is already in progress`], - ['removing network busy: docker network rm failed:', + ['test: removing network busy: docker network rm failed:', `Error response from daemon: error while removing network: network busy id ${other} has active endpoints`], - ['removing volume busy: docker volume rm failed:', `Error response from daemon: remove busy: volume is in use - [${id}]`], - ['removing container killed: docker inspect failed:', 'killed by SIGKILL'], - ['removing container lost: docker inspect failed:', + ['test: removing volume busy: docker volume rm failed:', `Error response from daemon: remove busy: volume is in use - [${id}]`], + ['test: removing container killed: docker container inspect failed:', 'killed by SIGKILL'], + ['test: removing container lost: docker container inspect failed:', 'Failed to initialize: unable to resolve docker endpoint: context "x": context not found: open /x/meta.json: no such file or directory'], ]); }); @@ -107,7 +107,7 @@ it('talks to the daemon set when it loaded, though a test changed DOCKER_HOST or vi.stubEnv('DOCKER_HOST', 'unix:///codeboost-missing.sock'); vi.stubEnv('DOCKER_CONTEXT', 'codeboost-missing'); try { - removeByName('container', 'x', join(root, 'docker')); + removeByName('container', 'x', 'test', join(root, 'docker')); removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test', join(root, 'docker')); } finally { vi.unstubAllEnvs(); } const lines = readFileSync(seen, 'utf8').split('\n').filter(Boolean); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index 34744189..2a5265b4 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -35,26 +35,27 @@ export const TEST_FILE_LABEL = 'io.codeboost.test-file'; /** * Removes the container, network or volume named exactly `name`, by its full ID (a volume by its name, which is its * ID). Docker would read a missing name as an ID prefix, so an object found under another name is left alone. It - * reports a failed call, other than an inspect that finds no such object, and never throws; each call has a 30 s limit. + * reports a failed call, other than an inspect that finds no such object, prefixed with `when`, and never throws; each + * call has a 30 s limit. */ -export function removeByName(kind: 'container' | 'network' | 'volume', name: string, docker = REAL_DOCKER): void { +export function removeByName(kind: 'container' | 'network' | 'volume', name: string, when: string, docker = REAL_DOCKER): void { const options = { encoding: 'utf8' as const, timeout: 30_000, killSignal: 'SIGKILL' as const, env: dockerEnv() }; - const failed = (result: SpawnSyncReturns, what: string) => console.warn(`removing ${kind} ${name}: docker ${what} failed:`, + const failed = (result: SpawnSyncReturns, what: string) => console.warn(`${when}: removing ${kind} ${name}: docker ${what} failed:`, result.error?.message ?? (result.signal ? `killed by ${result.signal}` : result.stderr.trim())); // As JSON: a network's name may hold spaces. Only a container's name comes with a leading slash. const found = spawnSync(docker, [kind, 'inspect', '--format', kind === 'volume' ? '{{json .Name}}' : '{{json .Id}} {{json .Name}}', name], options); // Docker 27.4's answers for a missing object, each naming it, so a failure before the daemon (an unknown context's // "context not found: … no such file or directory") is still reported. const missing = { container: `No such container: ${name}`, network: `network ${name} not found`, volume: `get ${name}: no such volume` }[kind]; - if (found.error || found.status !== 0) { if (found.status === null || !found.stderr.includes(missing)) failed(found, 'inspect'); return; } + if (found.error || found.status !== 0) { if (found.status === null || !found.stderr.includes(missing)) failed(found, `${kind} inspect`); return; } let id: unknown, actual: unknown; try { const at = found.stdout.indexOf(' '); [id, actual] = kind === 'volume' ? [name, JSON.parse(found.stdout)] : [JSON.parse(found.stdout.slice(0, at)), JSON.parse(found.stdout.slice(at + 1))]; - } catch { console.warn(`removing ${kind} ${name}: docker printed what is not JSON:`, found.stdout); return; } + } catch { console.warn(`${when}: removing ${kind} ${name}: docker printed what is not JSON:`, found.stdout); return; } if (typeof id !== 'string' || typeof actual !== 'string' || (kind === 'container' ? actual.replace(/^\//, '') : actual) !== name) return; const removed = spawnSync(docker, kind === 'container' ? ['rm', '--force', id] : [kind, 'rm', ...(kind === 'volume' ? ['--force'] : []), id], options); - if (removed.error || removed.status !== 0) failed(removed, `${kind} rm`); + if (removed.error || removed.status !== 0) failed(removed, kind === 'container' ? 'rm' : `${kind} rm`); } /** From 115a2b28056a4bcb0d03fdd97f3da6602eda8e8e Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 23:38:18 -0700 Subject: [PATCH 38/75] Sweep reports name their stage; a gone holder is not a failure; loaded daemon tested (#189) Review round 37: removeLabelledObjects names the stage of each failed call and says nothing of a holder gone since the listing (Docker 27.4's No such container for it), still reporting any other failure; a test loads a fresh copy of the module with DOCKER_HOST and DOCKER_CONTEXT set and shows the sweep keeps them; unit tests cover both holder inspect answers. Co-Authored-By: Claude Opus 5.5 --- test/docker-sweep.test.ts | 43 ++++++++++++++++++++++++++++++++++- test/fixtures/docker-sweep.ts | 17 ++++++++------ 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index 682654ac..0672e926 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -4,6 +4,7 @@ import { join } from 'node:path'; import { spawnSync } from 'node:child_process'; import { afterEach, expect, it, vi } from 'vitest'; import { removeByName, removeLabelledObjects, TEST_RUN_LABEL } from './fixtures/docker-sweep.ts'; +import type * as Sweep from './fixtures/docker-sweep.ts'; // The test sweep's and `removeByName`'s handling of answers they cannot use (#189), each test with its own fake // `docker`, described beside it. The sweep reads only stdout, the exit status, whether a signal killed the call and @@ -47,7 +48,7 @@ it('reads nothing from a docker call that was killed, whose output may be cut sh const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test', join(root, 'docker')); expect(readFileSync(calls, 'utf8').split('\n').filter(line => /^(ps|network rm) /.test(line))).toEqual([]); - expect(warn.mock.calls).toContainEqual(['test: docker network ls failed:', 'killed by SIGKILL']); + expect(warn.mock.calls).toContainEqual(['test: listing networks: docker network ls failed:', 'killed by SIGKILL']); }); it('removes by name only an object of exactly that name, and reports failures other than no such object', () => { @@ -114,3 +115,43 @@ it('talks to the daemon set when it loaded, though a test changed DOCKER_HOST or expect(lines.length).toBeGreaterThan(1); expect(new Set(lines)).toEqual(new Set([loaded])); }); + +it('takes the daemon set at load, not the default, when one was set', async () => { + const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); + const seen = join(root, 'seen'); + writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s|%s\\n' "\${DOCKER_HOST-unset}" "\${DOCKER_CONTEXT-unset}" >> '${seen}'`, + 'if [ "$2" = inspect ]; then echo; echo "Error response from daemon: No such container: x" >&2; exit 1; fi', 'exit 0'].join('\n'), + { mode: 0o755 }); + // A fresh copy of the module, loaded while a daemon is set, as on a machine with Colima or rootless Docker. + vi.stubEnv('DOCKER_HOST', 'unix:///codeboost-loaded.sock'); + vi.stubEnv('DOCKER_CONTEXT', 'codeboost-loaded'); + vi.resetModules(); + const fresh = await import('./fixtures/docker-sweep.ts') as typeof Sweep; + vi.stubEnv('DOCKER_HOST', 'unix:///codeboost-changed.sock'); + vi.stubEnv('DOCKER_CONTEXT', 'codeboost-changed'); + try { + fresh.removeByName('container', 'x', 'test', join(root, 'docker')); + fresh.removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test', join(root, 'docker')); + } finally { vi.unstubAllEnvs(); } + expect(new Set(readFileSync(seen, 'utf8').split('\n').filter(Boolean))).toEqual(new Set(['unix:///codeboost-loaded.sock|codeboost-loaded'])); +}); + +it.each([ + ['says nothing of a holder gone since the listing', 'Error response from daemon: No such container: b', []], + ['reports any other failure to read the holders', 'permission denied', [['test: reading holders: docker container inspect failed:', 'permission denied']]], +])('%s, and still removes the holders it read', (_label, stderr, warnings) => { + const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); + const calls = join(root, 'calls'), net = 'n'.repeat(64), a = 'a'.repeat(64), b = 'b'.repeat(64); + // Answers as Docker 27.4 does for one swept network with two candidate holders, one gone before the inspect: the + // inspect prints the other and exits 1. Every other call gets no output and exit 0, as for `ls` with no match. + writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${calls}'`, + `if [ "$1 $2" = "network ls" ]; then printf '%s\\n' '{"ID":"${net}","Name":"swept"}'; exit 0; fi`, + `if [ "$1" = ps ]; then printf '%s\\n' ${a} ${b}; exit 0; fi`, + `if [ "$1 $2" = "container inspect" ]; then printf '%s\\n' '"${a}" {"swept":{}}'; echo '${stderr}' >&2; exit 1; fi`, + 'exit 0'].join('\n'), + { mode: 0o755 }); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test', join(root, 'docker')); + expect(warn.mock.calls.filter(call => String(call[0]).endsWith('failed:'))).toEqual(warnings); + expect(readFileSync(calls, 'utf8').split('\n')).toContain(`rm --force ${a}`); +}); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index 2a5265b4..2edff36c 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -78,9 +78,12 @@ export function removeRunnerObjects(runnerOwner: string, when = 'afterAll'): voi * default (`REAL_DOCKER`). */ export function removeLabelledObjects(label: string, value: string, when: string, docker = REAL_DOCKER): void { - const run = (args: string[]) => { + const run = (stage: string, args: string[]) => { const result = spawnSync(docker, args, { encoding: 'utf8', timeout: 30_000, killSignal: 'SIGKILL', env: dockerEnv() }); - if (result.status !== 0) console.warn(`${when}: docker ${args.slice(0, 2).join(' ')} failed:`, + // A holder gone since the listing is expected: Docker 27.4 answers "No such container" for it, and prints the rest. + const holdersGone = stage === 'reading holders' && result.status === 1 + && result.stderr.trim().split('\n').every(line => line.startsWith('Error response from daemon: No such container: ')); + if (result.status !== 0 && !holdersGone) console.warn(`${when}: ${stage}: docker ${args.slice(0, 2).join(' ')} failed:`, result.error?.message ?? (result.signal ? `killed by ${result.signal}` : result.stderr.trim())); // An inspect of several objects exits 1 if one is gone, yet prints the rest (Docker 27.4), so the output of a call // that exited by itself is read whatever its status. A killed call's output may be cut short, so it is not. @@ -94,7 +97,7 @@ export function removeLabelledObjects(label: string, value: string, when: string // Networks with their names: `ps --filter network=` finds a created, never started container by the name only // (Docker 27.4). Each is removed by its full ID, which no other network's name can be mistaken for. // As JSON: a network's name may hold spaces. Only a container's name comes with a leading slash. - const listed = run([kind, 'ls', ...(kind === 'container' ? ['--all'] : []), ...(kind === 'network' + const listed = run(`listing ${kind}s`, [kind, 'ls', ...(kind === 'container' ? ['--all'] : []), ...(kind === 'network' ? ['--no-trunc', '--format', '{{json .}}'] : kind === 'container' ? ['--quiet', '--no-trunc'] : ['--quiet']), '--filter', `label=${label}=${value}`]) .flatMap(line => kind === 'network' ? parsed<{ ID?: unknown; Name?: unknown } | null>(line) @@ -107,18 +110,18 @@ export function removeLabelledObjects(label: string, value: string, when: string // Filters with one key match any of their values. The network filter also matches the start of another network's // ID, so a container counts only if it is on one of these networks by name (Docker refuses a second local network // of one name). Containers go by full ID: Docker resolves a name before an ID prefix. - const candidates = run(['ps', '--all', '--quiet', '--no-trunc', + const candidates = run('finding holders', ['ps', '--all', '--quiet', '--no-trunc', ...names.flatMap(name => ['--filter', `${kind}=${name}`])]); - const holders = kind === 'volume' || !candidates.length ? candidates : run(['container', 'inspect', '--format', + const holders = kind === 'volume' || !candidates.length ? candidates : run('reading holders', ['container', 'inspect', '--format', '{{json .Id}} {{json .NetworkSettings.Networks}}', ...candidates]).flatMap(line => { const at = line.indexOf(' '), id = parsed(line.slice(0, at)), on = parsed>(line.slice(at + 1)); return typeof id[0] === 'string' && typeof on[0] === 'object' ? [{ id: id[0], on: Object.keys(on[0] ?? {}) }] : []; }).filter(holder => holder.on.some(name => names.includes(name))).map(holder => holder.id); if (holders.length) { console.warn(`${when} removes the containers that hold them: ${holders.join(' ')}`); - run(['rm', '--force', ...holders]); + run('removing holders', ['rm', '--force', ...holders]); } } - run([kind, 'rm', ...(kind === 'network' ? [] : ['--force']), ...ids]); + run(`removing ${kind}s`, [kind, 'rm', ...(kind === 'network' ? [] : ['--force']), ...ids]); } } From e4e048d15b3895cb11cc2b0e6420bc8cb7ac9087 Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 23:48:06 -0700 Subject: [PATCH 39/75] Track the never-started unlabelled agent container by name; comment fixes (#189) Review round 38: Docker 27.4 removes a network a created, never started container still names, so that test's container is tracked for afterAll; the docs say which holder reads stay quiet, that the sweep reads stderr for them, and that a wrapper can stay on PATH for the rest of the file; the loaded-daemon fake says what it answers. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 13 +++++++++---- test/docker-sweep.test.ts | 6 ++++-- test/fixtures/docker-sweep.ts | 6 +++--- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 231eaf9a..b902c3fb 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -50,7 +50,8 @@ const containers = new Set(); /** * Put, beside its test-run ID, on every object the `withoutRunnerLabel` wrapper makes and on the holder test's swept * objects (#189): afterAll sweeps it. Other label-less objects are tracked by name (`containers`, `networks`, - * `volumes`), or found as holders of this run's networks and volumes. + * `volumes`), or found as holders of this run's volumes or of networks that afterAll still finds. (Docker 27.4 removes a + * network that a created, never started container still names, so such a container is tracked by name.) */ const TEST_FILE_RUN = randomUUID(); /** Networks a test makes without this run's runner label, removed by name in afterAll as `volumes` are. */ @@ -704,9 +705,10 @@ describe('real Docker agent isolation', () => { // holds one of this run's volumes or networks, and such a volume or network never, so the wrapper labels it with a // test-run ID and this file's run ID instead (#189). Whatever has that ID is removed when the test finishes (Vitest // runs `onTestFinished` after a pass, a failure or a timeout; the `it.fails` pair below tests the failure), and again - // in afterAll, for a create that lands after a timed-out test's removal (its body runs on). A timed-out test leaves the - // wrapper on PATH while its body runs on, so a later test's matching create is unlabelled too and that test fails; - // what it makes carries this file's run ID, so afterAll removes it. + // in afterAll, for a create that lands after a timed-out test's removal (its body runs on). A timed-out test can leave + // the wrapper on PATH for the rest of the file (a later shim saves the PATH that holds it, and puts it back), so a + // later test's matching create is unlabelled too and that test fails; what it makes carries this file's run ID, so + // afterAll removes it. const withoutRunnerLabel = async (prefixes: string[][], run: () => Promise | T): Promise => { if (!prefixes.every(prefix => prefix.includes('create'))) throw new Error('Only create commands can be unlabelled.'); const testRun = randomUUID(); @@ -970,6 +972,9 @@ describe('real Docker agent isolation', () => { it('never removes an agent container found by name after a killed create when a label is missing', async () => { const data = fixture(), live = await profile(data, 'planning', 'must-not-run'); + // Tracked for afterAll should the test time out: never started and without the runner label, it is no holder a + // sweep finds once its network is gone (#189). + containers.add(live.name); // The create lands without the runner label, and its client is killed before it reports the ID. Only this // container's: should the test time out with the shim still on PATH, other tests' creates pass through. const unlabelled = [ diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index 0672e926..40332f07 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -8,8 +8,9 @@ import type * as Sweep from './fixtures/docker-sweep.ts'; // The test sweep's and `removeByName`'s handling of answers they cannot use (#189), each test with its own fake // `docker`, described beside it. The sweep reads only stdout, the exit status, whether a signal killed the call and -// whether it could start, and stderr only to report a failure; `removeByName` also reads an inspect's stderr to tell a -// missing object from a failure. Against the daemon, agent-container's holder test covers the sweep and removeByName's +// whether it could start, and stderr to report a failure, and, for its read of holders, to tell one gone since the +// listing ("No such container") from a failure; `removeByName` also reads an inspect's stderr to tell a missing object +// from a failure. Against the daemon, agent-container's holder test covers the sweep and removeByName's // removal, its silence on a missing name, and its refusal of an object found by an ID prefix. const roots: string[] = []; afterEach(() => { @@ -119,6 +120,7 @@ it('talks to the daemon set when it loaded, though a test changed DOCKER_HOST or it('takes the daemon set at load, not the default, when one was set', async () => { const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); const seen = join(root, 'seen'); + // Records the address each call is given, and answers as Docker 27.4 does for a missing container and an empty `ls`. writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s|%s\\n' "\${DOCKER_HOST-unset}" "\${DOCKER_CONTEXT-unset}" >> '${seen}'`, 'if [ "$2" = inspect ]; then echo; echo "Error response from daemon: No such container: x" >&2; exit 1; fi', 'exit 0'].join('\n'), { mode: 0o755 }); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index 2edff36c..4581e9a1 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -73,9 +73,9 @@ export function removeRunnerObjects(runnerOwner: string, when = 'afterAll'): voi * use. Before it removes a network or a volume it also removes every container attached to it or mounting it, labelled * or not: a test can make a network or volume with one owner and the container that holds it with another, or without * one. Containers and volumes go by force; a network another container joins meanwhile stays. It reports what it finds - * and any Docker call that fails, prefixed with `when`, and never throws, so cleanup can call it without hiding an - * earlier error. Each call has a 30 s limit, so all eleven can take up to 5.5 minutes. It runs `docker`, the real one by - * default (`REAL_DOCKER`). + * and any Docker call that fails (but not a read of holders that fails only because one has gone since the listing), + * prefixed with `when`, and never throws, so cleanup can call it without hiding an earlier error. Each call has a 30 s + * limit, so all eleven can take up to 5.5 minutes. It runs `docker`, the real one by default (`REAL_DOCKER`). */ export function removeLabelledObjects(label: string, value: string, when: string, docker = REAL_DOCKER): void { const run = (stage: string, args: string[]) => { From ab215c685b784c8d6ce6225f30a230cb7b669478 Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 20 Oct 2026 23:57:58 -0700 Subject: [PATCH 40/75] Label agent-network's unnamed helper containers for afterAll; track the rogue endpoint (#189) Review round 39: a helper run killed between create and start is never removed by --rm, and is no holder once its vendor network is gone, so the curl, DNS and concurrent-network helpers carry this file's run ID and afterAll sweeps it (13-minute hook for two sweeps); agent-container's rogue endpoint is tracked by name; the holder fake says the removals' stdout is not read. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 2 ++ test/agent-network.test.ts | 25 ++++++++++++++++--------- test/docker-sweep.test.ts | 3 ++- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index b902c3fb..503d62f3 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -2787,6 +2787,8 @@ describe('real Docker agent isolation', () => { it('rejects a new endpoint attached to the invocation network before launch', async () => { const data = fixture(), valid = await profile(data, 'planning', 'must-not-run'); const rogue = `codeboost-rogue-${randomUUID()}`; + // Tracked for afterAll: its create may land after the finally, and once the network is gone it is no holder (#189). + containers.add(rogue); try { docker('run', '--detach', '--name', rogue, `--network=${valid.network.name}`, '--entrypoint', 'node', imageId, '-e', 'setInterval(()=>{},1000)'); diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index 982c2a99..b8969ea1 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -9,16 +9,22 @@ import { assertVendorNetwork, createVendorNetwork, removeVendorNetwork, VENDOR_H VendorNetworkCreationCleanupError, type VendorNetwork } from '../agents/network/network.ts'; import { captureInvocation, type InvocationInput } from '../agents/contract.ts'; -import { removeByName, removeRunnerObjects, requireRealDocker } from './fixtures/docker-sweep.ts'; +import { removeByName, removeLabelledObjects, removeRunnerObjects, requireRealDocker, TEST_FILE_LABEL } from './fixtures/docker-sweep.ts'; // Each run has its own runner owner, so the sweep after this file removes only what this run left behind. Network and // proxy names are random, so the fixed attempt IDs below reach only labels, where this owner tells runs apart. const TEST_RUNNER_OWNER = randomBytes(16).toString('hex'); +/** + * On the unnamed `run --rm` containers that join a vendor network (#189): a client killed after the create but before + * the start leaves a container `--rm` never removes, which is no holder once its network is gone (Docker 27.4 removes + * a network a created, never started container still names). afterAll sweeps this label. + */ +const TEST_FILE_RUN = randomUUID(), fileLabel = ['--label', `${TEST_FILE_LABEL}=${TEST_FILE_RUN}`]; let imageId = '', network: VendorNetwork, invocation: InvocationInput; const docker = (...args: string[]) => execFileSync('docker', args, { encoding: 'utf8', timeout: 60_000, stdio: ['ignore', 'pipe', 'pipe'], }).trim(); -const curl = (url: string, direct = false) => spawnSync('docker', ['run', '--rm', `--network=${network.name}`, +const curl = (url: string, direct = false) => spawnSync('docker', ['run', '--rm', ...fileLabel, `--network=${network.name}`, '--env', `HTTPS_PROXY=${network.proxyUrl}`, ...(direct ? ['--env', 'NO_PROXY=*'] : []), '--entrypoint', 'curl', imageId, '--silent', '--show-error', '--output', '/dev/null', '--write-out', '%{http_code}', '--max-time', '15', url], { encoding: 'utf8', timeout: 30_000, stdio: ['ignore', 'pipe', 'pipe'] }); @@ -54,14 +60,15 @@ afterAll(async () => { } finally { // Then remove whatever this run still owns, even when a test failed or the removal above threw. Objects a test - // makes without the runner label are found here only while they hold one of this run's networks or volumes; the - // tests remove their own, and the try above removes those registered in `unlabelled`. It may not run before a hook - // timeout. The hook's 7 minutes leave a minute and a half over this sweep's worst case (eleven calls of 30 s), for - // the cleanup before it. + // makes without the runner label are found here only while they hold one of this run's networks or volumes, or + // carry this file's run ID; the tests remove their own, and the try above removes those registered in + // `unlabelled`. It may not run before a hook timeout. The hook's 13 minutes leave two over the worst case of these + // two sweeps (eleven calls of 30 s each), for the cleanup before them. + removeLabelledObjects(TEST_FILE_LABEL, TEST_FILE_RUN, 'afterAll'); removeRunnerObjects(TEST_RUNNER_OWNER); if (records) rmSync(records, { recursive: true, force: true }); } -}, 7 * 60_000); +}, 13 * 60_000); describe('vendor-only egress', () => { it('keeps failed allocation and its cleanup inside the caller deadline', async () => { @@ -303,7 +310,7 @@ describe('vendor-only egress', () => { }, 60_000); it('does not forward arbitrary DNS even when the embedded resolver is addressed directly', async () => { - const result = spawnSync('docker', ['run', '--rm', `--network=${network.name}`, '--dns=127.0.0.1', + const result = spawnSync('docker', ['run', '--rm', ...fileLabel, `--network=${network.name}`, '--dns=127.0.0.1', '--entrypoint', 'node', imageId, '-e', [ "const dns=require('node:dns');dns.setServers(['127.0.0.11']);", "dns.resolve4('example.com',(error)=>process.exit(error?0:1));", @@ -328,7 +335,7 @@ describe('vendor-only egress', () => { try { docker('run', '--detach', '--name', peer, `--network=${network.name}`, '--network-alias', 'codeboost-peer', '--entrypoint', 'node', imageId, '-e', "require('node:net').createServer(()=>{}).listen(4567,'0.0.0.0');setInterval(()=>{},1000)"); - const result = spawnSync('docker', ['run', '--rm', `--network=${other.name}`, '--entrypoint', 'node', imageId, + const result = spawnSync('docker', ['run', '--rm', ...fileLabel, `--network=${other.name}`, '--entrypoint', 'node', imageId, '-e', "const s=require('node:net').connect(4567,'codeboost-peer');s.on('connect',()=>process.exit(0));s.on('error',()=>process.exit(1));setTimeout(()=>process.exit(2),3000)"], { encoding: 'utf8', timeout: 10_000, stdio: ['ignore', 'pipe', 'pipe'] }); expect(result.status).not.toBe(0); diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index 40332f07..5cee261e 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -145,7 +145,8 @@ it.each([ const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); const calls = join(root, 'calls'), net = 'n'.repeat(64), a = 'a'.repeat(64), b = 'b'.repeat(64); // Answers as Docker 27.4 does for one swept network with two candidate holders, one gone before the inspect: the - // inspect prints the other and exits 1. Every other call gets no output and exit 0, as for `ls` with no match. + // inspect prints the other and exits 1. Every other call gets no output and exit 0, as for `ls` with no match; the + // removals' stdout (the removed IDs or names in 27.4) is not read. writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${calls}'`, `if [ "$1 $2" = "network ls" ]; then printf '%s\\n' '{"ID":"${net}","Name":"swept"}'; exit 0; fi`, `if [ "$1" = ps ]; then printf '%s\\n' ${a} ${b}; exit 0; fi`, From 29744c791a6b64decbb55d95d5c5d9f85af72f5a Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 00:08:28 -0700 Subject: [PATCH 41/75] Track the peer, stand-in proxy and foreign keeper by name; holder test asserts the quiet sweep on the daemon (#189) Review round 40: three label-less or foreign containers made with a 60 s client could land after their test's finally, and once their network is gone none is a holder, so each is tracked by name for afterAll; the holder test checks the daemon's own answer for a gone container and that the sweep reports no failure for it; the holder fake says what it leaves out; the unit file unstubs env after each test; agent-network removes its records before the sweeps. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 8 +++++++- test/agent-network.test.ts | 7 ++++++- test/docker-sweep.test.ts | 5 ++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 503d62f3..097aab09 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -821,6 +821,7 @@ describe('real Docker agent isolation', () => { { encoding: 'utf8' }); expect(partial.status).not.toBe(0); expect(partial.stdout).toContain(holders[0]); + expect(partial.stderr).toMatch(/^Error response from daemon: No such container: codeboost-gone-/m); // The holders' inspect also names a container that does not exist, as one gone since the listing would be: Docker // 27.4 prints the others and exits 1, and the sweep must still remove them. const shim = mkdtempSync(join(tmpdir(), 'codeboost-sweep-docker-')); roots.push(shim); @@ -829,7 +830,10 @@ describe('real Docker agent isolation', () => { writeFileSync(join(shim, 'docker'), ['#!/bin/sh', `if [ "$1 $2" = "container inspect" ]; then exec '${realDocker}' "$@" codeboost-gone-${randomUUID()}; fi`, `exec '${realDocker}' "$@"`].join('\n'), { mode: 0o755 }); - removeLabelledObjects(TEST_RUN_LABEL, run, 'test', join(shim, 'docker')); + // And it says nothing of that gone holder: Docker's own answer is not a failure. + const sweepWarn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + try { removeLabelledObjects(TEST_RUN_LABEL, run, 'test', join(shim, 'docker')); } finally { sweepWarn.mockRestore(); } + expect(sweepWarn.mock.calls.filter(call => String(call[0]).endsWith('failed:'))).toEqual([]); // afterAll removes tracked names with removeByName, as here: an object of exactly that name goes, and a name that // is gone says nothing. const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); @@ -2549,6 +2553,8 @@ describe('real Docker agent isolation', () => { const data = fixture(), filesystems = data.filesystems, owner = taskFilesystemOwner(filesystems); taskFilesystems.splice(taskFilesystems.indexOf(filesystems), 1); docker('rm', '--force', filesystems.keeper); + // Tracked for afterAll too: its create may land after the finally (#189). + containers.add(filesystems.keeper); try { // Inside the try: this keeper carries another runner's label, so only the finally below removes it (#189). docker('run', '--detach', '--name', filesystems.keeper, '--label', `io.codeboost.runner=${'f'.repeat(32)}`, diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index b8969ea1..aeae2f55 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -64,9 +64,10 @@ afterAll(async () => { // carry this file's run ID; the tests remove their own, and the try above removes those registered in // `unlabelled`. It may not run before a hook timeout. The hook's 13 minutes leave two over the worst case of these // two sweeps (eleven calls of 30 s each), for the cleanup before them. + // The records were read above; removed first, so a hook timeout during the sweeps does not leave them. + if (records) rmSync(records, { recursive: true, force: true }); removeLabelledObjects(TEST_FILE_LABEL, TEST_FILE_RUN, 'afterAll'); removeRunnerObjects(TEST_RUNNER_OWNER); - if (records) rmSync(records, { recursive: true, force: true }); } }, 13 * 60_000); @@ -332,6 +333,8 @@ describe('vendor-only egress', () => { const otherInvocation = captureInvocation({ ...invocation, attemptId: 'concurrent-network-probe', deadline: Date.now() + 60_000 }); const other = await createVendorNetwork(otherInvocation, imageId, randomUUID()), peer = `codeboost-peer-${randomUUID()}`; + // Tracked for afterAll: its create may land after the finally, and once the network is gone it is no holder (#189). + unlabelled.push({ kind: 'container', name: peer }); try { docker('run', '--detach', '--name', peer, `--network=${network.name}`, '--network-alias', 'codeboost-peer', '--entrypoint', 'node', imageId, '-e', "require('node:net').createServer(()=>{}).listen(4567,'0.0.0.0');setInterval(()=>{},1000)"); @@ -366,6 +369,8 @@ describe('vendor-only egress', () => { { Config: { Labels: Record }; NetworkSettings: { Networks: Record } }; const allocation = inspected.Config.Labels['io.codeboost.egress']; const proxyIp = inspected.NetworkSettings.Networks[replacement.name]!.IPAddress; + // Tracked for afterAll: the stand-in has no runner label, and its create may land after the finally (#189). + unlabelled.push({ kind: 'container', name: replacement.proxyContainer }); try { docker('rm', '--force', replacement.proxyContainer); // Same name, label, image, lockdown and IP as the original, so only the extra option and the object ID differ. diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index 5cee261e..f0e4aecc 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -16,6 +16,7 @@ const roots: string[] = []; afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); vi.restoreAllMocks(); + vi.unstubAllEnvs(); }); // The fake answers as no real daemon does for `network ls`: a line that is not JSON, a JSON null and one network @@ -146,7 +147,9 @@ it.each([ const calls = join(root, 'calls'), net = 'n'.repeat(64), a = 'a'.repeat(64), b = 'b'.repeat(64); // Answers as Docker 27.4 does for one swept network with two candidate holders, one gone before the inspect: the // inspect prints the other and exits 1. Every other call gets no output and exit 0, as for `ls` with no match; the - // removals' stdout (the removed IDs or names in 27.4) is not read. + // removals' stdout (the removed IDs or names in 27.4) is not read. The network line holds only the fields the sweep + // reads (27.4 also prints CreatedAt, Driver, IPv6, Internal, Labels and Scope), and the inspect an empty endpoint for + // the network, where 27.4 gives its full settings: the sweep reads only the network names. writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${calls}'`, `if [ "$1 $2" = "network ls" ]; then printf '%s\\n' '{"ID":"${net}","Name":"swept"}'; exit 0; fi`, `if [ "$1" = ps ]; then printf '%s\\n' ${a} ${b}; exit 0; fi`, From 3d862913608a76ac68122eb821dcca2ff60a421e Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 00:18:38 -0700 Subject: [PATCH 42/75] Track the rogue-volume container before its create (#189) Review round 41: should the create land after its client is killed, afterAll removes the container by name before the label-less volume it mounts. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 097aab09..da39f3bc 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -484,7 +484,8 @@ describe('real Docker agent isolation', () => { volumes.add(rogue); docker('volume', 'create', rogue); const rogueArgs = valid.args.map(value => value.replace(data.filesystems.workVolume, rogue)); - docker(...rogueArgs); containers.add(valid.name); + // Tracked before the create: should it land late, afterAll removes it before the volume it mounts. + containers.add(valid.name); docker(...rogueArgs); await expect(validateContainer(valid.name, valid)).rejects.toThrow('captured identity'); docker('rm', '--force', valid.name); containers.delete(valid.name); } finally { From 4ead18db4c977bea78e35809de5148cb3c7babc5 Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 00:26:59 -0700 Subject: [PATCH 43/75] Read the holder test's sweep warnings before restoring the spy (#189) Review round 42: mockRestore clears the recorded calls, so the check that the sweep says nothing of a gone holder read an empty list and could not fail; the calls are copied first, and the test also checks the spy saw the sweep's own lines. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index da39f3bc..d090caed 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -833,8 +833,14 @@ describe('real Docker agent isolation', () => { `exec '${realDocker}' "$@"`].join('\n'), { mode: 0o755 }); // And it says nothing of that gone holder: Docker's own answer is not a failure. const sweepWarn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); - try { removeLabelledObjects(TEST_RUN_LABEL, run, 'test', join(shim, 'docker')); } finally { sweepWarn.mockRestore(); } - expect(sweepWarn.mock.calls.filter(call => String(call[0]).endsWith('failed:'))).toEqual([]); + // Copied before the restore, which clears the calls. + let sweepCalls: unknown[][] = []; + try { + removeLabelledObjects(TEST_RUN_LABEL, run, 'test', join(shim, 'docker')); + sweepCalls = [...sweepWarn.mock.calls]; + } finally { sweepWarn.mockRestore(); } + expect(sweepCalls.filter(call => String(call[0]).endsWith('failed:'))).toEqual([]); + expect(sweepCalls.length).toBeGreaterThan(0); // afterAll removes tracked names with removeByName, as here: an object of exactly that name goes, and a name that // is gone says nothing. const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); From 698f341f6b95f8c23a154f9395ed29d1b3e1fea6 Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 00:42:34 -0700 Subject: [PATCH 44/75] Storage release after a pass fails the test again; mark the invented fake answer (#189) Review round 43: in the two tests whose cleanup now warns instead of throwing, a release that fails after the test's own checks passed fails the test, as on main, while a failure above still stays the one reported; the second holder fake case says its answer is invented. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 21 +++++++++++++++------ test/docker-sweep.test.ts | 3 ++- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index d090caed..1d600e4d 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -2562,6 +2562,7 @@ describe('real Docker agent isolation', () => { docker('rm', '--force', filesystems.keeper); // Tracked for afterAll too: its create may land after the finally (#189). containers.add(filesystems.keeper); + let passed = false; try { // Inside the try: this keeper carries another runner's label, so only the finally below removes it (#189). docker('run', '--detach', '--name', filesystems.keeper, '--label', `io.codeboost.runner=${'f'.repeat(32)}`, @@ -2570,13 +2571,16 @@ describe('real Docker agent isolation', () => { expect(() => removeTaskFilesystems(filesystems)).toThrow('did not settle'); expect(docker('container', 'inspect', '--format', '{{.State.Running}}', filesystems.keeper)).toBe('true'); expect(spawnSync('docker', ['volume', 'inspect', filesystems.workVolume]).status).not.toBe(0); + passed = true; } finally { spawnSync('docker', ['rm', '--force', filesystems.keeper], { stdio: 'ignore' }); - // Never throws, so a failure above is the one reported: what the removal leaves, the allocation sweep below removes, - // the keeper with another runner's label included (#189). The sweep may take up to 5.5 minutes, which the test's - // budget allows for. - try { removeTaskFilesystems(filesystems); } catch (error) { console.warn('cleanup: removing the storage failed:', error); } + // A failure above stays the one reported: what the removal leaves, the allocation sweep below removes, the keeper + // with another runner's label included (#189). After a pass, the storage must release once the foreign keeper is + // gone, so its failure fails the test. The sweep may take up to 5.5 minutes, which the test's budget allows for. + let released: unknown; + try { removeTaskFilesystems(filesystems); } catch (error) { released = error; console.warn('cleanup: removing the storage failed:', error); } removeLabelledObjects('io.codeboost.allocation', owner.allocationId, 'test'); + if (passed && released) throw released; } }, 7 * 60_000); @@ -2586,15 +2590,20 @@ describe('real Docker agent isolation', () => { // It carries another runner's label, so no sweep of this run finds it: removed here, and whatever still carries its // allocation when the test finishes, should its creation or removal not settle or the test time out (#189). sweepWhenFinished('io.codeboost.allocation', allocationId); + let passed = false; try { foreign = prepareTaskFilesystems(data.clone, { workBytes: 16 * 1024 * 1024, workInodes: 512, metadataBytes: 16 * 1024 * 1024, metadataInodes: 512 }, imageId, { runnerOwner: 'f'.repeat(32), attemptId: 'other-runner', allocationId }); await expect(createContainerProfile({ ...await governed(invocation(data.clone, 'planning')), filesystems: foreign, inputDirectory: data.input, codexAuthFile: data.fakeAuth, imageId })).rejects.toThrow('another runner'); + passed = true; } finally { - // Never throws, so a failure above is the one reported; the sweep removes whatever the removal left. - try { if (foreign) removeTaskFilesystems(foreign); } catch (error) { console.warn('cleanup: removing the storage failed:', error); } + // A failure above stays the one reported, and the sweep removes whatever the removal left. After a pass, the + // storage must release by its handle, so its failure fails the test. + let released: unknown; + try { if (foreign) removeTaskFilesystems(foreign); } catch (error) { released = error; console.warn('cleanup: removing the storage failed:', error); } + if (passed && released) throw released; } }, 60_000); diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index f0e4aecc..c9420c1e 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -147,7 +147,8 @@ it.each([ const calls = join(root, 'calls'), net = 'n'.repeat(64), a = 'a'.repeat(64), b = 'b'.repeat(64); // Answers as Docker 27.4 does for one swept network with two candidate holders, one gone before the inspect: the // inspect prints the other and exits 1. Every other call gets no output and exit 0, as for `ls` with no match; the - // removals' stdout (the removed IDs or names in 27.4) is not read. The network line holds only the fields the sweep + // removals' stdout (the removed IDs or names in 27.4) is not read. The second case's answer is invented: it stands in + // for an unexpected inspect failure that still printed some holders, which 27.4 is not known to give. The network line holds only the fields the sweep // reads (27.4 also prints CreatedAt, Driver, IPv6, Internal, Labels and Scope), and the inspect an empty endpoint for // the network, where 27.4 gives its full settings: the sweep reads only the network names. writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${calls}'`, From bcf1e9fb9933396d89901f1012696a20cc2ee7d7 Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 00:55:03 -0700 Subject: [PATCH 45/75] removeByName: tested on unreadable output, quiet on a network gone before its removal (#189) Review round 44: the fake gives an inspect answer that is not JSON (reported, nothing removed, never thrown) and a network gone between inspect and removal, which Docker 27.4 answers 'network not found' and removeByName now treats as gone; the keeper comment says afterAll also removes it by name; a comment line is rewrapped. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 3 ++- test/docker-sweep.test.ts | 23 +++++++++++++++-------- test/fixtures/docker-sweep.ts | 5 ++++- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 1d600e4d..fce7d9be 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -2564,7 +2564,8 @@ describe('real Docker agent isolation', () => { containers.add(filesystems.keeper); let passed = false; try { - // Inside the try: this keeper carries another runner's label, so only the finally below removes it (#189). + // Inside the try: this keeper carries another runner's label, so no sweep of this run finds it; the finally below + // removes it, and afterAll by name (#189). docker('run', '--detach', '--name', filesystems.keeper, '--label', `io.codeboost.runner=${'f'.repeat(32)}`, '--label', `io.codeboost.attempt=${owner.attemptId}`, '--label', `io.codeboost.allocation=${owner.allocationId}`, '--network=none', '--entrypoint', 'sleep', imageId, 'infinity'); diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index c9420c1e..535e3dda 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -55,12 +55,14 @@ it('reads nothing from a docker call that was killed, whose output may be cut sh it('removes by name only an object of exactly that name, and reports failures other than no such object', () => { const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); - const calls = join(root, 'calls'), id = 'b'.repeat(64), other = 'c'.repeat(64); + const calls = join(root, 'calls'), id = 'b'.repeat(64), other = 'c'.repeat(64), raced = 'd'.repeat(64); // Answers as Docker 27.4 does for `inspect --format '{{json .Id}} {{json .Name}}'` (a volume: '{{json .Name}}') and // `rm`, in the parts removeByName reads: stdout of an inspect that succeeded, stderr and the exit status. A missing // object, an object found under another name (by an ID prefix), a refused removal, an inspect killed by a signal, and a - // failure before the daemon (an unknown context; its meta.json path shortened). The stdout of a failed call (a bare - // newline for a missing object in 27.4, nothing for the others) is not read. + // failure before the daemon (an unknown context; its meta.json path shortened). Also a network gone between its + // inspect and its removal, which 27.4 answers "network not found", and an inspect answer that is not JSON, which + // 27.4 is not known to give. The stdout of a failed call (a bare newline for a missing object in 27.4, nothing for + // the others) is not read. writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${calls}'`, 'for arg; do name="$arg"; done', 'if [ "$2" = inspect ]; then case "$1 $name" in', @@ -72,21 +74,25 @@ it('removes by name only an object of exactly that name, and reports failures ot ` 'container busy') printf '%s\\n' '"${id}" "/busy"';;`, ` 'network busy') printf '%s\\n' '"${other}" "busy"';;`, ` 'volume busy') printf '%s\\n' '"busy"';;`, + ` 'network raced') printf '%s\\n' '"${raced}" "raced"';;`, + " 'container junk') echo 'not json';;", " 'container killed') kill -9 $$;;", " 'container lost') echo 'Failed to initialize: unable to resolve docker endpoint: context \"x\": context not found: open /x/meta.json: no such file or directory' >&2; exit 1;;", 'esac; exit 0; fi', `if [ "$1" = rm ]; then echo 'Error response from daemon: removal of container ${id} is already in progress' >&2; exit 1; fi`, + `if [ "$1 $2 $3" = 'network rm ${raced}' ]; then echo 'Error response from daemon: network ${raced} not found' >&2; exit 1; fi`, `if [ "$1 $2" = 'network rm' ]; then echo 'Error response from daemon: error while removing network: network busy id ${other} has active endpoints' >&2; exit 1; fi`, `if [ "$1 $2" = 'volume rm' ]; then echo 'Error response from daemon: remove busy: volume is in use - [${id}]' >&2; exit 1; fi`, 'exit 0'].join('\n'), { mode: 0o755 }); const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); const objects = [['container', 'gone'], ['network', 'gone'], ['volume', 'gone'], ['container', 'prefix'], ['network', 'prefix'], - ['container', 'busy'], ['network', 'busy'], ['volume', 'busy'], ['container', 'killed'], ['container', 'lost']] as const; - for (const [kind, name] of objects) removeByName(kind, name, 'test', join(root, 'docker')); + ['container', 'busy'], ['network', 'busy'], ['volume', 'busy'], ['container', 'killed'], ['container', 'lost'], + ['network', 'raced'], ['container', 'junk']] as const; + expect(() => { for (const [kind, name] of objects) removeByName(kind, name, 'test', join(root, 'docker')); }).not.toThrow(); // Only the exact names are removed, by full ID (a volume's name is its ID). expect(readFileSync(calls, 'utf8').split('\n').filter(line => / rm |^rm /.test(line))) - .toEqual([`rm --force ${id}`, `network rm ${other}`, 'volume rm --force busy']); + .toEqual([`rm --force ${id}`, `network rm ${other}`, 'volume rm --force busy', `network rm ${raced}`]); expect(warn.mock.calls).toEqual([ ['test: removing container busy: docker rm failed:', `Error response from daemon: removal of container ${id} is already in progress`], @@ -96,6 +102,7 @@ it('removes by name only an object of exactly that name, and reports failures ot ['test: removing container killed: docker container inspect failed:', 'killed by SIGKILL'], ['test: removing container lost: docker container inspect failed:', 'Failed to initialize: unable to resolve docker endpoint: context "x": context not found: open /x/meta.json: no such file or directory'], + ['test: removing container junk: docker printed what is not JSON:', 'not json\n'], ]); }); @@ -148,8 +155,8 @@ it.each([ // Answers as Docker 27.4 does for one swept network with two candidate holders, one gone before the inspect: the // inspect prints the other and exits 1. Every other call gets no output and exit 0, as for `ls` with no match; the // removals' stdout (the removed IDs or names in 27.4) is not read. The second case's answer is invented: it stands in - // for an unexpected inspect failure that still printed some holders, which 27.4 is not known to give. The network line holds only the fields the sweep - // reads (27.4 also prints CreatedAt, Driver, IPv6, Internal, Labels and Scope), and the inspect an empty endpoint for + // for an unexpected inspect failure that still printed some holders, which 27.4 is not known to give. The network + // line holds only the fields the sweep reads (27.4 also prints CreatedAt, Driver, IPv6, Internal, Labels and Scope), and the inspect an empty endpoint for // the network, where 27.4 gives its full settings: the sweep reads only the network names. writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${calls}'`, `if [ "$1 $2" = "network ls" ]; then printf '%s\\n' '{"ID":"${net}","Name":"swept"}'; exit 0; fi`, diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index 4581e9a1..f3058183 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -55,7 +55,10 @@ export function removeByName(kind: 'container' | 'network' | 'volume', name: str } catch { console.warn(`${when}: removing ${kind} ${name}: docker printed what is not JSON:`, found.stdout); return; } if (typeof id !== 'string' || typeof actual !== 'string' || (kind === 'container' ? actual.replace(/^\//, '') : actual) !== name) return; const removed = spawnSync(docker, kind === 'container' ? ['rm', '--force', id] : [kind, 'rm', ...(kind === 'volume' ? ['--force'] : []), id], options); - if (removed.error || removed.status !== 0) failed(removed, kind === 'container' ? 'rm' : `${kind} rm`); + // A network gone since the inspect is gone: Docker 27.4 answers "network not found" (rm --force of a missing + // container or volume exits 0). + const raced = kind === 'network' && removed.status === 1 && removed.stderr.includes(`network ${id} not found`); + if ((removed.error || removed.status !== 0) && !raced) failed(removed, kind === 'container' ? 'rm' : `${kind} rm`); } /** From 6fe98b943990a8e55e03a9a2d2320f166a21741c Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 01:06:42 -0700 Subject: [PATCH 46/75] Restore two failures main's cleanups had; report an inspect answer with no ID and name (#189) Review round 45: the foreign-name supervisor test fails again when, after its checks pass, the container's removal fails; the unsettled-allocation test checks after its sweep that nothing of the allocation is left, as main's listing did; removeByName reports a parsed answer without an ID and a name, and its doc lists what it stays quiet about; comment blocks rewrapped. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 5 +++++ test/agent-supervisor.test.ts | 8 ++++++-- test/docker-sweep.test.ts | 19 +++++++++++-------- test/fixtures/docker-sweep.ts | 10 +++++++--- 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index fce7d9be..bf686e11 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -2375,6 +2375,10 @@ describe('real Docker agent isolation', () => { const data = fixture(), runnerOwner = randomBytes(16).toString('hex'); const owner = { runnerOwner, attemptId: 'unsettled-allocation', allocationId: randomUUID() }; // Another runner's objects, so no sweep of this run finds them: removed by allocation when the test finishes (#189). + // After a pass, nothing of the allocation may be left once that sweep has run, as on main; this check, registered + // first, runs after the sweep (Vitest runs onTestFinished hooks in reverse order). + let passed = false; + onTestFinished(() => { if (passed) expect(byAllocation(owner.allocationId)).toEqual([]); }, 2 * 60_000); sweepWhenFinished('io.codeboost.allocation', owner.allocationId); // The seeder is refused, and every volume removal fails, so the allocation's cleanup does not settle. const failing = [ @@ -2385,6 +2389,7 @@ describe('real Docker agent isolation', () => { await withDockerShim([], failing, () => expect(() => prepareTaskFilesystems(data.clone, storageLimits, imageId, owner)).toThrow('cleanup did not settle')); await expect(recoverLeftovers(runnerOwner)).rejects.toThrow('still holds task storage'); + passed = true; }, 60_000); it('recovers only its own runner after a restart, keeping its storage for removal by handle', async () => { diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 4828ac58..c0d1a18d 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -389,6 +389,7 @@ describe('container invocation supervisor', () => { const current = await profile(fixture(), 'finite-output', 'foreign-name'); // Removed by name when the test finishes, timeouts included: its label value is shared across runs (#189). onTestFinished(() => removeByName('container', current.name, 'onTestFinished'), 90_000); // two calls of up to 30 s + let passed = false; try { // Inside the try, so its removal runs even if the create's client fails after the daemon made it (#189). execFileSync('docker', ['create', '--name', current.name, '--label', 'io.codeboost.invocation=foreign', @@ -402,9 +403,12 @@ describe('container invocation supervisor', () => { expect(spawnSync('docker', ['network', 'inspect', current.network.name], { stdio: 'ignore' }).status).not.toBe(0); expect(execFileSync('docker', ['container', 'inspect', '--format', '{{index .Config.Labels "io.codeboost.invocation"}}', current.name], { encoding: 'utf8' }).trim()).toBe('foreign'); + passed = true; } finally { - // Never throws, whatever exit status this Docker version gives for a missing name. - spawnSync('docker', ['rm', '--force', current.name], { stdio: 'ignore' }); + // A failure above stays the one reported, whatever exit status this Docker version gives for a missing name. After + // a pass the container exists, and its removal must succeed, as on main. + const removed = spawnSync('docker', ['rm', '--force', current.name], { encoding: 'utf8' }); + if (passed && removed.status !== 0) throw new Error(`Removing ${current.name} failed: ${removed.stderr.trim()}`); } }, 3 * 60_000); diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index 535e3dda..9607cd50 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -58,11 +58,11 @@ it('removes by name only an object of exactly that name, and reports failures ot const calls = join(root, 'calls'), id = 'b'.repeat(64), other = 'c'.repeat(64), raced = 'd'.repeat(64); // Answers as Docker 27.4 does for `inspect --format '{{json .Id}} {{json .Name}}'` (a volume: '{{json .Name}}') and // `rm`, in the parts removeByName reads: stdout of an inspect that succeeded, stderr and the exit status. A missing - // object, an object found under another name (by an ID prefix), a refused removal, an inspect killed by a signal, and a - // failure before the daemon (an unknown context; its meta.json path shortened). Also a network gone between its - // inspect and its removal, which 27.4 answers "network not found", and an inspect answer that is not JSON, which - // 27.4 is not known to give. The stdout of a failed call (a bare newline for a missing object in 27.4, nothing for - // the others) is not read. + // object, an object found under another name (by an ID prefix), a refused removal, an inspect killed by a signal, and + // a failure before the daemon (an unknown context; its meta.json path shortened). Also a network gone between its + // inspect and its removal, which 27.4 answers "network not found", and inspect answers that are not JSON or not + // an ID and a name, which 27.4 is not known to give. The stdout of a failed call (a bare newline for a missing object + // in 27.4, nothing for the others) is not read. writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${calls}'`, 'for arg; do name="$arg"; done', 'if [ "$2" = inspect ]; then case "$1 $name" in', @@ -76,6 +76,7 @@ it('removes by name only an object of exactly that name, and reports failures ot ` 'volume busy') printf '%s\\n' '"busy"';;`, ` 'network raced') printf '%s\\n' '"${raced}" "raced"';;`, " 'container junk') echo 'not json';;", + " 'container odd') echo '1 2';;", " 'container killed') kill -9 $$;;", " 'container lost') echo 'Failed to initialize: unable to resolve docker endpoint: context \"x\": context not found: open /x/meta.json: no such file or directory' >&2; exit 1;;", 'esac; exit 0; fi', @@ -88,7 +89,7 @@ it('removes by name only an object of exactly that name, and reports failures ot const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); const objects = [['container', 'gone'], ['network', 'gone'], ['volume', 'gone'], ['container', 'prefix'], ['network', 'prefix'], ['container', 'busy'], ['network', 'busy'], ['volume', 'busy'], ['container', 'killed'], ['container', 'lost'], - ['network', 'raced'], ['container', 'junk']] as const; + ['network', 'raced'], ['container', 'junk'], ['container', 'odd']] as const; expect(() => { for (const [kind, name] of objects) removeByName(kind, name, 'test', join(root, 'docker')); }).not.toThrow(); // Only the exact names are removed, by full ID (a volume's name is its ID). expect(readFileSync(calls, 'utf8').split('\n').filter(line => / rm |^rm /.test(line))) @@ -103,6 +104,7 @@ it('removes by name only an object of exactly that name, and reports failures ot ['test: removing container lost: docker container inspect failed:', 'Failed to initialize: unable to resolve docker endpoint: context "x": context not found: open /x/meta.json: no such file or directory'], ['test: removing container junk: docker printed what is not JSON:', 'not json\n'], + ['test: removing container odd: docker printed no ID and name:', '1 2\n'], ]); }); @@ -156,8 +158,9 @@ it.each([ // inspect prints the other and exits 1. Every other call gets no output and exit 0, as for `ls` with no match; the // removals' stdout (the removed IDs or names in 27.4) is not read. The second case's answer is invented: it stands in // for an unexpected inspect failure that still printed some holders, which 27.4 is not known to give. The network - // line holds only the fields the sweep reads (27.4 also prints CreatedAt, Driver, IPv6, Internal, Labels and Scope), and the inspect an empty endpoint for - // the network, where 27.4 gives its full settings: the sweep reads only the network names. + // line holds only the fields the sweep reads (27.4 also prints CreatedAt, Driver, IPv6, Internal, Labels and Scope), + // and the inspect an empty endpoint for the network, where 27.4 gives its full settings: the sweep reads only the + // network names. writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${calls}'`, `if [ "$1 $2" = "network ls" ]; then printf '%s\\n' '{"ID":"${net}","Name":"swept"}'; exit 0; fi`, `if [ "$1" = ps ]; then printf '%s\\n' ${a} ${b}; exit 0; fi`, diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index f3058183..5afc516d 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -35,8 +35,9 @@ export const TEST_FILE_LABEL = 'io.codeboost.test-file'; /** * Removes the container, network or volume named exactly `name`, by its full ID (a volume by its name, which is its * ID). Docker would read a missing name as an ID prefix, so an object found under another name is left alone. It - * reports a failed call, other than an inspect that finds no such object, prefixed with `when`, and never throws; each - * call has a 30 s limit. + * reports, prefixed with `when`, a failed call and an answer it cannot read, but not an object that is gone: an inspect + * that finds no such object, or a network gone between its inspect and its removal. It never throws; each call has a + * 30 s limit. */ export function removeByName(kind: 'container' | 'network' | 'volume', name: string, when: string, docker = REAL_DOCKER): void { const options = { encoding: 'utf8' as const, timeout: 30_000, killSignal: 'SIGKILL' as const, env: dockerEnv() }; @@ -53,7 +54,10 @@ export function removeByName(kind: 'container' | 'network' | 'volume', name: str const at = found.stdout.indexOf(' '); [id, actual] = kind === 'volume' ? [name, JSON.parse(found.stdout)] : [JSON.parse(found.stdout.slice(0, at)), JSON.parse(found.stdout.slice(at + 1))]; } catch { console.warn(`${when}: removing ${kind} ${name}: docker printed what is not JSON:`, found.stdout); return; } - if (typeof id !== 'string' || typeof actual !== 'string' || (kind === 'container' ? actual.replace(/^\//, '') : actual) !== name) return; + if (typeof id !== 'string' || typeof actual !== 'string') { + console.warn(`${when}: removing ${kind} ${name}: docker printed no ID and name:`, found.stdout); return; + } + if ((kind === 'container' ? actual.replace(/^\//, '') : actual) !== name) return; const removed = spawnSync(docker, kind === 'container' ? ['rm', '--force', id] : [kind, 'rm', ...(kind === 'volume' ? ['--force'] : []), id], options); // A network gone since the inspect is gone: Docker 27.4 answers "network not found" (rm --force of a missing // container or volume exits 0). From 59ed09f42d33627e38948bad1a4e875ffd20aa3b Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 01:18:29 -0700 Subject: [PATCH 47/75] Comment fixes: what main's check was; what removeByName reads (#189) Review round 46. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 5 +++-- test/docker-sweep.test.ts | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index bf686e11..e08ee7ac 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -2375,8 +2375,9 @@ describe('real Docker agent isolation', () => { const data = fixture(), runnerOwner = randomBytes(16).toString('hex'); const owner = { runnerOwner, attemptId: 'unsettled-allocation', allocationId: randomUUID() }; // Another runner's objects, so no sweep of this run finds them: removed by allocation when the test finishes (#189). - // After a pass, nothing of the allocation may be left once that sweep has run, as on main; this check, registered - // first, runs after the sweep (Vitest runs onTestFinished hooks in reverse order). + // After a pass, a listing that fails fails the test, as on main, and now nothing of the allocation may be left once + // that sweep has run either; this check, registered first, runs after the sweep (Vitest runs onTestFinished hooks + // in reverse order). let passed = false; onTestFinished(() => { if (passed) expect(byAllocation(owner.allocationId)).toEqual([]); }, 2 * 60_000); sweepWhenFinished('io.codeboost.allocation', owner.allocationId); diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index 9607cd50..8a521f70 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -9,8 +9,9 @@ import type * as Sweep from './fixtures/docker-sweep.ts'; // The test sweep's and `removeByName`'s handling of answers they cannot use (#189), each test with its own fake // `docker`, described beside it. The sweep reads only stdout, the exit status, whether a signal killed the call and // whether it could start, and stderr to report a failure, and, for its read of holders, to tell one gone since the -// listing ("No such container") from a failure; `removeByName` also reads an inspect's stderr to tell a missing object -// from a failure. Against the daemon, agent-container's holder test covers the sweep and removeByName's +// listing ("No such container") from a failure. `removeByName` reads the same, and also an inspect's stdout (the ID and +// name) and its stderr, to tell a missing object from a failure, and a network removal's stderr, to tell a network +// gone since its inspect. Against the daemon, agent-container's holder test covers the sweep and removeByName's // removal, its silence on a missing name, and its refusal of an object found by an ID prefix. const roots: string[] = []; afterEach(() => { From a0bcb1df190a00a6a4ce46cee28cf060f0648c7f Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 01:28:08 -0700 Subject: [PATCH 48/75] Test that the real docker is found once at load and that shims refuse without one (#189) Review round 47: a unit test loads the fixture with docker on PATH, then puts another first, and checks REAL_DOCKER keeps the first; loaded with no docker, requireRealDocker refuses. A docker found through a relative PATH entry is resolved to an absolute path. The never-started agent container's comment gives the real reason it is tracked. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 4 ++-- test/docker-sweep.test.ts | 25 ++++++++++++++++++++++++- test/fixtures/docker-sweep.ts | 7 +++++-- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index e08ee7ac..0afd91a9 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -983,8 +983,8 @@ describe('real Docker agent isolation', () => { it('never removes an agent container found by name after a killed create when a label is missing', async () => { const data = fixture(), live = await profile(data, 'planning', 'must-not-run'); - // Tracked for afterAll should the test time out: never started and without the runner label, it is no holder a - // sweep finds once its network is gone (#189). + // Tracked for afterAll should the test time out: it has no runner label, and until it goes the task volumes it + // mounts cannot be removed (#189). containers.add(live.name); // The create lands without the runner label, and its client is killed before it reports the ID. Only this // container's: should the test time out with the shim still on PATH, other tests' creates pass through. diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index 8a521f70..adb2282e 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -1,4 +1,4 @@ -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { spawnSync } from 'node:child_process'; @@ -173,3 +173,26 @@ it.each([ expect(warn.mock.calls.filter(call => String(call[0]).endsWith('failed:'))).toEqual(warnings); expect(readFileSync(calls, 'utf8').split('\n')).toContain(`rm --force ${a}`); }); + +it('finds the real docker once, at load, and refuses a shim when none was found', async () => { + const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); + // Two directories, each with a docker and the sh the lookup runs, and one with only sh. + const bin = (name: string, docker: boolean) => { + const dir = join(root, name); mkdirSync(dir); + symlinkSync(spawnSync('sh', ['-c', 'command -v sh'], { encoding: 'utf8' }).stdout.trim(), join(dir, 'sh')); + if (docker) writeFileSync(join(dir, 'docker'), '#!/bin/sh\nexit 0\n', { mode: 0o755 }); + return dir; + }; + const first = bin('first', true), second = bin('second', true), none = bin('none', false); + vi.stubEnv('PATH', first); + vi.resetModules(); + const loaded = await import('./fixtures/docker-sweep.ts') as typeof Sweep; + // A shim put on PATH later does not change it. + vi.stubEnv('PATH', second); + expect(loaded.REAL_DOCKER).toBe(join(first, 'docker')); + expect(loaded.requireRealDocker()).toBe(join(first, 'docker')); + vi.stubEnv('PATH', none); + vi.resetModules(); + const missing = await import('./fixtures/docker-sweep.ts') as typeof Sweep; + expect(() => missing.requireRealDocker()).toThrow('No docker on PATH when the test files loaded.'); +}); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index 5afc516d..b8f3857a 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -1,4 +1,5 @@ import { spawnSync, type SpawnSyncReturns } from 'node:child_process'; +import { isAbsolute, resolve } from 'node:path'; import { RUNNER_LABEL } from '../../agents/labels.ts'; /** @@ -6,7 +7,9 @@ import { RUNNER_LABEL } from '../../agents/labels.ts'; * shim is still on PATH (after a test timed out inside one) would otherwise wait on the shim's own child, which the * call's SIGKILL does not reach. */ -export const REAL_DOCKER = spawnSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).stdout?.trim() || 'docker'; +const FOUND_DOCKER = spawnSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).stdout?.trim(); +// Resolved, as a PATH entry may be relative. +export const REAL_DOCKER = FOUND_DOCKER ? resolve(FOUND_DOCKER) : 'docker'; /** * The daemon `docker` talks to when this module loads. A test that times out may leave DOCKER_HOST or DOCKER_CONTEXT * changed (agent-supervisor points DOCKER_HOST at a missing socket until its handle settles), so every call here @@ -20,7 +23,7 @@ const dockerEnv = () => { }; /** The real `docker` for a shim to call; refuses when none was found, since a shim calling `docker` would call itself. */ export function requireRealDocker(): string { - if (!REAL_DOCKER.startsWith('/')) throw new Error('No docker on PATH when the test files loaded.'); + if (!isAbsolute(REAL_DOCKER)) throw new Error('No docker on PATH when the test files loaded.'); return REAL_DOCKER; } From 515fe0c53521bf67f12877fc108711fd455f555e Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 01:40:38 -0700 Subject: [PATCH 49/75] Test the load-time docker as the default and a relative PATH entry; client limit; messages (#189) Review round 48: the finds-docker test loads the fixture through a relative PATH entry, then puts another docker first and calls both removals with no docker given, checking only the docker found at load ran; the network listing in the killed-create test gets its 60 s client limit back; the supervisor's removal message handles a docker that could not start; the fake says what it answers. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 2 +- test/agent-supervisor.test.ts | 2 +- test/docker-sweep.test.ts | 22 +++++++++++++++------- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 0afd91a9..060c9c7b 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -974,7 +974,7 @@ describe('real Docker agent isolation', () => { const created = await withDockerShim(['network', 'create'], foreign, () => createVendorNetwork(invocation(data.clone, 'planning'), imageId, allocationId).then(() => undefined, error => error)); const left = spawnSync('docker', ['network', 'ls', '--quiet', '--filter', `label=io.codeboost.allocation=${allocationId}`], - { encoding: 'utf8' }).stdout?.trim() ?? ''; + { encoding: 'utf8', timeout: 60_000, killSignal: 'SIGKILL' }).stdout?.trim() ?? ''; expect(created).toBeInstanceOf(VendorNetworkCreationCleanupError); const cleanup = (created as VendorNetworkCreationCleanupError).errors[1] as AggregateError; expect(cleanup.errors.map(error => (error as Error).message)).toEqual(['Refused to remove unowned vendor network.']); diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index c0d1a18d..942495a8 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -408,7 +408,7 @@ describe('container invocation supervisor', () => { // A failure above stays the one reported, whatever exit status this Docker version gives for a missing name. After // a pass the container exists, and its removal must succeed, as on main. const removed = spawnSync('docker', ['rm', '--force', current.name], { encoding: 'utf8' }); - if (passed && removed.status !== 0) throw new Error(`Removing ${current.name} failed: ${removed.stderr.trim()}`); + if (passed && removed.status !== 0) throw new Error(`Removing ${current.name} failed: ${removed.error?.message ?? removed.stderr?.trim()}`); } }, 3 * 60_000); diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index adb2282e..482361be 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -1,6 +1,6 @@ -import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { join, relative } from 'node:path'; import { spawnSync } from 'node:child_process'; import { afterEach, expect, it, vi } from 'vitest'; import { removeByName, removeLabelledObjects, TEST_RUN_LABEL } from './fixtures/docker-sweep.ts'; @@ -174,23 +174,31 @@ it.each([ expect(readFileSync(calls, 'utf8').split('\n')).toContain(`rm --force ${a}`); }); -it('finds the real docker once, at load, and refuses a shim when none was found', async () => { +it('finds the real docker once, at load, uses it by default, and refuses a shim when none was found', async () => { const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); - // Two directories, each with a docker and the sh the lookup runs, and one with only sh. + // Two directories, each with a docker and the sh the lookup runs, and one with only sh. Each docker records its calls + // and answers as Docker 27.4 does for a missing container and for an `ls` with no match; nothing else is asked of it. const bin = (name: string, docker: boolean) => { const dir = join(root, name); mkdirSync(dir); symlinkSync(spawnSync('sh', ['-c', 'command -v sh'], { encoding: 'utf8' }).stdout.trim(), join(dir, 'sh')); - if (docker) writeFileSync(join(dir, 'docker'), '#!/bin/sh\nexit 0\n', { mode: 0o755 }); + if (docker) writeFileSync(join(dir, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${join(dir, 'calls')}'`, + 'if [ "$2" = inspect ]; then echo; echo "Error response from daemon: No such container: x" >&2; exit 1; fi', 'exit 0'].join('\n'), + { mode: 0o755 }); return dir; }; const first = bin('first', true), second = bin('second', true), none = bin('none', false); - vi.stubEnv('PATH', first); + // A relative PATH entry, resolved at load. + vi.stubEnv('PATH', relative(process.cwd(), first)); vi.resetModules(); const loaded = await import('./fixtures/docker-sweep.ts') as typeof Sweep; - // A shim put on PATH later does not change it. + // A shim put on PATH later does not change it, and the removals use it when given no docker. vi.stubEnv('PATH', second); expect(loaded.REAL_DOCKER).toBe(join(first, 'docker')); expect(loaded.requireRealDocker()).toBe(join(first, 'docker')); + loaded.removeByName('container', 'x', 'test'); + loaded.removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test'); + expect(readFileSync(join(first, 'calls'), 'utf8').split('\n').filter(Boolean).length).toBeGreaterThan(1); + expect(existsSync(join(second, 'calls'))).toBe(false); vi.stubEnv('PATH', none); vi.resetModules(); const missing = await import('./fixtures/docker-sweep.ts') as typeof Sweep; From 820c2a27fa84303c5644eb9a2b66bc61ac3d9de1 Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 01:51:52 -0700 Subject: [PATCH 50/75] Every Docker client call the branch adds or moves in a test has a time limit (#189) Review round 49: synchronous calls cannot be stopped by a Vitest timer, so each one the branch added, moved or rewrote (removals, inspects, the foreign creates and the ack-script run) takes CLIENT_LIMIT, 60 s with SIGKILL, as main's docker() helper did; the supervisor's removal message names a kill by signal. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 18 +++++++++--------- test/agent-network.test.ts | 8 ++++---- test/agent-supervisor.test.ts | 12 ++++++------ test/fixtures/docker-sweep.ts | 5 +++++ 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 060c9c7b..fb9ce17d 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -1,5 +1,5 @@ import { execFileSync, spawnSync } from 'node:child_process'; -import { removeByName, removeLabelledObjects, removeRunnerObjects, requireRealDocker, TEST_FILE_LABEL, TEST_RUN_LABEL } from './fixtures/docker-sweep.ts'; +import { CLIENT_LIMIT, removeByName, removeLabelledObjects, removeRunnerObjects, requireRealDocker, TEST_FILE_LABEL, TEST_RUN_LABEL } from './fixtures/docker-sweep.ts'; import { fixtureGit } from './fixtures/git.ts'; import { createHash, randomBytes, randomUUID } from 'node:crypto'; import { chmodSync, existsSync, mkdtempSync, mkdirSync, readdirSync, readFileSync, realpathSync, renameSync, rmSync, statSync, @@ -490,8 +490,8 @@ describe('real Docker agent isolation', () => { docker('rm', '--force', valid.name); containers.delete(valid.name); } finally { // The container first: a volume it mounts cannot be removed. - spawnSync('docker', ['rm', '--force', valid.name], { stdio: 'ignore' }); - spawnSync('docker', ['volume', 'rm', '--force', rogue], { stdio: 'ignore' }); + spawnSync('docker', ['rm', '--force', valid.name], { stdio: 'ignore', ...CLIENT_LIMIT }); + spawnSync('docker', ['volume', 'rm', '--force', rogue], { stdio: 'ignore', ...CLIENT_LIMIT }); } }, 60_000); @@ -819,7 +819,7 @@ describe('real Docker agent isolation', () => { expect(docker('network', 'inspect', '--format', '{{.Id}}', network.slice(0, 12))).toBe(lookalike); // And an inspect naming a missing container exits non-zero yet prints the others, the answer the shim below gives. const partial = spawnSync('docker', ['container', 'inspect', '--format', '{{.Id}}', holders[0]!, `codeboost-gone-${randomUUID()}`], - { encoding: 'utf8' }); + { encoding: 'utf8', ...CLIENT_LIMIT }); expect(partial.status).not.toBe(0); expect(partial.stdout).toContain(holders[0]); expect(partial.stderr).toMatch(/^Error response from daemon: No such container: codeboost-gone-/m); @@ -846,7 +846,7 @@ describe('real Docker agent isolation', () => { const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); try { for (const [kind, name] of byName) removeByName(kind, name, 'test'); - expect(byName.filter(([kind, name]) => spawnSync('docker', [kind, 'inspect', name], { stdio: 'ignore' }).status === 0)).toEqual([]); + expect(byName.filter(([kind, name]) => spawnSync('docker', [kind, 'inspect', name], { stdio: 'ignore', ...CLIENT_LIMIT }).status === 0)).toEqual([]); for (const [kind, name] of byName) removeByName(kind, name, 'test'); expect(warn).not.toHaveBeenCalled(); } finally { warn.mockRestore(); } @@ -856,7 +856,7 @@ describe('real Docker agent isolation', () => { removeByName('network', lookalike.slice(0, 12), 'test'); expect(docker('container', 'inspect', '--format', '{{.Id}}', bystanders[0]!.slice(0, 12))).toBe(bystanders[0]); removeByName('container', bystanders[0]!.slice(0, 12), 'test'); - const exists = (kind: string, ref: string) => spawnSync('docker', [kind, 'inspect', ref], { stdio: 'ignore' }).status === 0; + const exists = (kind: string, ref: string) => spawnSync('docker', [kind, 'inspect', ref], { stdio: 'ignore', ...CLIENT_LIMIT }).status === 0; for (const holder of holders) expect(exists('container', holder)).toBe(false); expect([exists('network', network), exists('network', prefixNamed), exists('volume', volume)]).toEqual([false, false, false]); expect(exists('container', labelled)).toBe(false); @@ -893,7 +893,7 @@ describe('real Docker agent isolation', () => { // First, so afterAll does not refuse a keeper this test finds left: its file-label sweep removes that keeper, then // its runner sweep the volumes. taskFilesystems.splice(taskFilesystems.indexOf(left), 1); - expect(spawnSync('docker', ['container', 'inspect', left.keeper], { stdio: 'ignore' }).status).not.toBe(0); + expect(spawnSync('docker', ['container', 'inspect', left.keeper], { stdio: 'ignore', ...CLIENT_LIMIT }).status).not.toBe(0); // Its volumes carry the runner label, so ordinary cleanup can remove them now that no keeper holds them. removeTaskFilesystems(left); expect(byAllocation(allocationId)).toEqual([]); @@ -2559,7 +2559,7 @@ describe('real Docker agent isolation', () => { removeTaskFilesystems(handle); expect(isRecoveredTaskStorage(handle)).toBe(false); for (const [kind, ref] of [['container', left.keeper], ['volume', left.workVolume], ['volume', left.metadataVolume]] as const) - expect(spawnSync('docker', [kind, 'inspect', ref], { stdio: 'ignore' }).status).not.toBe(0); + expect(spawnSync('docker', [kind, 'inspect', ref], { stdio: 'ignore', ...CLIENT_LIMIT }).status).not.toBe(0); }, 180_000); it('never removes a task keeper replaced by another runner, but still removes the owned volumes', () => { @@ -2580,7 +2580,7 @@ describe('real Docker agent isolation', () => { expect(spawnSync('docker', ['volume', 'inspect', filesystems.workVolume]).status).not.toBe(0); passed = true; } finally { - spawnSync('docker', ['rm', '--force', filesystems.keeper], { stdio: 'ignore' }); + spawnSync('docker', ['rm', '--force', filesystems.keeper], { stdio: 'ignore', ...CLIENT_LIMIT }); // A failure above stays the one reported: what the removal leaves, the allocation sweep below removes, the keeper // with another runner's label included (#189). After a pass, the storage must release once the foreign keeper is // gone, so its failure fails the test. The sweep may take up to 5.5 minutes, which the test's budget allows for. diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index aeae2f55..2be757bc 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -9,7 +9,7 @@ import { assertVendorNetwork, createVendorNetwork, removeVendorNetwork, VENDOR_H VendorNetworkCreationCleanupError, type VendorNetwork } from '../agents/network/network.ts'; import { captureInvocation, type InvocationInput } from '../agents/contract.ts'; -import { removeByName, removeLabelledObjects, removeRunnerObjects, requireRealDocker, TEST_FILE_LABEL } from './fixtures/docker-sweep.ts'; +import { CLIENT_LIMIT, removeByName, removeLabelledObjects, removeRunnerObjects, requireRealDocker, TEST_FILE_LABEL } from './fixtures/docker-sweep.ts'; // Each run has its own runner owner, so the sweep after this file removes only what this run left behind. Network and // proxy names are random, so the fixed attempt IDs below reach only labels, where this owner tells runs apart. const TEST_RUNNER_OWNER = randomBytes(16).toString('hex'); @@ -135,7 +135,7 @@ describe('vendor-only egress', () => { const made = existsSync(recorded) ? readFileSync(recorded, 'utf8').split('\n').filter(Boolean) : []; foreign = made[0] ?? ''; // afterAll reads the record again: the object may land after this removal (#189). - if (foreign) survived = spawnSync('docker', ['network', 'inspect', foreign], { stdio: 'ignore' }).status === 0; + if (foreign) survived = spawnSync('docker', ['network', 'inspect', foreign], { stdio: 'ignore', ...CLIENT_LIMIT }).status === 0; for (const name of made) removeByName('network', name, 'finally'); rmSync(shim, { recursive: true, force: true }); } @@ -191,7 +191,7 @@ describe('vendor-only egress', () => { const made = existsSync(recorded) ? readFileSync(recorded, 'utf8').split('\n').filter(Boolean) : []; foreign = made[0] ?? ''; // afterAll reads the record again: the object may land after this removal (#189). - if (foreign) survived = spawnSync('docker', ['container', 'inspect', foreign], { stdio: 'ignore' }).status === 0; + if (foreign) survived = spawnSync('docker', ['container', 'inspect', foreign], { stdio: 'ignore', ...CLIENT_LIMIT }).status === 0; for (const name of made) removeByName('container', name, 'finally'); rmSync(shim, { recursive: true, force: true }); } @@ -247,7 +247,7 @@ describe('vendor-only egress', () => { const made = existsSync(recorded) ? readFileSync(recorded, 'utf8').split('\n').filter(Boolean) : []; impostor = made[0] ?? ''; // afterAll reads the record again: the object may land after this removal (#189). - if (impostor) survived = spawnSync('docker', ['container', 'inspect', impostor], { stdio: 'ignore' }).status === 0; + if (impostor) survived = spawnSync('docker', ['container', 'inspect', impostor], { stdio: 'ignore', ...CLIENT_LIMIT }).status === 0; for (const name of made) removeByName('container', name, 'finally'); rmSync(shim, { recursive: true, force: true }); } diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 942495a8..5a138295 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -1,6 +1,6 @@ import { randomBytes, randomUUID } from 'node:crypto'; import { execFileSync, spawnSync } from 'node:child_process'; -import { removeByName, removeRunnerObjects, requireRealDocker } from './fixtures/docker-sweep.ts'; +import { CLIENT_LIMIT, removeByName, removeRunnerObjects, requireRealDocker } from './fixtures/docker-sweep.ts'; import { fixtureGit } from './fixtures/git.ts'; import { chmodSync, mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; @@ -393,7 +393,7 @@ describe('container invocation supervisor', () => { try { // Inside the try, so its removal runs even if the create's client fails after the daemon made it (#189). execFileSync('docker', ['create', '--name', current.name, '--label', 'io.codeboost.invocation=foreign', - '--entrypoint', 'true', imageId], { stdio: 'ignore' }); + '--entrypoint', 'true', imageId], { stdio: 'ignore', ...CLIENT_LIMIT }); const result = await startProfileInvocation(current).settled; expect(result.stopReason).toBe('capture-failure'); expect(result.stderr).toMatch(/already in use|Conflict/); @@ -407,8 +407,8 @@ describe('container invocation supervisor', () => { } finally { // A failure above stays the one reported, whatever exit status this Docker version gives for a missing name. After // a pass the container exists, and its removal must succeed, as on main. - const removed = spawnSync('docker', ['rm', '--force', current.name], { encoding: 'utf8' }); - if (passed && removed.status !== 0) throw new Error(`Removing ${current.name} failed: ${removed.error?.message ?? removed.stderr?.trim()}`); + const removed = spawnSync('docker', ['rm', '--force', current.name], { encoding: 'utf8', ...CLIENT_LIMIT }); + if (passed && removed.status !== 0) throw new Error(`Removing ${current.name} failed: ${removed.error?.message ?? (removed.signal ? `killed by ${removed.signal}` : removed.stderr?.trim())}`); } }, 3 * 60_000); @@ -450,7 +450,7 @@ describe('container invocation supervisor', () => { onTestFinished(() => removeByName('container', current.name, 'onTestFinished'), 90_000); // two calls of up to 30 s try { execFileSync('docker', ['create', '--name', current.name, '--label', 'io.codeboost.invocation=someone-else', - '--entrypoint', 'true', imageId], { stdio: 'ignore' }); + '--entrypoint', 'true', imageId], { stdio: 'ignore', ...CLIENT_LIMIT }); await expectRejected(startProfileInvocation(current, { timeoutMs: 10 * 60_000 + 1 }), 'ceiling'); expect(isContainerProfileAuthentic(current)).toBe(false); expect(spawnSync('docker', ['container', 'inspect', current.name], { stdio: 'ignore' }).status).toBe(0); @@ -658,7 +658,7 @@ describe('container invocation supervisor', () => { execFileSync('docker', ['run', '-d', '--rm', '--name', name, '--network', 'none', '--cap-drop=ALL', '--security-opt=no-new-privileges', '--user', '10001:10001', '--entrypoint', 'sleep', '--tmpfs', '/run/codeboost-control:rw,nosuid,nodev,noexec,size=65536,nr_inodes=16,uid=0,gid=0,mode=0711', - imageId, '120'], { stdio: 'ignore' }); + imageId, '120'], { stdio: 'ignore', ...CLIENT_LIMIT }); const acknowledge = () => spawnSync('docker', ['exec', '--user', '0', name, 'node', '-e', ACKNOWLEDGEMENT_SCRIPT, token], { encoding: 'utf8' }); const content = () => execFileSync('docker', ['exec', name, 'cat', file], { encoding: 'utf8' }); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index b8f3857a..8f576c4b 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -21,6 +21,11 @@ const dockerEnv = () => { for (const [name, value] of Object.entries(LOADED_ENV)) if (value === undefined) delete env[name]; else env[name] = value; return env; }; +/** + * A Docker client call's own limit, for a test's synchronous calls (#189): no Vitest timer can stop one, so a stalled + * daemon would otherwise hold the whole suite. + */ +export const CLIENT_LIMIT = { timeout: 60_000, killSignal: 'SIGKILL' as const }; /** The real `docker` for a shim to call; refuses when none was found, since a shim calling `docker` would call itself. */ export function requireRealDocker(): string { if (!isAbsolute(REAL_DOCKER)) throw new Error('No docker on PATH when the test files loaded.'); From 067c7133d4c941ebd1b16446e7e89a7ead093155 Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 02:08:31 -0700 Subject: [PATCH 51/75] Track the ack-script container by name; time limits on the moved and attached calls (#189) Review round 50: agent-supervisor's label-less ack-script container is tracked by name before its create and removed by afterAll; runner-workspace's docker helper, which now runs calls this branch moved out of a try, takes CLIENT_LIMIT; the seccomp probe and agent-network's attached helper runs are killed with SIGKILL at their limit, as an attached run forwards SIGTERM; the killed-create test's listing reports its own failure. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 9 ++++++--- test/agent-network.test.ts | 6 +++--- test/agent-supervisor.test.ts | 8 +++++++- test/runner-workspace.test.ts | 5 +++-- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index fb9ce17d..aef86d95 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -973,8 +973,11 @@ describe('real Docker agent isolation', () => { ].join('\n'); const created = await withDockerShim(['network', 'create'], foreign, () => createVendorNetwork(invocation(data.clone, 'planning'), imageId, allocationId).then(() => undefined, error => error)); - const left = spawnSync('docker', ['network', 'ls', '--quiet', '--filter', `label=io.codeboost.allocation=${allocationId}`], - { encoding: 'utf8', timeout: 60_000, killSignal: 'SIGKILL' }).stdout?.trim() ?? ''; + const listing = spawnSync('docker', ['network', 'ls', '--quiet', '--filter', `label=io.codeboost.allocation=${allocationId}`], + { encoding: 'utf8', ...CLIENT_LIMIT }); + // A listing that failed fails here with its cause, as main's helper did. + expect(listing.status, listing.error?.message ?? (listing.signal ? `killed by ${listing.signal}` : listing.stderr)).toBe(0); + const left = listing.stdout.trim(); expect(created).toBeInstanceOf(VendorNetworkCreationCleanupError); const cleanup = (created as VendorNetworkCreationCleanupError).errors[1] as AggregateError; expect(cleanup.errors.map(error => (error as Error).message)).toEqual(['Refused to remove unowned vendor network.']); @@ -2639,7 +2642,7 @@ describe('real Docker agent isolation', () => { containers.add(name); const result = spawnSync('docker', ['run', '--rm', '--name', name, '--read-only', '--user', '10001:10001', '--cap-drop=ALL', '--security-opt=no-new-privileges', '--security-opt=seccomp=unconfined', '--network=none', imageId, 'true'], - { encoding: 'utf8', timeout: 60_000 }); + { encoding: 'utf8', ...CLIENT_LIMIT }); expect(result.status).toBe(78); expect(result.stderr).toContain('seccomp syscall filter must be enforced'); }, 60_000); diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index 2be757bc..cf58f515 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -27,7 +27,7 @@ const docker = (...args: string[]) => execFileSync('docker', args, { const curl = (url: string, direct = false) => spawnSync('docker', ['run', '--rm', ...fileLabel, `--network=${network.name}`, '--env', `HTTPS_PROXY=${network.proxyUrl}`, ...(direct ? ['--env', 'NO_PROXY=*'] : []), '--entrypoint', 'curl', imageId, '--silent', '--show-error', '--output', '/dev/null', '--write-out', '%{http_code}', - '--max-time', '15', url], { encoding: 'utf8', timeout: 30_000, stdio: ['ignore', 'pipe', 'pipe'] }); + '--max-time', '15', url], { encoding: 'utf8', timeout: 30_000, killSignal: 'SIGKILL', stdio: ['ignore', 'pipe', 'pipe'] }); beforeAll(async () => { imageId = buildAgentImage(); @@ -316,7 +316,7 @@ describe('vendor-only egress', () => { "const dns=require('node:dns');dns.setServers(['127.0.0.11']);", "dns.resolve4('example.com',(error)=>process.exit(error?0:1));", 'setTimeout(()=>process.exit(0),3000);', - ].join('')], { encoding: 'utf8', timeout: 10_000, stdio: ['ignore', 'pipe', 'pipe'] }); + ].join('')], { encoding: 'utf8', timeout: 10_000, killSignal: 'SIGKILL', stdio: ['ignore', 'pipe', 'pipe'] }); expect(result.status).toBe(0); }, 30_000); @@ -340,7 +340,7 @@ describe('vendor-only egress', () => { '--entrypoint', 'node', imageId, '-e', "require('node:net').createServer(()=>{}).listen(4567,'0.0.0.0');setInterval(()=>{},1000)"); const result = spawnSync('docker', ['run', '--rm', ...fileLabel, `--network=${other.name}`, '--entrypoint', 'node', imageId, '-e', "const s=require('node:net').connect(4567,'codeboost-peer');s.on('connect',()=>process.exit(0));s.on('error',()=>process.exit(1));setTimeout(()=>process.exit(2),3000)"], - { encoding: 'utf8', timeout: 10_000, stdio: ['ignore', 'pipe', 'pipe'] }); + { encoding: 'utf8', timeout: 10_000, killSignal: 'SIGKILL', stdio: ['ignore', 'pipe', 'pipe'] }); expect(result.status).not.toBe(0); } finally { spawnSync('docker', ['rm', '--force', peer], { stdio: 'ignore' }); diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 5a138295..2f8b0cde 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -30,6 +30,8 @@ const testOwner = (attemptId = 'fixture') => ({ runnerOwner: TEST_RUNNER_OWNER, const roots: string[] = [], profiles: ContainerProfile[] = []; const allocations: ReturnType[] = []; +/** Label-less containers, by name, removed in afterAll should their create land after their test's finally (#189). */ +const unlabelled = new Set(); let imageId = ''; const git = fixtureGit; @@ -70,6 +72,7 @@ async function profile(data: ReturnType, probe: IsolationProbe, beforeAll(async () => { imageId = buildAgentImage(); }, 10 * 60_000); afterAll(async () => { try { + for (const name of unlabelled) removeByName('container', name, 'afterAll'); for (const profile of profiles) await disposeContainerProfile(profile); for (const allocation of allocations.reverse()) removeTaskFilesystems(allocation); for (const root of roots.reverse()) { @@ -653,8 +656,11 @@ describe('container invocation supervisor', () => { it('writes the deferred output acknowledgement idempotently, repairing a partial one', () => { const token = randomUUID(), file = `/run/codeboost-control/collected-${token}`; const name = `codeboost-ack-script-${randomUUID()}`; + // Tracked before it is made: it has no labels, and a create that lands after its client was killed is never started, + // so `--rm` does not apply and only removal by name finds it (#189). + unlabelled.add(name); try { - // Inside the try: the container has no labels, so only the finally below removes it (#189). + // Inside the try: the container has no labels, so the finally below removes it, and afterAll by name (#189). execFileSync('docker', ['run', '-d', '--rm', '--name', name, '--network', 'none', '--cap-drop=ALL', '--security-opt=no-new-privileges', '--user', '10001:10001', '--entrypoint', 'sleep', '--tmpfs', '/run/codeboost-control:rw,nosuid,nodev,noexec,size=65536,nr_inodes=16,uid=0,gid=0,mode=0711', diff --git a/test/runner-workspace.test.ts b/test/runner-workspace.test.ts index ad55ca11..ee0fbafe 100644 --- a/test/runner-workspace.test.ts +++ b/test/runner-workspace.test.ts @@ -15,7 +15,7 @@ import { attemptRef, openRunnerRepository } from '../runner/runner-repository.ts import { ReviewService } from '../runner/review.ts'; import { Store } from '../runner/store.ts'; import { createTaskWorkspace } from '../runner/workspace.ts'; -import { removeRunnerObjects, requireRealDocker } from './fixtures/docker-sweep.ts'; +import { CLIENT_LIMIT, removeRunnerObjects, requireRealDocker } from './fixtures/docker-sweep.ts'; import { fixtureGit as git } from './fixtures/git.ts'; // F2b against the real lane D (#87): each item runs in fresh task storage cloned from the runner-owned repository, an @@ -43,7 +43,8 @@ afterAll(() => { removeRunnerObjects(RUNNER_OWNER); } }, 7 * 60_000); -const docker = (...args: string[]) => execFileSync('docker', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); +const docker = (...args: string[]) => execFileSync('docker', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], + ...CLIENT_LIMIT }).trim(); async function setup(agent: Record) { const root = mkdtempSync(join(tmpdir(), 'runner-workspace-')); roots.push(root); From 607f7ede2e23f8de6ee00be711251e4c3e9f850f Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 02:21:34 -0700 Subject: [PATCH 52/75] Name a failed foreign-keeper removal after a pass (#189) Review round 51: as on main, a docker rm of the foreign keeper that fails after the test's checks pass fails the test with Docker's own error, before the storage release's. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index aef86d95..f405abcf 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -2583,13 +2583,16 @@ describe('real Docker agent isolation', () => { expect(spawnSync('docker', ['volume', 'inspect', filesystems.workVolume]).status).not.toBe(0); passed = true; } finally { - spawnSync('docker', ['rm', '--force', filesystems.keeper], { stdio: 'ignore', ...CLIENT_LIMIT }); + const keeperRemoved = spawnSync('docker', ['rm', '--force', filesystems.keeper], { encoding: 'utf8', ...CLIENT_LIMIT }); // A failure above stays the one reported: what the removal leaves, the allocation sweep below removes, the keeper - // with another runner's label included (#189). After a pass, the storage must release once the foreign keeper is - // gone, so its failure fails the test. The sweep may take up to 5.5 minutes, which the test's budget allows for. + // with another runner's label included (#189). After a pass, the foreign keeper's removal and then the storage's + // release must succeed, as on main, so either failure fails the test, the keeper's first. The sweep may take up to + // 5.5 minutes, which the test's budget allows for. let released: unknown; try { removeTaskFilesystems(filesystems); } catch (error) { released = error; console.warn('cleanup: removing the storage failed:', error); } removeLabelledObjects('io.codeboost.allocation', owner.allocationId, 'test'); + if (passed && keeperRemoved.status !== 0) throw new Error(`docker rm --force ${filesystems.keeper} failed: ` + + (keeperRemoved.error?.message ?? (keeperRemoved.signal ? `killed by ${keeperRemoved.signal}` : keeperRemoved.stderr?.trim()))); if (passed && released) throw released; } }, 7 * 60_000); From 582f786d552aa017455ee1d9b808d1e87be11b49 Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 02:35:58 -0700 Subject: [PATCH 53/75] Workflow limits for the longer teardown; full test of the loaded daemon; keeper warning (#189) Review round 52: the Live probes job limit rises to 90 minutes, as the teardown hooks it covers now allow 24 (vitest-config pins it); the Agent isolation comment says its 45 minutes cover ordinary runs; the loaded-daemon test answers one network with a holder and one named container, so every kind of call is checked; a failed foreign-keeper removal is warned of when the body failed. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/agent-isolation.yml | 4 +++- .github/workflows/live-probes.yml | 10 +++++----- test/agent-container.test.ts | 6 ++++-- test/docker-sweep.test.ts | 24 ++++++++++++++++++------ test/vitest-config.test.ts | 2 +- 5 files changed, 31 insertions(+), 15 deletions(-) diff --git a/.github/workflows/agent-isolation.yml b/.github/workflows/agent-isolation.yml index 838004a6..b00f86e0 100644 --- a/.github/workflows/agent-isolation.yml +++ b/.github/workflows/agent-isolation.yml @@ -9,7 +9,9 @@ permissions: jobs: real-docker: runs-on: ubuntu-latest - # Covers the 10 min image build, 2 min teardown and the per-test Docker budgets. + # Covers the 10 min image build and ordinary run times of the Docker-backed files. It does not cover every test + # and hook at its own Vitest limit: the five afterAll hooks that sweep allow 46 min between them (#189), which only + # a stalled daemon reaches. timeout-minutes: 45 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/live-probes.yml b/.github/workflows/live-probes.yml index 31728b85..78e30e37 100644 --- a/.github/workflows/live-probes.yml +++ b/.github/workflows/live-probes.yml @@ -15,11 +15,11 @@ jobs: runs-on: ubuntu-latest # The probes' own time limits total 33 min (6, 8, 8 and 11). agent-container and agent-supervisor may each spend # up to 10 min building the image before their probes; agent-question builds inside its probe's own 11 min. The - # teardown hooks have Vitest limits that total 9 min: agent-container 2 after each test and 2 at the end, - # agent-supervisor 3 and agent-question 2. That is 62 min, plus setup. These are only Vitest's limits: the afterAll - # hooks in agent-container and agent-question call docker through spawnSync with no timeout of their own, and - # Vitest's hook timer cannot fire while such a call blocks. The 75 min job limit caps the run. - timeout-minutes: 75 + # teardown hooks have Vitest limits that total 24 min: agent-container 2 after each test and 13 at the end (two + # sweeps of up to 5.5 min each, #189), agent-supervisor 7 and agent-question 2. That is 77 min, plus setup. These + # are only Vitest's limits: Vitest's hook timer cannot fire while a synchronous docker call blocks, and some calls in + # these hooks have no timeout of their own (the sweeps' calls have 30 s each). The 90 min job limit caps the run. + timeout-minutes: 90 steps: # Fail at once when the secret is missing, so a missing secret never looks like a passing run. - name: Require the Claude token diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index f405abcf..115613ec 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -2591,8 +2591,10 @@ describe('real Docker agent isolation', () => { let released: unknown; try { removeTaskFilesystems(filesystems); } catch (error) { released = error; console.warn('cleanup: removing the storage failed:', error); } removeLabelledObjects('io.codeboost.allocation', owner.allocationId, 'test'); - if (passed && keeperRemoved.status !== 0) throw new Error(`docker rm --force ${filesystems.keeper} failed: ` - + (keeperRemoved.error?.message ?? (keeperRemoved.signal ? `killed by ${keeperRemoved.signal}` : keeperRemoved.stderr?.trim()))); + const keeperFailure = keeperRemoved.status === 0 ? undefined : `docker rm --force ${filesystems.keeper} failed: ` + + (keeperRemoved.error?.message ?? (keeperRemoved.signal ? `killed by ${keeperRemoved.signal}` : keeperRemoved.stderr?.trim())); + if (keeperFailure && !passed) console.warn('cleanup:', keeperFailure); + if (keeperFailure && passed) throw new Error(keeperFailure); if (passed && released) throw released; } }, 7 * 60_000); diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index 482361be..522c0df8 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -128,12 +128,19 @@ it('talks to the daemon set when it loaded, though a test changed DOCKER_HOST or expect(new Set(lines)).toEqual(new Set([loaded])); }); -it('takes the daemon set at load, not the default, when one was set', async () => { +it('takes the daemon set at load, not the default, when one was set, for every call', async () => { const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); - const seen = join(root, 'seen'); - // Records the address each call is given, and answers as Docker 27.4 does for a missing container and an empty `ls`. - writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s|%s\\n' "\${DOCKER_HOST-unset}" "\${DOCKER_CONTEXT-unset}" >> '${seen}'`, - 'if [ "$2" = inspect ]; then echo; echo "Error response from daemon: No such container: x" >&2; exit 1; fi', 'exit 0'].join('\n'), + const seen = join(root, 'seen'), net = 'n'.repeat(64), holder = 'a'.repeat(64), named = 'b'.repeat(64); + // Records the address and command of each call. Answers as Docker 27.4 does, in the parts the code reads, for one + // swept network with one holder and for a container named x; every other call gets no output and exit 0, as for + // `ls` with no match, and the removals' stdout is not read. The network line holds only the fields the sweep reads. + writeFileSync(join(root, 'docker'), ['#!/bin/sh', 'for arg; do last="$arg"; done', + `printf '%s|%s|%s %s\\n' "\${DOCKER_HOST-unset}" "\${DOCKER_CONTEXT-unset}" "$1" "$2" >> '${seen}'`, + `if [ "$1 $2" = "network ls" ]; then printf '%s\\n' '{"ID":"${net}","Name":"swept"}'; exit 0; fi`, + `if [ "$1" = ps ]; then printf '%s\\n' ${holder}; exit 0; fi`, + `if [ "$1 $2" = "container inspect" ] && [ "$last" = x ]; then printf '%s\\n' '"${named}" "/x"'; exit 0; fi`, + `if [ "$1 $2" = "container inspect" ]; then printf '%s\\n' '"${holder}" {"swept":{}}'; exit 0; fi`, + 'exit 0'].join('\n'), { mode: 0o755 }); // A fresh copy of the module, loaded while a daemon is set, as on a machine with Colima or rootless Docker. vi.stubEnv('DOCKER_HOST', 'unix:///codeboost-loaded.sock'); @@ -146,7 +153,12 @@ it('takes the daemon set at load, not the default, when one was set', async () = fresh.removeByName('container', 'x', 'test', join(root, 'docker')); fresh.removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test', join(root, 'docker')); } finally { vi.unstubAllEnvs(); } - expect(new Set(readFileSync(seen, 'utf8').split('\n').filter(Boolean))).toEqual(new Set(['unix:///codeboost-loaded.sock|codeboost-loaded'])); + const lines = readFileSync(seen, 'utf8').split('\n').filter(Boolean).map(line => line.split('|')); + // Every kind of call the two make: inspect and removal by name, the listings, the holder lookup and read, the + // holders' removal and the network's. + expect(new Set(lines.map(line => line[2]))).toEqual(new Set(['container inspect', 'rm --force', 'container ls', 'network ls', + 'volume ls', 'ps --all', 'network rm'])); + expect(new Set(lines.map(line => `${line[0]}|${line[1]}`))).toEqual(new Set(['unix:///codeboost-loaded.sock|codeboost-loaded'])); }); it.each([ diff --git a/test/vitest-config.test.ts b/test/vitest-config.test.ts index d010c604..7c229ff6 100644 --- a/test/vitest-config.test.ts +++ b/test/vitest-config.test.ts @@ -317,7 +317,7 @@ describe('CI workflows', () => { const job = live.jobs['live-probes']!; // No `defaults`, `env`, `strategy` or `container`: any of them could change what the steps run. expect(Object.keys(job).toSorted()).toEqual(['runs-on', 'steps', 'timeout-minutes']); - expect(job).toMatchObject({ 'runs-on': 'ubuntu-latest', 'timeout-minutes': 75 }); + expect(job).toMatchObject({ 'runs-on': 'ubuntu-latest', 'timeout-minutes': 90 }); const token = { CLAUDE_CODE_OAUTH_TOKEN: '${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}' }; expect(job.steps.map(step => Object.keys(step).toSorted())).toEqual([['env', 'name', 'run'], ['uses'], ['uses', 'with'], ['run'], ['env', 'name', 'run'], ['name', 'run']]); From 2af834e9bf179c66ecbb3b94b89812298708bcf8 Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 02:46:34 -0700 Subject: [PATCH 54/75] Live probes limit counts agent-planning's probe and teardown; clearer missing-record failures (#189) Review round 53: the comment counted four probes, before #193 added agent-planning's; with its 11-minute probe and 2-minute teardown the limits total 90 minutes plus setup, so the job limit rises to 100 (pinned in vitest-config); the refused-network and refused-proxy tests say so when their shim recorded nothing. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/live-probes.yml | 15 ++++++++------- test/agent-network.test.ts | 2 ++ test/vitest-config.test.ts | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/live-probes.yml b/.github/workflows/live-probes.yml index 78e30e37..515da1e6 100644 --- a/.github/workflows/live-probes.yml +++ b/.github/workflows/live-probes.yml @@ -13,13 +13,14 @@ concurrency: jobs: live-probes: runs-on: ubuntu-latest - # The probes' own time limits total 33 min (6, 8, 8 and 11). agent-container and agent-supervisor may each spend - # up to 10 min building the image before their probes; agent-question builds inside its probe's own 11 min. The - # teardown hooks have Vitest limits that total 24 min: agent-container 2 after each test and 13 at the end (two - # sweeps of up to 5.5 min each, #189), agent-supervisor 7 and agent-question 2. That is 77 min, plus setup. These - # are only Vitest's limits: Vitest's hook timer cannot fire while a synchronous docker call blocks, and some calls in - # these hooks have no timeout of their own (the sweeps' calls have 30 s each). The 90 min job limit caps the run. - timeout-minutes: 90 + # The probes' own time limits total 44 min (6, 8, 8, 11 and 11). agent-container and agent-supervisor may each spend + # up to 10 min building the image before their probes; agent-question and agent-planning build inside their probes' + # own 11 min. The teardown hooks have Vitest limits that total 26 min: agent-container 2 after each test and 13 at + # the end (two sweeps of up to 5.5 min each, #189), agent-supervisor 7, agent-question 2 and agent-planning 2. That + # is 90 min, plus setup. These are only Vitest's limits: Vitest's hook timer cannot fire while a synchronous docker + # call blocks, and some calls in these hooks have no timeout of their own (the sweeps' calls have 30 s each). The + # 100 min job limit caps the run. + timeout-minutes: 100 steps: # Fail at once when the secret is missing, so a missing secret never looks like a passing run. - name: Require the Claude token diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index cf58f515..dea0d365 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -143,6 +143,7 @@ describe('vendor-only egress', () => { expect(error).toBeInstanceOf(Error); expect(error).not.toBeInstanceOf(VendorNetworkCreationCleanupError); expect(String(error)).toMatch(/already exists/); + expect(foreign, 'the shim recorded no foreign network').not.toBe(''); expect(survived).toBe(true); }, 60_000); @@ -199,6 +200,7 @@ describe('vendor-only egress', () => { expect(error).toBeInstanceOf(Error); expect(error).not.toBeInstanceOf(VendorNetworkCreationCleanupError); expect(String(error)).toMatch(/already in use|Conflict/); + expect(foreign, 'the shim recorded no foreign proxy').not.toBe(''); expect(survived).toBe(true); }, 60_000); diff --git a/test/vitest-config.test.ts b/test/vitest-config.test.ts index 7c229ff6..3ea47599 100644 --- a/test/vitest-config.test.ts +++ b/test/vitest-config.test.ts @@ -317,7 +317,7 @@ describe('CI workflows', () => { const job = live.jobs['live-probes']!; // No `defaults`, `env`, `strategy` or `container`: any of them could change what the steps run. expect(Object.keys(job).toSorted()).toEqual(['runs-on', 'steps', 'timeout-minutes']); - expect(job).toMatchObject({ 'runs-on': 'ubuntu-latest', 'timeout-minutes': 90 }); + expect(job).toMatchObject({ 'runs-on': 'ubuntu-latest', 'timeout-minutes': 100 }); const token = { CLAUDE_CODE_OAUTH_TOKEN: '${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}' }; expect(job.steps.map(step => Object.keys(step).toSorted())).toEqual([['env', 'name', 'run'], ['uses'], ['uses', 'with'], ['run'], ['env', 'name', 'run'], ['name', 'run']]); From 07938e2ff666083211f826666bbbcab83db47778 Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 03:01:49 -0700 Subject: [PATCH 55/75] Sweep quiet on a network gone since its listing; full loaded-daemon test; hook budget (#189) Review round 54: the sweep treats Docker 27.4's 'network not found' on its network removal as gone, as removeByName does, with a test that still reports any other failure; the loaded-daemon test now lists a container, a network and a volume, each with a holder, and checks every command line; the after-sweep check's budget covers its three listings; the Agent isolation comment names the shared sweep. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/agent-isolation.yml | 2 +- test/agent-container.test.ts | 3 +- test/docker-sweep.test.ts | 50 ++++++++++++++++++++++----- test/fixtures/docker-sweep.ts | 13 ++++--- 4 files changed, 52 insertions(+), 16 deletions(-) diff --git a/.github/workflows/agent-isolation.yml b/.github/workflows/agent-isolation.yml index b00f86e0..a941b592 100644 --- a/.github/workflows/agent-isolation.yml +++ b/.github/workflows/agent-isolation.yml @@ -10,7 +10,7 @@ jobs: real-docker: runs-on: ubuntu-latest # Covers the 10 min image build and ordinary run times of the Docker-backed files. It does not cover every test - # and hook at its own Vitest limit: the five afterAll hooks that sweep allow 46 min between them (#189), which only + # and hook at its own Vitest limit: the five afterAll hooks that call the shared sweep allow 46 min between them (#189), which only # a stalled daemon reaches. timeout-minutes: 45 steps: diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 115613ec..7acbf17a 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -2382,7 +2382,8 @@ describe('real Docker agent isolation', () => { // that sweep has run either; this check, registered first, runs after the sweep (Vitest runs onTestFinished hooks // in reverse order). let passed = false; - onTestFinished(() => { if (passed) expect(byAllocation(owner.allocationId)).toEqual([]); }, 2 * 60_000); + // Its budget covers byAllocation's three listings at their 60 s limits, plus a minute. + onTestFinished(() => { if (passed) expect(byAllocation(owner.allocationId)).toEqual([]); }, 4 * 60_000); sweepWhenFinished('io.codeboost.allocation', owner.allocationId); // The seeder is refused, and every volume removal fails, so the allocation's cleanup does not settle. const failing = [ diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index 522c0df8..a2398bf9 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -130,13 +130,16 @@ it('talks to the daemon set when it loaded, though a test changed DOCKER_HOST or it('takes the daemon set at load, not the default, when one was set, for every call', async () => { const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); - const seen = join(root, 'seen'), net = 'n'.repeat(64), holder = 'a'.repeat(64), named = 'b'.repeat(64); - // Records the address and command of each call. Answers as Docker 27.4 does, in the parts the code reads, for one - // swept network with one holder and for a container named x; every other call gets no output and exit 0, as for - // `ls` with no match, and the removals' stdout is not read. The network line holds only the fields the sweep reads. + const seen = join(root, 'seen'), net = 'n'.repeat(64), holder = 'a'.repeat(64), named = 'b'.repeat(64), labelled = 'c'.repeat(64); + // Records the address and the whole command of each call. Answers as Docker 27.4 does, in the parts the code reads, + // for one labelled container, one swept network and one swept volume, each held by one container, and for a container + // named x; every other call gets no output and exit 0, and the removals' stdout is not read. The network line holds + // only the fields the sweep reads, and the holder's inspect an empty endpoint for its network. writeFileSync(join(root, 'docker'), ['#!/bin/sh', 'for arg; do last="$arg"; done', - `printf '%s|%s|%s %s\\n' "\${DOCKER_HOST-unset}" "\${DOCKER_CONTEXT-unset}" "$1" "$2" >> '${seen}'`, + `printf '%s|%s|%s\\n' "\${DOCKER_HOST-unset}" "\${DOCKER_CONTEXT-unset}" "$*" >> '${seen}'`, + `if [ "$1 $2" = "container ls" ]; then printf '%s\\n' ${labelled}; exit 0; fi`, `if [ "$1 $2" = "network ls" ]; then printf '%s\\n' '{"ID":"${net}","Name":"swept"}'; exit 0; fi`, + `if [ "$1 $2" = "volume ls" ]; then printf '%s\\n' swept-volume; exit 0; fi`, `if [ "$1" = ps ]; then printf '%s\\n' ${holder}; exit 0; fi`, `if [ "$1 $2" = "container inspect" ] && [ "$last" = x ]; then printf '%s\\n' '"${named}" "/x"'; exit 0; fi`, `if [ "$1 $2" = "container inspect" ]; then printf '%s\\n' '"${holder}" {"swept":{}}'; exit 0; fi`, @@ -149,18 +152,47 @@ it('takes the daemon set at load, not the default, when one was set, for every c const fresh = await import('./fixtures/docker-sweep.ts') as typeof Sweep; vi.stubEnv('DOCKER_HOST', 'unix:///codeboost-changed.sock'); vi.stubEnv('DOCKER_CONTEXT', 'codeboost-changed'); + vi.spyOn(console, 'warn').mockImplementation(() => undefined); try { fresh.removeByName('container', 'x', 'test', join(root, 'docker')); fresh.removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test', join(root, 'docker')); } finally { vi.unstubAllEnvs(); } const lines = readFileSync(seen, 'utf8').split('\n').filter(Boolean).map(line => line.split('|')); - // Every kind of call the two make: inspect and removal by name, the listings, the holder lookup and read, the - // holders' removal and the network's. - expect(new Set(lines.map(line => line[2]))).toEqual(new Set(['container inspect', 'rm --force', 'container ls', 'network ls', - 'volume ls', 'ps --all', 'network rm'])); + const commands = lines.map(line => line[2]); + // Every call the two make, each checked by its own command line. + for (const command of [ + `container inspect --format {{json .Id}} {{json .Name}} x`, `rm --force ${named}`, + `container ls --all --quiet --no-trunc --filter label=${TEST_RUN_LABEL}=x`, `container rm --force ${labelled}`, + `network ls --no-trunc --format {{json .}} --filter label=${TEST_RUN_LABEL}=x`, + 'ps --all --quiet --no-trunc --filter network=swept', + `container inspect --format {{json .Id}} {{json .NetworkSettings.Networks}} ${holder}`, `rm --force ${holder}`, + `network rm ${net}`, + `volume ls --quiet --filter label=${TEST_RUN_LABEL}=x`, 'ps --all --quiet --no-trunc --filter volume=swept-volume', + 'volume rm --force swept-volume', + ]) expect(commands).toContain(command); expect(new Set(lines.map(line => `${line[0]}|${line[1]}`))).toEqual(new Set(['unix:///codeboost-loaded.sock|codeboost-loaded'])); }); +it('says nothing of a network gone since the listing, and reports any other failure to remove one', () => { + const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); + const net = 'd'.repeat(64), answer = join(root, 'answer'); + // Answers as Docker 27.4 does for one swept network with no holder, whose removal gives the stderr in `answer`; every + // other call gets no output and exit 0, as for `ls` with no match. The network line holds only the fields the sweep + // reads. The second answer is invented, as a failure 27.4 might give. + writeFileSync(join(root, 'docker'), ['#!/bin/sh', + `if [ "$1 $2" = "network ls" ]; then printf '%s\\n' '{"ID":"${net}","Name":"swept"}'; exit 0; fi`, + `if [ "$1 $2" = "network rm" ]; then cat '${answer}' >&2; exit 1; fi`, 'exit 0'].join('\n'), + { mode: 0o755 }); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + const failures = () => warn.mock.calls.filter(call => String(call[0]).endsWith('failed:')); + writeFileSync(answer, `Error response from daemon: network ${net} not found\n`); + removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test', join(root, 'docker')); + expect(failures()).toEqual([]); + writeFileSync(answer, 'Error response from daemon: permission denied\n'); + removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test', join(root, 'docker')); + expect(failures()).toEqual([['test: removing networks: docker network rm failed:', 'Error response from daemon: permission denied']]); +}); + it.each([ ['says nothing of a holder gone since the listing', 'Error response from daemon: No such container: b', []], ['reports any other failure to read the holders', 'permission denied', [['test: reading holders: docker container inspect failed:', 'permission denied']]], diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index 8f576c4b..e9f2d917 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -88,17 +88,20 @@ export function removeRunnerObjects(runnerOwner: string, when = 'afterAll'): voi * use. Before it removes a network or a volume it also removes every container attached to it or mounting it, labelled * or not: a test can make a network or volume with one owner and the container that holds it with another, or without * one. Containers and volumes go by force; a network another container joins meanwhile stays. It reports what it finds - * and any Docker call that fails (but not a read of holders that fails only because one has gone since the listing), - * prefixed with `when`, and never throws, so cleanup can call it without hiding an earlier error. Each call has a 30 s + * and any Docker call that fails (but not one that fails only because what it names has gone since the listing: a + * holder it reads, or a network it removes), prefixed with `when`, and never throws, so cleanup can call it without hiding an earlier error. Each call has a 30 s * limit, so all eleven can take up to 5.5 minutes. It runs `docker`, the real one by default (`REAL_DOCKER`). */ export function removeLabelledObjects(label: string, value: string, when: string, docker = REAL_DOCKER): void { const run = (stage: string, args: string[]) => { const result = spawnSync(docker, args, { encoding: 'utf8', timeout: 30_000, killSignal: 'SIGKILL', env: dockerEnv() }); // A holder gone since the listing is expected: Docker 27.4 answers "No such container" for it, and prints the rest. - const holdersGone = stage === 'reading holders' && result.status === 1 - && result.stderr.trim().split('\n').every(line => line.startsWith('Error response from daemon: No such container: ')); - if (result.status !== 0 && !holdersGone) console.warn(`${when}: ${stage}: docker ${args.slice(0, 2).join(' ')} failed:`, + // So is a network gone since the listing (removed by its own test meanwhile): "network not found". + const errors = result.stderr?.trim().split('\n') ?? []; + const gone = result.status === 1 && (stage === 'reading holders' + ? errors.every(line => line.startsWith('Error response from daemon: No such container: ')) + : stage === 'removing networks' && errors.every(line => /^Error response from daemon: network [0-9a-f]{64} not found$/.test(line))); + if (result.status !== 0 && !gone) console.warn(`${when}: ${stage}: docker ${args.slice(0, 2).join(' ')} failed:`, result.error?.message ?? (result.signal ? `killed by ${result.signal}` : result.stderr.trim())); // An inspect of several objects exits 1 if one is gone, yet prints the rest (Docker 27.4), so the output of a call // that exited by itself is read whatever its status. A killed call's output may be cut short, so it is not. From 008c2d0d4ba1d15faa5fab84a5131ae2ab734598 Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 03:12:42 -0700 Subject: [PATCH 56/75] Holder test checks its gone-holder shim fired; header and workflow comments (#189) Review round 55: the holder test's shim leaves a marker, and the test checks it before trusting the quiet sweep; the unit file's header says the sweep reads a network removal's stderr too; the Agent isolation comment names everything its 45 minutes cover. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/agent-isolation.yml | 6 +++--- test/agent-container.test.ts | 5 ++++- test/docker-sweep.test.ts | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/agent-isolation.yml b/.github/workflows/agent-isolation.yml index a941b592..20a6517f 100644 --- a/.github/workflows/agent-isolation.yml +++ b/.github/workflows/agent-isolation.yml @@ -9,9 +9,9 @@ permissions: jobs: real-docker: runs-on: ubuntu-latest - # Covers the 10 min image build and ordinary run times of the Docker-backed files. It does not cover every test - # and hook at its own Vitest limit: the five afterAll hooks that call the shared sweep allow 46 min between them (#189), which only - # a stalled daemon reaches. + # Covers the typecheck, the ordinary Vitest project, the 10 min image build and ordinary run times of the + # Docker-backed files. It does not cover every test and hook at its own Vitest limit: the five afterAll hooks that + # call the shared sweep allow 46 min between them (#189), which only a stalled daemon reaches. timeout-minutes: 45 steps: - uses: actions/checkout@v4 diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 7acbf17a..877f4f9a 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -828,8 +828,10 @@ describe('real Docker agent isolation', () => { const shim = mkdtempSync(join(tmpdir(), 'codeboost-sweep-docker-')); roots.push(shim); mkdirSync(join(shim, 'input')); // afterAll resets this path's mode const realDocker = requireRealDocker(); + // It leaves a marker, so the test knows the sweep's holder read went through it. + const marker = join(shim, 'gone-added'); writeFileSync(join(shim, 'docker'), ['#!/bin/sh', - `if [ "$1 $2" = "container inspect" ]; then exec '${realDocker}' "$@" codeboost-gone-${randomUUID()}; fi`, + `if [ "$1 $2" = "container inspect" ]; then : > '${marker}'; exec '${realDocker}' "$@" codeboost-gone-${randomUUID()}; fi`, `exec '${realDocker}' "$@"`].join('\n'), { mode: 0o755 }); // And it says nothing of that gone holder: Docker's own answer is not a failure. const sweepWarn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); @@ -839,6 +841,7 @@ describe('real Docker agent isolation', () => { removeLabelledObjects(TEST_RUN_LABEL, run, 'test', join(shim, 'docker')); sweepCalls = [...sweepWarn.mock.calls]; } finally { sweepWarn.mockRestore(); } + expect(existsSync(marker), 'the sweep read its holders through the shim').toBe(true); expect(sweepCalls.filter(call => String(call[0]).endsWith('failed:'))).toEqual([]); expect(sweepCalls.length).toBeGreaterThan(0); // afterAll removes tracked names with removeByName, as here: an object of exactly that name goes, and a name that diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index a2398bf9..cf52f84e 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -8,8 +8,8 @@ import type * as Sweep from './fixtures/docker-sweep.ts'; // The test sweep's and `removeByName`'s handling of answers they cannot use (#189), each test with its own fake // `docker`, described beside it. The sweep reads only stdout, the exit status, whether a signal killed the call and -// whether it could start, and stderr to report a failure, and, for its read of holders, to tell one gone since the -// listing ("No such container") from a failure. `removeByName` reads the same, and also an inspect's stdout (the ID and +// whether it could start, and stderr to report a failure, and, for its read of holders and its removal of networks, +// to tell one gone since the listing ("No such container", "network not found") from a failure. `removeByName` reads the same, and also an inspect's stdout (the ID and // name) and its stderr, to tell a missing object from a failure, and a network removal's stderr, to tell a network // gone since its inspect. Against the daemon, agent-container's holder test covers the sweep and removeByName's // removal, its silence on a missing name, and its refusal of an object found by an ID prefix. From 6411bda7b1b11b16cc11590e6d44965e82e560e5 Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 03:28:04 -0700 Subject: [PATCH 57/75] Rewrap two comment blocks (#189) Review round 56. Co-Authored-By: Claude Opus 5.5 --- test/docker-sweep.test.ts | 11 ++++++----- test/fixtures/docker-sweep.ts | 5 +++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index cf52f84e..a42fcee6 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -8,11 +8,12 @@ import type * as Sweep from './fixtures/docker-sweep.ts'; // The test sweep's and `removeByName`'s handling of answers they cannot use (#189), each test with its own fake // `docker`, described beside it. The sweep reads only stdout, the exit status, whether a signal killed the call and -// whether it could start, and stderr to report a failure, and, for its read of holders and its removal of networks, -// to tell one gone since the listing ("No such container", "network not found") from a failure. `removeByName` reads the same, and also an inspect's stdout (the ID and -// name) and its stderr, to tell a missing object from a failure, and a network removal's stderr, to tell a network -// gone since its inspect. Against the daemon, agent-container's holder test covers the sweep and removeByName's -// removal, its silence on a missing name, and its refusal of an object found by an ID prefix. +// whether it could start, and stderr to report a failure, and, for its read of holders and its removal of networks, to +// tell one gone since the listing ("No such container", "network not found") from a failure. `removeByName` reads +// the same, and also an inspect's stdout (the ID and name) and its stderr, to tell a missing object from a failure, and +// a network removal's stderr, to tell a network gone since its inspect. Against the daemon, agent-container's holder +// test covers the sweep and removeByName's removal, its silence on a missing name, and its refusal of an object found +// by an ID prefix. const roots: string[] = []; afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index e9f2d917..9aa3212d 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -89,8 +89,9 @@ export function removeRunnerObjects(runnerOwner: string, when = 'afterAll'): voi * or not: a test can make a network or volume with one owner and the container that holds it with another, or without * one. Containers and volumes go by force; a network another container joins meanwhile stays. It reports what it finds * and any Docker call that fails (but not one that fails only because what it names has gone since the listing: a - * holder it reads, or a network it removes), prefixed with `when`, and never throws, so cleanup can call it without hiding an earlier error. Each call has a 30 s - * limit, so all eleven can take up to 5.5 minutes. It runs `docker`, the real one by default (`REAL_DOCKER`). + * holder it reads, or a network it removes), prefixed with `when`, and never throws, so cleanup can call it without + * hiding an earlier error. Each call has a 30 s limit, so all eleven can take up to 5.5 minutes. It runs `docker`, the + * real one by default (`REAL_DOCKER`). */ export function removeLabelledObjects(label: string, value: string, when: string, docker = REAL_DOCKER): void { const run = (stage: string, args: string[]) => { From 07e4a7b0347dc244124d9e6c1e1d3d159298d614 Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 03:40:44 -0700 Subject: [PATCH 58/75] The loaded-daemon fake gives the volume its own holder (#189) Review round 57: the stateless fake listed the network's holder again for the volume, after the sweep had removed it, which Docker would not; the volume now has its own holder, and the test checks its removal. Co-Authored-By: Claude Opus 5.5 --- test/docker-sweep.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index a42fcee6..e1bb1739 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -132,6 +132,7 @@ it('talks to the daemon set when it loaded, though a test changed DOCKER_HOST or it('takes the daemon set at load, not the default, when one was set, for every call', async () => { const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); const seen = join(root, 'seen'), net = 'n'.repeat(64), holder = 'a'.repeat(64), named = 'b'.repeat(64), labelled = 'c'.repeat(64); + const volumeHolder = 'e'.repeat(64); // Records the address and the whole command of each call. Answers as Docker 27.4 does, in the parts the code reads, // for one labelled container, one swept network and one swept volume, each held by one container, and for a container // named x; every other call gets no output and exit 0, and the removals' stdout is not read. The network line holds @@ -141,6 +142,7 @@ it('takes the daemon set at load, not the default, when one was set, for every c `if [ "$1 $2" = "container ls" ]; then printf '%s\\n' ${labelled}; exit 0; fi`, `if [ "$1 $2" = "network ls" ]; then printf '%s\\n' '{"ID":"${net}","Name":"swept"}'; exit 0; fi`, `if [ "$1 $2" = "volume ls" ]; then printf '%s\\n' swept-volume; exit 0; fi`, + `if [ "$1" = ps ] && [ "$last" = volume=swept-volume ]; then printf '%s\\n' ${volumeHolder}; exit 0; fi`, `if [ "$1" = ps ]; then printf '%s\\n' ${holder}; exit 0; fi`, `if [ "$1 $2" = "container inspect" ] && [ "$last" = x ]; then printf '%s\\n' '"${named}" "/x"'; exit 0; fi`, `if [ "$1 $2" = "container inspect" ]; then printf '%s\\n' '"${holder}" {"swept":{}}'; exit 0; fi`, @@ -169,7 +171,7 @@ it('takes the daemon set at load, not the default, when one was set, for every c `container inspect --format {{json .Id}} {{json .NetworkSettings.Networks}} ${holder}`, `rm --force ${holder}`, `network rm ${net}`, `volume ls --quiet --filter label=${TEST_RUN_LABEL}=x`, 'ps --all --quiet --no-trunc --filter volume=swept-volume', - 'volume rm --force swept-volume', + `rm --force ${volumeHolder}`, 'volume rm --force swept-volume', ]) expect(commands).toContain(command); expect(new Set(lines.map(line => `${line[0]}|${line[1]}`))).toEqual(new Set(['unix:///codeboost-loaded.sock|codeboost-loaded'])); }); From 50285e3e64b69b461474d73e0d7b1b2ba07a3a56 Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 03:54:57 -0700 Subject: [PATCH 59/75] Track the supervisor's foreign-name containers for afterAll; stage in unreadable-line reports (#189) Review round 58: the two foreign-name containers are tracked by name for afterAll too, should a create land after their finish-time removal; the sweep's report of a line that is not JSON names its stage. Co-Authored-By: Claude Opus 5.5 --- test/agent-supervisor.test.ts | 8 ++++++-- test/docker-sweep.test.ts | 2 +- test/fixtures/docker-sweep.ts | 9 +++++---- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 2f8b0cde..5f85241d 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -390,7 +390,9 @@ describe('container invocation supervisor', () => { it('releases its own resources, and leaves a same-named container another invocation owns', async () => { const current = await profile(fixture(), 'finite-output', 'foreign-name'); - // Removed by name when the test finishes, timeouts included: its label value is shared across runs (#189). + // Removed by name when the test finishes, timeouts included: its label value is shared across runs (#189). Tracked + // for afterAll too, should its create land after that. + unlabelled.add(current.name); onTestFinished(() => removeByName('container', current.name, 'onTestFinished'), 90_000); // two calls of up to 30 s let passed = false; try { @@ -449,7 +451,9 @@ describe('container invocation supervisor', () => { it('releases only the profile when rejecting before creation, even if its name is held elsewhere', async () => { const current = await profile(fixture(), 'noop'); // A foreign container occupies the deterministic name, so container-level cleanup could never settle. - // Removed by name when the test finishes, timeouts included: its label value is shared across runs (#189). + // Removed by name when the test finishes, timeouts included: its label value is shared across runs (#189). Tracked + // for afterAll too, should its create land after that. + unlabelled.add(current.name); onTestFinished(() => removeByName('container', current.name, 'onTestFinished'), 90_000); // two calls of up to 30 s try { execFileSync('docker', ['create', '--name', current.name, '--label', 'io.codeboost.invocation=someone-else', diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index e1bb1739..b78f5cc0 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -31,7 +31,7 @@ it('skips output that is not a network, reports what is not JSON, and never thro { mode: 0o755 }); const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); expect(() => removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test', join(root, 'docker'))).not.toThrow(); - expect(warn.mock.calls.map(call => String(call[0]))).toContain('test: docker printed a line that is not JSON:'); + expect(warn.mock.calls.map(call => String(call[0]))).toContain('test: listing networks: docker printed a line that is not JSON:'); // Nothing it could not read is removed or looked up as a holder. expect(readFileSync(calls, 'utf8').split('\n').filter(line => /^(ps|rm|network rm|volume rm|container rm|container inspect) /.test(line))).toEqual([]); }); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index 9aa3212d..a3f43bca 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -109,8 +109,8 @@ export function removeLabelledObjects(label: string, value: string, when: string return result.error || result.status === null ? [] : result.stdout.split('\n').filter(Boolean); }; // A line Docker printed that is not JSON is reported and skipped, so the sweep never throws. - const parsed = (text: string): T[] => { - try { return [JSON.parse(text) as T]; } catch { console.warn(`${when}: docker printed a line that is not JSON:`, text); return []; } + const parsed = (stage: string, text: string): T[] => { + try { return [JSON.parse(text) as T]; } catch { console.warn(`${when}: ${stage}: docker printed a line that is not JSON:`, text); return []; } }; for (const kind of ['container', 'network', 'volume'] as const) { // Networks with their names: `ps --filter network=` finds a created, never started container by the name only @@ -119,7 +119,7 @@ export function removeLabelledObjects(label: string, value: string, when: string const listed = run(`listing ${kind}s`, [kind, 'ls', ...(kind === 'container' ? ['--all'] : []), ...(kind === 'network' ? ['--no-trunc', '--format', '{{json .}}'] : kind === 'container' ? ['--quiet', '--no-trunc'] : ['--quiet']), '--filter', `label=${label}=${value}`]) - .flatMap(line => kind === 'network' ? parsed<{ ID?: unknown; Name?: unknown } | null>(line) + .flatMap(line => kind === 'network' ? parsed<{ ID?: unknown; Name?: unknown } | null>('listing networks', line) .filter((object): object is { ID: string; Name: string } => typeof object?.ID === 'string' && typeof object.Name === 'string') : [{ ID: line, Name: line }]); const ids = listed.map(object => object.ID), names = listed.map(object => object.Name); @@ -133,7 +133,8 @@ export function removeLabelledObjects(label: string, value: string, when: string ...names.flatMap(name => ['--filter', `${kind}=${name}`])]); const holders = kind === 'volume' || !candidates.length ? candidates : run('reading holders', ['container', 'inspect', '--format', '{{json .Id}} {{json .NetworkSettings.Networks}}', ...candidates]).flatMap(line => { - const at = line.indexOf(' '), id = parsed(line.slice(0, at)), on = parsed>(line.slice(at + 1)); + const at = line.indexOf(' '), id = parsed('reading holders', line.slice(0, at)), + on = parsed>('reading holders', line.slice(at + 1)); return typeof id[0] === 'string' && typeof on[0] === 'object' ? [{ id: id[0], on: Object.keys(on[0] ?? {}) }] : []; }).filter(holder => holder.on.some(name => names.includes(name))).map(holder => holder.id); if (holders.length) { From 4de7621627cfcfecd3ca0fa61933fd019e2345fb Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 04:11:39 -0700 Subject: [PATCH 60/75] Failure messages name the docker command; record checks first (#189) Review round 59: the killed-create listing and the supervisor's after-pass removal name the command that failed, as main's helper did; the refused-network and refused-proxy tests check their shim's record before the error, so a shim that no longer matches is reported as itself. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 3 ++- test/agent-network.test.ts | 6 ++++-- test/agent-supervisor.test.ts | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 877f4f9a..30cc9db0 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -979,7 +979,8 @@ describe('real Docker agent isolation', () => { const listing = spawnSync('docker', ['network', 'ls', '--quiet', '--filter', `label=io.codeboost.allocation=${allocationId}`], { encoding: 'utf8', ...CLIENT_LIMIT }); // A listing that failed fails here with its cause, as main's helper did. - expect(listing.status, listing.error?.message ?? (listing.signal ? `killed by ${listing.signal}` : listing.stderr)).toBe(0); + expect(listing.status, `docker network ls --filter label=io.codeboost.allocation=${allocationId} failed: ` + + (listing.error?.message ?? (listing.signal ? `killed by ${listing.signal}` : listing.stderr))).toBe(0); const left = listing.stdout.trim(); expect(created).toBeInstanceOf(VendorNetworkCreationCleanupError); const cleanup = (created as VendorNetworkCreationCleanupError).errors[1] as AggregateError; diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index dea0d365..1359df0e 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -140,10 +140,11 @@ describe('vendor-only egress', () => { rmSync(shim, { recursive: true, force: true }); } // The refused create is reported as itself, with no cleanup left to retry, and the foreign network survives. + // First, so a shim that no longer matches is reported as itself. + expect(foreign, 'the shim recorded no foreign network').not.toBe(''); expect(error).toBeInstanceOf(Error); expect(error).not.toBeInstanceOf(VendorNetworkCreationCleanupError); expect(String(error)).toMatch(/already exists/); - expect(foreign, 'the shim recorded no foreign network').not.toBe(''); expect(survived).toBe(true); }, 60_000); @@ -197,10 +198,11 @@ describe('vendor-only egress', () => { rmSync(shim, { recursive: true, force: true }); } // The refused create is reported as itself: our network was removed, nothing is left to retry or report. + // First, so a shim that no longer matches is reported as itself. + expect(foreign, 'the shim recorded no foreign proxy').not.toBe(''); expect(error).toBeInstanceOf(Error); expect(error).not.toBeInstanceOf(VendorNetworkCreationCleanupError); expect(String(error)).toMatch(/already in use|Conflict/); - expect(foreign, 'the shim recorded no foreign proxy').not.toBe(''); expect(survived).toBe(true); }, 60_000); diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 5f85241d..d5abcedd 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -413,7 +413,7 @@ describe('container invocation supervisor', () => { // A failure above stays the one reported, whatever exit status this Docker version gives for a missing name. After // a pass the container exists, and its removal must succeed, as on main. const removed = spawnSync('docker', ['rm', '--force', current.name], { encoding: 'utf8', ...CLIENT_LIMIT }); - if (passed && removed.status !== 0) throw new Error(`Removing ${current.name} failed: ${removed.error?.message ?? (removed.signal ? `killed by ${removed.signal}` : removed.stderr?.trim())}`); + if (passed && removed.status !== 0) throw new Error(`docker rm --force ${current.name} failed: ${removed.error?.message ?? (removed.signal ? `killed by ${removed.signal}` : removed.stderr?.trim())}`); } }, 3 * 60_000); From df7eca1cbc50a0cd8f08d1e2a1f558b5976ff2b4 Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 04:25:07 -0700 Subject: [PATCH 61/75] Exact command in the listing's failure; holder test checks every sweep line (#189) Review round 60: the killed-create listing's failure message is built from the arguments it ran; the holder test requires the gone-holder inspect's exit 1, and accepts only the sweep's own removal lines, including the holders line, so any failure or unreadable answer fails it. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 30cc9db0..c0dacaa6 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -820,7 +820,7 @@ describe('real Docker agent isolation', () => { // And an inspect naming a missing container exits non-zero yet prints the others, the answer the shim below gives. const partial = spawnSync('docker', ['container', 'inspect', '--format', '{{.Id}}', holders[0]!, `codeboost-gone-${randomUUID()}`], { encoding: 'utf8', ...CLIENT_LIMIT }); - expect(partial.status).not.toBe(0); + expect(partial.status).toBe(1); expect(partial.stdout).toContain(holders[0]); expect(partial.stderr).toMatch(/^Error response from daemon: No such container: codeboost-gone-/m); // The holders' inspect also names a container that does not exist, as one gone since the listing would be: Docker @@ -842,8 +842,9 @@ describe('real Docker agent isolation', () => { sweepCalls = [...sweepWarn.mock.calls]; } finally { sweepWarn.mockRestore(); } expect(existsSync(marker), 'the sweep read its holders through the shim').toBe(true); - expect(sweepCalls.filter(call => String(call[0]).endsWith('failed:'))).toEqual([]); - expect(sweepCalls.length).toBeGreaterThan(0); + // Only the sweep's own lines: what it removes. Any failure or unreadable answer is a different line. + expect(sweepCalls.map(call => String(call[0])).filter(line => !/^test removes /.test(line))).toEqual([]); + expect(sweepCalls.some(call => String(call[0]).startsWith('test removes the containers that hold them: '))).toBe(true); // afterAll removes tracked names with removeByName, as here: an object of exactly that name goes, and a name that // is gone says nothing. const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); @@ -976,10 +977,10 @@ describe('real Docker agent isolation', () => { ].join('\n'); const created = await withDockerShim(['network', 'create'], foreign, () => createVendorNetwork(invocation(data.clone, 'planning'), imageId, allocationId).then(() => undefined, error => error)); - const listing = spawnSync('docker', ['network', 'ls', '--quiet', '--filter', `label=io.codeboost.allocation=${allocationId}`], - { encoding: 'utf8', ...CLIENT_LIMIT }); - // A listing that failed fails here with its cause, as main's helper did. - expect(listing.status, `docker network ls --filter label=io.codeboost.allocation=${allocationId} failed: ` + const listArgs = ['network', 'ls', '--quiet', '--filter', `label=io.codeboost.allocation=${allocationId}`]; + const listing = spawnSync('docker', listArgs, { encoding: 'utf8', ...CLIENT_LIMIT }); + // A listing that failed fails here with the command and its cause, as main's helper did. + expect(listing.status, `docker ${listArgs.join(' ')} failed: ` + (listing.error?.message ?? (listing.signal ? `killed by ${listing.signal}` : listing.stderr))).toBe(0); const left = listing.stdout.trim(); expect(created).toBeInstanceOf(VendorNetworkCreationCleanupError); From 0183899562b79007780af146ef4b70100487cf99 Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 04:41:48 -0700 Subject: [PATCH 62/75] Report sweep lines of the wrong shape; exact commands in removeByName's reports (#189) Review round 61: the sweep reports a network listing line that is not a network and a holder line that is not an ID and networks, as removeByName reports an answer that is not an ID and a name, with tests; removeByName's failure reports name the exact command and ID. Co-Authored-By: Claude Opus 5.5 --- test/docker-sweep.test.ts | 33 ++++++++++++++++++++++++++------- test/fixtures/docker-sweep.ts | 22 +++++++++++++++------- 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index b78f5cc0..c390d61b 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -23,7 +23,7 @@ afterEach(() => { // The fake answers as no real daemon does for `network ls`: a line that is not JSON, a JSON null and one network // without a name. To every other call it answers nothing and exit 0, as Docker 27.4 does for `ls` with no match. -it('skips output that is not a network, reports what is not JSON, and never throws', () => { +it('skips and reports output that is not a network, and never throws', () => { const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); const calls = join(root, 'calls'); writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${calls}'`, @@ -31,7 +31,9 @@ it('skips output that is not a network, reports what is not JSON, and never thro { mode: 0o755 }); const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); expect(() => removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test', join(root, 'docker'))).not.toThrow(); - expect(warn.mock.calls.map(call => String(call[0]))).toContain('test: listing networks: docker printed a line that is not JSON:'); + expect(warn.mock.calls).toEqual([['test: listing networks: docker printed a line that is not JSON:', 'not json'], + ['test: listing networks: docker printed a line that is not a network:', 'null'], + ['test: listing networks: docker printed a line that is not a network:', '{"ID":"abc"}']]); // Nothing it could not read is removed or looked up as a holder. expect(readFileSync(calls, 'utf8').split('\n').filter(line => /^(ps|rm|network rm|volume rm|container rm|container inspect) /.test(line))).toEqual([]); }); @@ -97,13 +99,13 @@ it('removes by name only an object of exactly that name, and reports failures ot expect(readFileSync(calls, 'utf8').split('\n').filter(line => / rm |^rm /.test(line))) .toEqual([`rm --force ${id}`, `network rm ${other}`, 'volume rm --force busy', `network rm ${raced}`]); expect(warn.mock.calls).toEqual([ - ['test: removing container busy: docker rm failed:', + [`test: removing container busy: docker rm --force ${id} failed:`, `Error response from daemon: removal of container ${id} is already in progress`], - ['test: removing network busy: docker network rm failed:', + [`test: removing network busy: docker network rm ${other} failed:`, `Error response from daemon: error while removing network: network busy id ${other} has active endpoints`], - ['test: removing volume busy: docker volume rm failed:', `Error response from daemon: remove busy: volume is in use - [${id}]`], - ['test: removing container killed: docker container inspect failed:', 'killed by SIGKILL'], - ['test: removing container lost: docker container inspect failed:', + ['test: removing volume busy: docker volume rm --force busy failed:', `Error response from daemon: remove busy: volume is in use - [${id}]`], + ['test: removing container killed: docker container inspect --format {{json .Id}} {{json .Name}} killed failed:', 'killed by SIGKILL'], + ['test: removing container lost: docker container inspect --format {{json .Id}} {{json .Name}} lost failed:', 'Failed to initialize: unable to resolve docker endpoint: context "x": context not found: open /x/meta.json: no such file or directory'], ['test: removing container junk: docker printed what is not JSON:', 'not json\n'], ['test: removing container odd: docker printed no ID and name:', '1 2\n'], @@ -251,3 +253,20 @@ it('finds the real docker once, at load, uses it by default, and refuses a shim const missing = await import('./fixtures/docker-sweep.ts') as typeof Sweep; expect(() => missing.requireRealDocker()).toThrow('No docker on PATH when the test files loaded.'); }); + +it('reports a holder line that is not an ID and networks, and removes no holder from it', () => { + const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); + const calls = join(root, 'calls'), net = 'd'.repeat(64); + // One swept network with one candidate holder, whose inspect line parses but is not an ID and networks, which Docker + // 27.4 is not known to print. Every other call gets no output and exit 0, as for `ls` with no match. The network line + // holds only the fields the sweep reads. + writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${calls}'`, + `if [ "$1 $2" = "network ls" ]; then printf '%s\\n' '{"ID":"${net}","Name":"swept"}'; exit 0; fi`, + `if [ "$1" = ps ]; then printf '%s\\n' ${'a'.repeat(64)}; exit 0; fi`, + `if [ "$1 $2" = "container inspect" ]; then printf '%s\\n' '123 null'; exit 0; fi`, 'exit 0'].join('\n'), + { mode: 0o755 }); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test', join(root, 'docker')); + expect(warn.mock.calls).toContainEqual(['test: reading holders: docker printed a line that is not an ID and networks:', '123 null']); + expect(readFileSync(calls, 'utf8').split('\n').filter(line => line.startsWith('rm '))).toEqual([]); +}); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index a3f43bca..a0661272 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -49,14 +49,15 @@ export const TEST_FILE_LABEL = 'io.codeboost.test-file'; */ export function removeByName(kind: 'container' | 'network' | 'volume', name: string, when: string, docker = REAL_DOCKER): void { const options = { encoding: 'utf8' as const, timeout: 30_000, killSignal: 'SIGKILL' as const, env: dockerEnv() }; - const failed = (result: SpawnSyncReturns, what: string) => console.warn(`${when}: removing ${kind} ${name}: docker ${what} failed:`, + const failed = (result: SpawnSyncReturns, args: string[]) => console.warn(`${when}: removing ${kind} ${name}: docker ${args.join(' ')} failed:`, result.error?.message ?? (result.signal ? `killed by ${result.signal}` : result.stderr.trim())); // As JSON: a network's name may hold spaces. Only a container's name comes with a leading slash. - const found = spawnSync(docker, [kind, 'inspect', '--format', kind === 'volume' ? '{{json .Name}}' : '{{json .Id}} {{json .Name}}', name], options); + const inspect = [kind, 'inspect', '--format', kind === 'volume' ? '{{json .Name}}' : '{{json .Id}} {{json .Name}}', name]; + const found = spawnSync(docker, inspect, options); // Docker 27.4's answers for a missing object, each naming it, so a failure before the daemon (an unknown context's // "context not found: … no such file or directory") is still reported. const missing = { container: `No such container: ${name}`, network: `network ${name} not found`, volume: `get ${name}: no such volume` }[kind]; - if (found.error || found.status !== 0) { if (found.status === null || !found.stderr.includes(missing)) failed(found, `${kind} inspect`); return; } + if (found.error || found.status !== 0) { if (found.status === null || !found.stderr.includes(missing)) failed(found, inspect); return; } let id: unknown, actual: unknown; try { const at = found.stdout.indexOf(' '); @@ -66,11 +67,12 @@ export function removeByName(kind: 'container' | 'network' | 'volume', name: str console.warn(`${when}: removing ${kind} ${name}: docker printed no ID and name:`, found.stdout); return; } if ((kind === 'container' ? actual.replace(/^\//, '') : actual) !== name) return; - const removed = spawnSync(docker, kind === 'container' ? ['rm', '--force', id] : [kind, 'rm', ...(kind === 'volume' ? ['--force'] : []), id], options); + const remove = kind === 'container' ? ['rm', '--force', id] : [kind, 'rm', ...(kind === 'volume' ? ['--force'] : []), id]; + const removed = spawnSync(docker, remove, options); // A network gone since the inspect is gone: Docker 27.4 answers "network not found" (rm --force of a missing // container or volume exits 0). const raced = kind === 'network' && removed.status === 1 && removed.stderr.includes(`network ${id} not found`); - if ((removed.error || removed.status !== 0) && !raced) failed(removed, kind === 'container' ? 'rm' : `${kind} rm`); + if ((removed.error || removed.status !== 0) && !raced) failed(removed, remove); } /** @@ -120,7 +122,10 @@ export function removeLabelledObjects(label: string, value: string, when: string ? ['--no-trunc', '--format', '{{json .}}'] : kind === 'container' ? ['--quiet', '--no-trunc'] : ['--quiet']), '--filter', `label=${label}=${value}`]) .flatMap(line => kind === 'network' ? parsed<{ ID?: unknown; Name?: unknown } | null>('listing networks', line) - .filter((object): object is { ID: string; Name: string } => typeof object?.ID === 'string' && typeof object.Name === 'string') + .filter((object): object is { ID: string; Name: string } => { + if (typeof object?.ID === 'string' && typeof object.Name === 'string') return true; + console.warn(`${when}: listing networks: docker printed a line that is not a network:`, line); return false; + }) : [{ ID: line, Name: line }]); const ids = listed.map(object => object.ID), names = listed.map(object => object.Name); if (!ids.length) continue; @@ -135,7 +140,10 @@ export function removeLabelledObjects(label: string, value: string, when: string '{{json .Id}} {{json .NetworkSettings.Networks}}', ...candidates]).flatMap(line => { const at = line.indexOf(' '), id = parsed('reading holders', line.slice(0, at)), on = parsed>('reading holders', line.slice(at + 1)); - return typeof id[0] === 'string' && typeof on[0] === 'object' ? [{ id: id[0], on: Object.keys(on[0] ?? {}) }] : []; + if (typeof id[0] === 'string' && typeof on[0] === 'object' && on[0] !== null) return [{ id: id[0], on: Object.keys(on[0]) }]; + // A line already reported as not JSON is not reported again. + if (id.length && on.length) console.warn(`${when}: reading holders: docker printed a line that is not an ID and networks:`, line); + return []; }).filter(holder => holder.on.some(name => names.includes(name))).map(holder => holder.id); if (holders.length) { console.warn(`${when} removes the containers that hold them: ${holders.join(' ')}`); From 3faffdd9556b30fe83fc29523e77478eb2d9c1c8 Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 05:00:41 -0700 Subject: [PATCH 63/75] Gone checks need Docker's missing answer; sweep checks listed IDs and names; full commands (#189) Review round 62: objectExists counts only Docker 27.4's missing answer as gone and throws on any other failure, so a killed inspect no longer passes a removal check (the export, follow-up and holder tests; runner-workspace's volume check wants 'no such volume'); the sweep names whole commands in its reports, reports and skips listed lines that are not a container ID or a volume name, and reports a holder line with no space once; agent-network's records removal cannot stop the sweeps; the keeper test's finally sweep says finally. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 12 +++++------ test/agent-network.test.ts | 2 +- test/docker-sweep.test.ts | 26 ++++++++++++++++++++---- test/fixtures/docker-sweep.ts | 38 +++++++++++++++++++++++++++++------ test/runner-workspace.test.ts | 2 +- 5 files changed, 62 insertions(+), 18 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index c0dacaa6..f5474697 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -1,5 +1,5 @@ import { execFileSync, spawnSync } from 'node:child_process'; -import { CLIENT_LIMIT, removeByName, removeLabelledObjects, removeRunnerObjects, requireRealDocker, TEST_FILE_LABEL, TEST_RUN_LABEL } from './fixtures/docker-sweep.ts'; +import { CLIENT_LIMIT, objectExists, removeByName, removeLabelledObjects, removeRunnerObjects, requireRealDocker, TEST_FILE_LABEL, TEST_RUN_LABEL } from './fixtures/docker-sweep.ts'; import { fixtureGit } from './fixtures/git.ts'; import { createHash, randomBytes, randomUUID } from 'node:crypto'; import { chmodSync, existsSync, mkdtempSync, mkdirSync, readdirSync, readFileSync, realpathSync, renameSync, rmSync, statSync, @@ -850,7 +850,7 @@ describe('real Docker agent isolation', () => { const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); try { for (const [kind, name] of byName) removeByName(kind, name, 'test'); - expect(byName.filter(([kind, name]) => spawnSync('docker', [kind, 'inspect', name], { stdio: 'ignore', ...CLIENT_LIMIT }).status === 0)).toEqual([]); + expect(byName.filter(([kind, name]) => objectExists(kind, name))).toEqual([]); for (const [kind, name] of byName) removeByName(kind, name, 'test'); expect(warn).not.toHaveBeenCalled(); } finally { warn.mockRestore(); } @@ -860,7 +860,7 @@ describe('real Docker agent isolation', () => { removeByName('network', lookalike.slice(0, 12), 'test'); expect(docker('container', 'inspect', '--format', '{{.Id}}', bystanders[0]!.slice(0, 12))).toBe(bystanders[0]); removeByName('container', bystanders[0]!.slice(0, 12), 'test'); - const exists = (kind: string, ref: string) => spawnSync('docker', [kind, 'inspect', ref], { stdio: 'ignore', ...CLIENT_LIMIT }).status === 0; + const exists = objectExists; for (const holder of holders) expect(exists('container', holder)).toBe(false); expect([exists('network', network), exists('network', prefixNamed), exists('volume', volume)]).toEqual([false, false, false]); expect(exists('container', labelled)).toBe(false); @@ -897,7 +897,7 @@ describe('real Docker agent isolation', () => { // First, so afterAll does not refuse a keeper this test finds left: its file-label sweep removes that keeper, then // its runner sweep the volumes. taskFilesystems.splice(taskFilesystems.indexOf(left), 1); - expect(spawnSync('docker', ['container', 'inspect', left.keeper], { stdio: 'ignore', ...CLIENT_LIMIT }).status).not.toBe(0); + expect(objectExists('container', left.keeper)).toBe(false); // Its volumes carry the runner label, so ordinary cleanup can remove them now that no keeper holds them. removeTaskFilesystems(left); expect(byAllocation(allocationId)).toEqual([]); @@ -2568,7 +2568,7 @@ describe('real Docker agent isolation', () => { removeTaskFilesystems(handle); expect(isRecoveredTaskStorage(handle)).toBe(false); for (const [kind, ref] of [['container', left.keeper], ['volume', left.workVolume], ['volume', left.metadataVolume]] as const) - expect(spawnSync('docker', [kind, 'inspect', ref], { stdio: 'ignore', ...CLIENT_LIMIT }).status).not.toBe(0); + expect(objectExists(kind, ref)).toBe(false); }, 180_000); it('never removes a task keeper replaced by another runner, but still removes the owned volumes', () => { @@ -2596,7 +2596,7 @@ describe('real Docker agent isolation', () => { // 5.5 minutes, which the test's budget allows for. let released: unknown; try { removeTaskFilesystems(filesystems); } catch (error) { released = error; console.warn('cleanup: removing the storage failed:', error); } - removeLabelledObjects('io.codeboost.allocation', owner.allocationId, 'test'); + removeLabelledObjects('io.codeboost.allocation', owner.allocationId, 'finally'); const keeperFailure = keeperRemoved.status === 0 ? undefined : `docker rm --force ${filesystems.keeper} failed: ` + (keeperRemoved.error?.message ?? (keeperRemoved.signal ? `killed by ${keeperRemoved.signal}` : keeperRemoved.stderr?.trim())); if (keeperFailure && !passed) console.warn('cleanup:', keeperFailure); diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index 1359df0e..d303c7ff 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -65,7 +65,7 @@ afterAll(async () => { // `unlabelled`. It may not run before a hook timeout. The hook's 13 minutes leave two over the worst case of these // two sweeps (eleven calls of 30 s each), for the cleanup before them. // The records were read above; removed first, so a hook timeout during the sweeps does not leave them. - if (records) rmSync(records, { recursive: true, force: true }); + try { if (records) rmSync(records, { recursive: true, force: true }); } catch (error) { console.warn('afterAll: removing the records failed:', error); } removeLabelledObjects(TEST_FILE_LABEL, TEST_FILE_RUN, 'afterAll'); removeRunnerObjects(TEST_RUNNER_OWNER); } diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index c390d61b..8043933f 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -54,7 +54,8 @@ it('reads nothing from a docker call that was killed, whose output may be cut sh const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test', join(root, 'docker')); expect(readFileSync(calls, 'utf8').split('\n').filter(line => /^(ps|network rm) /.test(line))).toEqual([]); - expect(warn.mock.calls).toContainEqual(['test: listing networks: docker network ls failed:', 'killed by SIGKILL']); + expect(warn.mock.calls).toContainEqual([`test: listing networks: docker network ls --no-trunc --format {{json .}} --filter label=${TEST_RUN_LABEL}=x failed:`, + 'killed by SIGKILL']); }); it('removes by name only an object of exactly that name, and reports failures other than no such object', () => { @@ -195,12 +196,12 @@ it('says nothing of a network gone since the listing, and reports any other fail expect(failures()).toEqual([]); writeFileSync(answer, 'Error response from daemon: permission denied\n'); removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test', join(root, 'docker')); - expect(failures()).toEqual([['test: removing networks: docker network rm failed:', 'Error response from daemon: permission denied']]); + expect(failures()).toEqual([[`test: removing networks: docker network rm ${net} failed:`, 'Error response from daemon: permission denied']]); }); it.each([ ['says nothing of a holder gone since the listing', 'Error response from daemon: No such container: b', []], - ['reports any other failure to read the holders', 'permission denied', [['test: reading holders: docker container inspect failed:', 'permission denied']]], + ['reports any other failure to read the holders', 'permission denied', [['test: reading holders: docker container inspect --format {{json .Id}} {{json .NetworkSettings.Networks}} HOLDERS failed:', 'permission denied']]], ])('%s, and still removes the holders it read', (_label, stderr, warnings) => { const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); const calls = join(root, 'calls'), net = 'n'.repeat(64), a = 'a'.repeat(64), b = 'b'.repeat(64); @@ -219,7 +220,8 @@ it.each([ { mode: 0o755 }); const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test', join(root, 'docker')); - expect(warn.mock.calls.filter(call => String(call[0]).endsWith('failed:'))).toEqual(warnings); + expect(warn.mock.calls.filter(call => String(call[0]).endsWith('failed:'))) + .toEqual(warnings.map(([line, cause]) => [line!.replace('HOLDERS', `${a} ${b}`), cause])); expect(readFileSync(calls, 'utf8').split('\n')).toContain(`rm --force ${a}`); }); @@ -270,3 +272,19 @@ it('reports a holder line that is not an ID and networks, and removes no holder expect(warn.mock.calls).toContainEqual(['test: reading holders: docker printed a line that is not an ID and networks:', '123 null']); expect(readFileSync(calls, 'utf8').split('\n').filter(line => line.startsWith('rm '))).toEqual([]); }); + +it('reports and skips listed lines that are not a container ID or a volume name, so none becomes an option', () => { + const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); + const calls = join(root, 'calls'); + // Lines Docker 27.4 is not known to print for these listings; every other call gets no output and exit 0, as for + // `ls` with no match. + writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${calls}'`, + `if [ "$1 $2" = "container ls" ]; then printf '%s\\n' '--all'; exit 0; fi`, + `if [ "$1 $2" = "volume ls" ]; then printf '%s\\n' '-v'; exit 0; fi`, 'exit 0'].join('\n'), + { mode: 0o755 }); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test', join(root, 'docker')); + expect(warn.mock.calls).toEqual([['test: listing containers: docker printed a line that is not a container ID:', '--all'], + ['test: listing volumes: docker printed a line that is not a volume name:', '-v']]); + expect(readFileSync(calls, 'utf8').split('\n').filter(line => / rm |^rm |^ps /.test(line))).toEqual([]); +}); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index a0661272..f4770139 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -40,6 +40,22 @@ export const TEST_RUN_LABEL = 'io.codeboost.test-run'; /** Beside it, one value per test file run, so afterAll removes them all with one sweep. */ export const TEST_FILE_LABEL = 'io.codeboost.test-file'; +/** Docker 27.4's answer to an inspect of a missing object, which names it. */ +const missingAnswer = (kind: 'container' | 'network' | 'volume', ref: string) => + ({ container: `No such container: ${ref}`, network: `network ${ref} not found`, volume: `get ${ref}: no such volume` })[kind]; +/** + * Whether the container, network or volume `ref` exists, for a test's checks: only Docker's own missing answer counts as + * gone, and any other failure (a killed call, a daemon that cannot be reached) throws, naming the command. + */ +export function objectExists(kind: 'container' | 'network' | 'volume', ref: string): boolean { + const args = [kind, 'inspect', ref]; + const found = spawnSync(REAL_DOCKER, args, { encoding: 'utf8', ...CLIENT_LIMIT, env: dockerEnv() }); + if (found.status === 0) return true; + if (found.status === 1 && found.stderr.includes(missingAnswer(kind, ref))) return false; + throw new Error(`docker ${args.join(' ')} failed: ` + + (found.error?.message ?? (found.signal ? `killed by ${found.signal}` : found.stderr.trim()))); +} + /** * Removes the container, network or volume named exactly `name`, by its full ID (a volume by its name, which is its * ID). Docker would read a missing name as an ID prefix, so an object found under another name is left alone. It @@ -56,7 +72,7 @@ export function removeByName(kind: 'container' | 'network' | 'volume', name: str const found = spawnSync(docker, inspect, options); // Docker 27.4's answers for a missing object, each naming it, so a failure before the daemon (an unknown context's // "context not found: … no such file or directory") is still reported. - const missing = { container: `No such container: ${name}`, network: `network ${name} not found`, volume: `get ${name}: no such volume` }[kind]; + const missing = missingAnswer(kind, name); if (found.error || found.status !== 0) { if (found.status === null || !found.stderr.includes(missing)) failed(found, inspect); return; } let id: unknown, actual: unknown; try { @@ -85,6 +101,8 @@ export function removeRunnerObjects(runnerOwner: string, when = 'afterAll'): voi removeLabelledObjects(RUNNER_LABEL, runnerOwner, when); } +// What `--quiet --no-trunc` lists for a container, and the names Docker 27.4 allows a volume. +const CONTAINER_ID = /^[0-9a-f]{64}$/, VOLUME_NAME = /^[a-zA-Z0-9][a-zA-Z0-9_.-]*$/; /** * Removes the containers, networks and volumes that carry `label=value`, containers first, as they hold the others in * use. Before it removes a network or a volume it also removes every container attached to it or mounting it, labelled @@ -104,12 +122,19 @@ export function removeLabelledObjects(label: string, value: string, when: string const gone = result.status === 1 && (stage === 'reading holders' ? errors.every(line => line.startsWith('Error response from daemon: No such container: ')) : stage === 'removing networks' && errors.every(line => /^Error response from daemon: network [0-9a-f]{64} not found$/.test(line))); - if (result.status !== 0 && !gone) console.warn(`${when}: ${stage}: docker ${args.slice(0, 2).join(' ')} failed:`, + if (result.status !== 0 && !gone) console.warn(`${when}: ${stage}: docker ${args.join(' ')} failed:`, result.error?.message ?? (result.signal ? `killed by ${result.signal}` : result.stderr.trim())); // An inspect of several objects exits 1 if one is gone, yet prints the rest (Docker 27.4), so the output of a call // that exited by itself is read whatever its status. A killed call's output may be cut short, so it is not. return result.error || result.status === null ? [] : result.stdout.split('\n').filter(Boolean); }; + // A container ID line or a volume name line of another shape is reported and skipped: it would be passed on as an + // argument, where a leading dash would read as an option. + const shaped = (stage: string, pattern: RegExp) => (line: string) => { + if (pattern.test(line)) return true; + console.warn(`${when}: ${stage}: docker printed a line that is not ${pattern === CONTAINER_ID ? 'a container ID' : 'a volume name'}:`, line); + return false; + }; // A line Docker printed that is not JSON is reported and skipped, so the sweep never throws. const parsed = (stage: string, text: string): T[] => { try { return [JSON.parse(text) as T]; } catch { console.warn(`${when}: ${stage}: docker printed a line that is not JSON:`, text); return []; } @@ -126,7 +151,7 @@ export function removeLabelledObjects(label: string, value: string, when: string if (typeof object?.ID === 'string' && typeof object.Name === 'string') return true; console.warn(`${when}: listing networks: docker printed a line that is not a network:`, line); return false; }) - : [{ ID: line, Name: line }]); + : shaped(`listing ${kind}s`, kind === 'container' ? CONTAINER_ID : VOLUME_NAME)(line) ? [{ ID: line, Name: line }] : []); const ids = listed.map(object => object.ID), names = listed.map(object => object.Name); if (!ids.length) continue; console.warn(`${when} removes leftover ${kind}s labelled ${label}=${value}: ${names.join(' ')}`); @@ -135,11 +160,12 @@ export function removeLabelledObjects(label: string, value: string, when: string // ID, so a container counts only if it is on one of these networks by name (Docker refuses a second local network // of one name). Containers go by full ID: Docker resolves a name before an ID prefix. const candidates = run('finding holders', ['ps', '--all', '--quiet', '--no-trunc', - ...names.flatMap(name => ['--filter', `${kind}=${name}`])]); + ...names.flatMap(name => ['--filter', `${kind}=${name}`])]).filter(shaped('finding holders', CONTAINER_ID)); const holders = kind === 'volume' || !candidates.length ? candidates : run('reading holders', ['container', 'inspect', '--format', '{{json .Id}} {{json .NetworkSettings.Networks}}', ...candidates]).flatMap(line => { - const at = line.indexOf(' '), id = parsed('reading holders', line.slice(0, at)), - on = parsed>('reading holders', line.slice(at + 1)); + const at = line.indexOf(' '); + if (at < 0) { console.warn(`${when}: reading holders: docker printed a line that is not an ID and networks:`, line); return []; } + const id = parsed('reading holders', line.slice(0, at)), on = parsed>('reading holders', line.slice(at + 1)); if (typeof id[0] === 'string' && typeof on[0] === 'object' && on[0] !== null) return [{ id: id[0], on: Object.keys(on[0]) }]; // A line already reported as not JSON is not reported again. if (id.length && on.length) console.warn(`${when}: reading holders: docker printed a line that is not an ID and networks:`, line); diff --git a/test/runner-workspace.test.ts b/test/runner-workspace.test.ts index ee0fbafe..9983f7d1 100644 --- a/test/runner-workspace.test.ts +++ b/test/runner-workspace.test.ts @@ -235,7 +235,7 @@ describe('task storage path identity (#162)', () => { expect(probe.volume).toMatch(/^[0-9a-f]{64}$/); expect(docker('volume', 'inspect', '--format', '{{json .Options}}', probe.volume)).toContain('size=1048576,nr_inodes=64,'); } finally { if (probe) docker('rm', '--force', '--volumes', probe.name); } - expect(() => docker('volume', 'inspect', probe.volume)).toThrow(); + expect(() => docker('volume', 'inspect', probe.volume)).toThrow(/no such volume/i); }, 120_000); it('removes a probe a killed runner left, with its volume, in the next startup\'s recovery, and no other runner\'s (#181)', async () => { // Tokens of their own: recovery removes every agent container of its runner, and this file's may have some. From 96cf19ad83586aa492c03c9b1dbb4833e5723b77 Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 05:16:06 -0700 Subject: [PATCH 64/75] agent-network's survival checks use objectExists (#189) Review round 63: a killed or failed inspect in the finally no longer reads as the foreign object being removed; Docker's missing answer counts as gone, and any other failure is rethrown, naming the command, after the body's own checks. Co-Authored-By: Claude Opus 5.5 --- test/agent-network.test.ts | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index d303c7ff..f74d8ad4 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -9,7 +9,7 @@ import { assertVendorNetwork, createVendorNetwork, removeVendorNetwork, VENDOR_H VendorNetworkCreationCleanupError, type VendorNetwork } from '../agents/network/network.ts'; import { captureInvocation, type InvocationInput } from '../agents/contract.ts'; -import { CLIENT_LIMIT, removeByName, removeLabelledObjects, removeRunnerObjects, requireRealDocker, TEST_FILE_LABEL } from './fixtures/docker-sweep.ts'; +import { CLIENT_LIMIT, objectExists, removeByName, removeLabelledObjects, removeRunnerObjects, requireRealDocker, TEST_FILE_LABEL } from './fixtures/docker-sweep.ts'; // Each run has its own runner owner, so the sweep after this file removes only what this run left behind. Network and // proxy names are random, so the fixed attempt IDs below reach only labels, where this owner tells runs apart. const TEST_RUNNER_OWNER = randomBytes(16).toString('hex'); @@ -126,7 +126,7 @@ describe('vendor-only egress', () => { deadline: Date.now() + 60_000 }); const path = process.env.PATH; process.env.PATH = `${shim}:${path}`; - let foreign = '', error: unknown, survived = false; + let foreign = '', error: unknown, survived: boolean | Error = false; try { try { await createVendorNetwork(refused, imageId, randomUUID()); } catch (caught) { error = caught; } } finally { @@ -135,7 +135,8 @@ describe('vendor-only egress', () => { const made = existsSync(recorded) ? readFileSync(recorded, 'utf8').split('\n').filter(Boolean) : []; foreign = made[0] ?? ''; // afterAll reads the record again: the object may land after this removal (#189). - if (foreign) survived = spawnSync('docker', ['network', 'inspect', foreign], { stdio: 'ignore', ...CLIENT_LIMIT }).status === 0; + // Only Docker's own missing answer counts as gone; any other failure is kept and reported below, after the body's. + if (foreign) try { survived = objectExists('network', foreign); } catch (caught) { survived = caught as Error; } for (const name of made) removeByName('network', name, 'finally'); rmSync(shim, { recursive: true, force: true }); } @@ -145,6 +146,7 @@ describe('vendor-only egress', () => { expect(error).toBeInstanceOf(Error); expect(error).not.toBeInstanceOf(VendorNetworkCreationCleanupError); expect(String(error)).toMatch(/already exists/); + if (survived instanceof Error) throw survived; expect(survived).toBe(true); }, 60_000); @@ -184,7 +186,7 @@ describe('vendor-only egress', () => { deadline: Date.now() + 60_000 }); const path = process.env.PATH; process.env.PATH = `${shim}:${path}`; - let foreign = '', error: unknown, survived = false; + let foreign = '', error: unknown, survived: boolean | Error = false; try { try { await createVendorNetwork(refused, imageId, randomUUID()); } catch (caught) { error = caught; } } finally { @@ -193,7 +195,8 @@ describe('vendor-only egress', () => { const made = existsSync(recorded) ? readFileSync(recorded, 'utf8').split('\n').filter(Boolean) : []; foreign = made[0] ?? ''; // afterAll reads the record again: the object may land after this removal (#189). - if (foreign) survived = spawnSync('docker', ['container', 'inspect', foreign], { stdio: 'ignore', ...CLIENT_LIMIT }).status === 0; + // Only Docker's own missing answer counts as gone; any other failure is kept and reported below, after the body's. + if (foreign) try { survived = objectExists('container', foreign); } catch (caught) { survived = caught as Error; } for (const name of made) removeByName('container', name, 'finally'); rmSync(shim, { recursive: true, force: true }); } @@ -203,6 +206,7 @@ describe('vendor-only egress', () => { expect(error).toBeInstanceOf(Error); expect(error).not.toBeInstanceOf(VendorNetworkCreationCleanupError); expect(String(error)).toMatch(/already in use|Conflict/); + if (survived instanceof Error) throw survived; expect(survived).toBe(true); }, 60_000); @@ -242,7 +246,7 @@ describe('vendor-only egress', () => { deadline: Date.now() + 60_000 }); const path = process.env.PATH; process.env.PATH = `${shim}:${path}`; - let impostor = '', survived = false; + let impostor = '', survived: boolean | Error = false; try { await expect(createVendorNetwork(failing, imageId, randomUUID())).rejects.toThrow(); } finally { @@ -251,11 +255,13 @@ describe('vendor-only egress', () => { const made = existsSync(recorded) ? readFileSync(recorded, 'utf8').split('\n').filter(Boolean) : []; impostor = made[0] ?? ''; // afterAll reads the record again: the object may land after this removal (#189). - if (impostor) survived = spawnSync('docker', ['container', 'inspect', impostor], { stdio: 'ignore', ...CLIENT_LIMIT }).status === 0; + // Only Docker's own missing answer counts as gone; any other failure is kept and reported below, after the body's. + if (impostor) try { survived = objectExists('container', impostor); } catch (caught) { survived = caught as Error; } for (const name of made) removeByName('container', name, 'finally'); rmSync(shim, { recursive: true, force: true }); } expect(impostor).toMatch(/^codeboost-proxy-/); + if (survived instanceof Error) throw survived; expect(survived).toBe(true); }, 60_000); From c23d896277a3413a5e71a9bc2c9a8960d3512c22 Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 05:32:34 -0700 Subject: [PATCH 65/75] Every ID passed on as an argument has a checked shape (#189) Review round 64: network IDs from the listing, holder IDs and removeByName's inspected IDs must be full IDs, or are reported and skipped, so none reads as an option; an inspect line with no space is reported instead of parsed twice; fake cases for each. Co-Authored-By: Claude Opus 5.5 --- test/docker-sweep.test.ts | 19 +++++++++++++------ test/fixtures/docker-sweep.ts | 20 ++++++++++++-------- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index 8043933f..4a79c7b2 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -82,6 +82,8 @@ it('removes by name only an object of exactly that name, and reports failures ot ` 'network raced') printf '%s\\n' '"${raced}" "raced"';;`, " 'container junk') echo 'not json';;", " 'container odd') echo '1 2';;", + " 'container nospace') echo '\"nospace\"';;", + " 'container optid') echo '\"--help\" \"/optid\"';;", " 'container killed') kill -9 $$;;", " 'container lost') echo 'Failed to initialize: unable to resolve docker endpoint: context \"x\": context not found: open /x/meta.json: no such file or directory' >&2; exit 1;;", 'esac; exit 0; fi', @@ -94,7 +96,7 @@ it('removes by name only an object of exactly that name, and reports failures ot const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); const objects = [['container', 'gone'], ['network', 'gone'], ['volume', 'gone'], ['container', 'prefix'], ['network', 'prefix'], ['container', 'busy'], ['network', 'busy'], ['volume', 'busy'], ['container', 'killed'], ['container', 'lost'], - ['network', 'raced'], ['container', 'junk'], ['container', 'odd']] as const; + ['network', 'raced'], ['container', 'junk'], ['container', 'odd'], ['container', 'nospace'], ['container', 'optid']] as const; expect(() => { for (const [kind, name] of objects) removeByName(kind, name, 'test', join(root, 'docker')); }).not.toThrow(); // Only the exact names are removed, by full ID (a volume's name is its ID). expect(readFileSync(calls, 'utf8').split('\n').filter(line => / rm |^rm /.test(line))) @@ -110,6 +112,8 @@ it('removes by name only an object of exactly that name, and reports failures ot 'Failed to initialize: unable to resolve docker endpoint: context "x": context not found: open /x/meta.json: no such file or directory'], ['test: removing container junk: docker printed what is not JSON:', 'not json\n'], ['test: removing container odd: docker printed no ID and name:', '1 2\n'], + ['test: removing container nospace: docker printed what is not JSON:', '"nospace"\n'], + ['test: removing container optid: docker printed no ID and name:', '"--help" "/optid"\n'], ]); }); @@ -134,7 +138,7 @@ it('talks to the daemon set when it loaded, though a test changed DOCKER_HOST or it('takes the daemon set at load, not the default, when one was set, for every call', async () => { const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); - const seen = join(root, 'seen'), net = 'n'.repeat(64), holder = 'a'.repeat(64), named = 'b'.repeat(64), labelled = 'c'.repeat(64); + const seen = join(root, 'seen'), net = 'f'.repeat(64), holder = 'a'.repeat(64), named = 'b'.repeat(64), labelled = 'c'.repeat(64); const volumeHolder = 'e'.repeat(64); // Records the address and the whole command of each call. Answers as Docker 27.4 does, in the parts the code reads, // for one labelled container, one swept network and one swept volume, each held by one container, and for a container @@ -204,7 +208,7 @@ it.each([ ['reports any other failure to read the holders', 'permission denied', [['test: reading holders: docker container inspect --format {{json .Id}} {{json .NetworkSettings.Networks}} HOLDERS failed:', 'permission denied']]], ])('%s, and still removes the holders it read', (_label, stderr, warnings) => { const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); - const calls = join(root, 'calls'), net = 'n'.repeat(64), a = 'a'.repeat(64), b = 'b'.repeat(64); + const calls = join(root, 'calls'), net = 'f'.repeat(64), a = 'a'.repeat(64), b = 'b'.repeat(64); // Answers as Docker 27.4 does for one swept network with two candidate holders, one gone before the inspect: the // inspect prints the other and exits 1. Every other call gets no output and exit 0, as for `ls` with no match; the // removals' stdout (the removed IDs or names in 27.4) is not read. The second case's answer is invented: it stands in @@ -265,26 +269,29 @@ it('reports a holder line that is not an ID and networks, and removes no holder writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${calls}'`, `if [ "$1 $2" = "network ls" ]; then printf '%s\\n' '{"ID":"${net}","Name":"swept"}'; exit 0; fi`, `if [ "$1" = ps ]; then printf '%s\\n' ${'a'.repeat(64)}; exit 0; fi`, - `if [ "$1 $2" = "container inspect" ]; then printf '%s\\n' '123 null'; exit 0; fi`, 'exit 0'].join('\n'), + `if [ "$1 $2" = "container inspect" ]; then printf '%s\\n' '123 null' '"--all" {"swept":{}}'; exit 0; fi`, 'exit 0'].join('\n'), { mode: 0o755 }); const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test', join(root, 'docker')); expect(warn.mock.calls).toContainEqual(['test: reading holders: docker printed a line that is not an ID and networks:', '123 null']); + expect(warn.mock.calls).toContainEqual(['test: reading holders: docker printed a line that is not an ID and networks:', '"--all" {"swept":{}}']); expect(readFileSync(calls, 'utf8').split('\n').filter(line => line.startsWith('rm '))).toEqual([]); }); -it('reports and skips listed lines that are not a container ID or a volume name, so none becomes an option', () => { +it('reports and skips listed lines that are not a container ID, a network or a volume name, so none becomes an option', () => { const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); const calls = join(root, 'calls'); // Lines Docker 27.4 is not known to print for these listings; every other call gets no output and exit 0, as for // `ls` with no match. writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${calls}'`, `if [ "$1 $2" = "container ls" ]; then printf '%s\\n' '--all'; exit 0; fi`, - `if [ "$1 $2" = "volume ls" ]; then printf '%s\\n' '-v'; exit 0; fi`, 'exit 0'].join('\n'), + `if [ "$1 $2" = "volume ls" ]; then printf '%s\\n' '-v'; exit 0; fi`, + `if [ "$1 $2" = "network ls" ]; then printf '%s\\n' '{"ID":"--help","Name":"x"}'; exit 0; fi`, 'exit 0'].join('\n'), { mode: 0o755 }); const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test', join(root, 'docker')); expect(warn.mock.calls).toEqual([['test: listing containers: docker printed a line that is not a container ID:', '--all'], + ['test: listing networks: docker printed a line that is not a network:', '{"ID":"--help","Name":"x"}'], ['test: listing volumes: docker printed a line that is not a volume name:', '-v']]); expect(readFileSync(calls, 'utf8').split('\n').filter(line => / rm |^rm |^ps /.test(line))).toEqual([]); }); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index f4770139..a1f2fb43 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -77,9 +77,11 @@ export function removeByName(kind: 'container' | 'network' | 'volume', name: str let id: unknown, actual: unknown; try { const at = found.stdout.indexOf(' '); + if (kind !== 'volume' && at < 0) throw new Error('no space'); [id, actual] = kind === 'volume' ? [name, JSON.parse(found.stdout)] : [JSON.parse(found.stdout.slice(0, at)), JSON.parse(found.stdout.slice(at + 1))]; } catch { console.warn(`${when}: removing ${kind} ${name}: docker printed what is not JSON:`, found.stdout); return; } - if (typeof id !== 'string' || typeof actual !== 'string') { + // A container's or network's ID is checked too, as it is passed on as an argument. + if (typeof id !== 'string' || typeof actual !== 'string' || (kind !== 'volume' && !OBJECT_ID.test(id))) { console.warn(`${when}: removing ${kind} ${name}: docker printed no ID and name:`, found.stdout); return; } if ((kind === 'container' ? actual.replace(/^\//, '') : actual) !== name) return; @@ -101,8 +103,9 @@ export function removeRunnerObjects(runnerOwner: string, when = 'afterAll'): voi removeLabelledObjects(RUNNER_LABEL, runnerOwner, when); } -// What `--quiet --no-trunc` lists for a container, and the names Docker 27.4 allows a volume. -const CONTAINER_ID = /^[0-9a-f]{64}$/, VOLUME_NAME = /^[a-zA-Z0-9][a-zA-Z0-9_.-]*$/; +// A container's or network's full ID, as `--no-trunc` and inspect give it, and the names Docker 27.4 allows a volume. +// Every value the code passes on as an argument has one of these shapes, so none can read as an option. +const OBJECT_ID = /^[0-9a-f]{64}$/, VOLUME_NAME = /^[a-zA-Z0-9][a-zA-Z0-9_.-]*$/; /** * Removes the containers, networks and volumes that carry `label=value`, containers first, as they hold the others in * use. Before it removes a network or a volume it also removes every container attached to it or mounting it, labelled @@ -132,7 +135,7 @@ export function removeLabelledObjects(label: string, value: string, when: string // argument, where a leading dash would read as an option. const shaped = (stage: string, pattern: RegExp) => (line: string) => { if (pattern.test(line)) return true; - console.warn(`${when}: ${stage}: docker printed a line that is not ${pattern === CONTAINER_ID ? 'a container ID' : 'a volume name'}:`, line); + console.warn(`${when}: ${stage}: docker printed a line that is not ${pattern === OBJECT_ID ? 'a container ID' : 'a volume name'}:`, line); return false; }; // A line Docker printed that is not JSON is reported and skipped, so the sweep never throws. @@ -148,10 +151,10 @@ export function removeLabelledObjects(label: string, value: string, when: string '--filter', `label=${label}=${value}`]) .flatMap(line => kind === 'network' ? parsed<{ ID?: unknown; Name?: unknown } | null>('listing networks', line) .filter((object): object is { ID: string; Name: string } => { - if (typeof object?.ID === 'string' && typeof object.Name === 'string') return true; + if (typeof object?.ID === 'string' && OBJECT_ID.test(object.ID) && typeof object.Name === 'string') return true; console.warn(`${when}: listing networks: docker printed a line that is not a network:`, line); return false; }) - : shaped(`listing ${kind}s`, kind === 'container' ? CONTAINER_ID : VOLUME_NAME)(line) ? [{ ID: line, Name: line }] : []); + : shaped(`listing ${kind}s`, kind === 'container' ? OBJECT_ID : VOLUME_NAME)(line) ? [{ ID: line, Name: line }] : []); const ids = listed.map(object => object.ID), names = listed.map(object => object.Name); if (!ids.length) continue; console.warn(`${when} removes leftover ${kind}s labelled ${label}=${value}: ${names.join(' ')}`); @@ -160,13 +163,14 @@ export function removeLabelledObjects(label: string, value: string, when: string // ID, so a container counts only if it is on one of these networks by name (Docker refuses a second local network // of one name). Containers go by full ID: Docker resolves a name before an ID prefix. const candidates = run('finding holders', ['ps', '--all', '--quiet', '--no-trunc', - ...names.flatMap(name => ['--filter', `${kind}=${name}`])]).filter(shaped('finding holders', CONTAINER_ID)); + ...names.flatMap(name => ['--filter', `${kind}=${name}`])]).filter(shaped('finding holders', OBJECT_ID)); const holders = kind === 'volume' || !candidates.length ? candidates : run('reading holders', ['container', 'inspect', '--format', '{{json .Id}} {{json .NetworkSettings.Networks}}', ...candidates]).flatMap(line => { const at = line.indexOf(' '); if (at < 0) { console.warn(`${when}: reading holders: docker printed a line that is not an ID and networks:`, line); return []; } const id = parsed('reading holders', line.slice(0, at)), on = parsed>('reading holders', line.slice(at + 1)); - if (typeof id[0] === 'string' && typeof on[0] === 'object' && on[0] !== null) return [{ id: id[0], on: Object.keys(on[0]) }]; + if (typeof id[0] === 'string' && OBJECT_ID.test(id[0]) && typeof on[0] === 'object' && on[0] !== null) + return [{ id: id[0], on: Object.keys(on[0]) }]; // A line already reported as not JSON is not reported again. if (id.length && on.length) console.warn(`${when}: reading holders: docker printed a line that is not an ID and networks:`, line); return []; From 43a451044d793f22bf5d9b5ca1d460b915c984a5 Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 05:46:56 -0700 Subject: [PATCH 66/75] removeByName refuses a name that could read as an option; fake notes (#189) Review round 65: removeByName checks the name it passes on (a volume's against Docker's allowed shape, others for a leading dash) and reports a refusal, with fake cases; the volume-name shape needs two characters, as Docker 27.4 does; agent-network's two rewritten shims say which answers are Docker's and which are invented. Co-Authored-By: Claude Opus 5.5 --- test/agent-network.test.ts | 6 ++++-- test/docker-sweep.test.ts | 5 ++++- test/fixtures/docker-sweep.ts | 9 +++++++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index f74d8ad4..3336d83d 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -116,7 +116,8 @@ describe('vendor-only egress', () => { const shim = mkdtempSync(join(tmpdir(), 'docker-shim-')), recorded = recordFile(); unlabelled.push({ kind: 'network', file: recorded }); const realDocker = requireRealDocker(); - // The name is taken by someone else's network just before our create, so the daemon refuses it. + // The name is taken by someone else's network just before our create, so the daemon refuses it. The refusal is + // Docker 27.4's own text, and the shim makes the network with the real docker. writeFileSync(join(shim, 'docker'), ['#!/bin/sh', `if [ "$1" = network ] && [ "$2" = create ]; then for arg; do last="$arg"; done; ` + `printf '%s\\n' "$last" >> '${recorded}'; '${realDocker}' network create "$last" >/dev/null; ` @@ -232,7 +233,8 @@ describe('vendor-only egress', () => { unlabelled.push({ kind: 'container', file: recorded }); const realDocker = requireRealDocker(); // The readiness exec swaps the proxy for a same-named, same-labelled stand-in, then fails setup. Only an exec in a - // proxy: any other call passes through, so the shim never makes a stand-in with an empty, random name. + // proxy: any other call passes through, so the shim never makes a stand-in with an empty, random name. Its exit 1 + // with no stderr is invented, as an exec that failed; the code reads only the status. writeFileSync(join(shim, 'docker'), ['#!/bin/sh', 'if [ "$1" = exec ]; then', ` name=$('${realDocker}' inspect -f '{{.Name}}' "$2" 2>/dev/null | sed 's#^/##')`, ' case "$name" in codeboost-proxy-*)', diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index 4a79c7b2..48744a12 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -96,7 +96,8 @@ it('removes by name only an object of exactly that name, and reports failures ot const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); const objects = [['container', 'gone'], ['network', 'gone'], ['volume', 'gone'], ['container', 'prefix'], ['network', 'prefix'], ['container', 'busy'], ['network', 'busy'], ['volume', 'busy'], ['container', 'killed'], ['container', 'lost'], - ['network', 'raced'], ['container', 'junk'], ['container', 'odd'], ['container', 'nospace'], ['container', 'optid']] as const; + ['network', 'raced'], ['container', 'junk'], ['container', 'odd'], ['container', 'nospace'], ['container', 'optid'], + ['container', '--help'], ['volume', '-v']] as const; expect(() => { for (const [kind, name] of objects) removeByName(kind, name, 'test', join(root, 'docker')); }).not.toThrow(); // Only the exact names are removed, by full ID (a volume's name is its ID). expect(readFileSync(calls, 'utf8').split('\n').filter(line => / rm |^rm /.test(line))) @@ -114,6 +115,8 @@ it('removes by name only an object of exactly that name, and reports failures ot ['test: removing container odd: docker printed no ID and name:', '1 2\n'], ['test: removing container nospace: docker printed what is not JSON:', '"nospace"\n'], ['test: removing container optid: docker printed no ID and name:', '"--help" "/optid"\n'], + ['test: removing container --help: refused a name that is not one Docker gives'], + ['test: removing volume -v: refused a name that is not one Docker gives'], ]); }); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index a1f2fb43..e1354448 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -67,6 +67,10 @@ export function removeByName(kind: 'container' | 'network' | 'volume', name: str const options = { encoding: 'utf8' as const, timeout: 30_000, killSignal: 'SIGKILL' as const, env: dockerEnv() }; const failed = (result: SpawnSyncReturns, args: string[]) => console.warn(`${when}: removing ${kind} ${name}: docker ${args.join(' ')} failed:`, result.error?.message ?? (result.signal ? `killed by ${result.signal}` : result.stderr.trim())); + // The name is passed on too: refused if it could read as an option (a volume's must have the shape Docker allows). + if (kind === 'volume' ? !VOLUME_NAME.test(name) : name.startsWith('-') || !name) { + console.warn(`${when}: removing ${kind} ${name}: refused a name that is not one Docker gives`); return; + } // As JSON: a network's name may hold spaces. Only a container's name comes with a leading slash. const inspect = [kind, 'inspect', '--format', kind === 'volume' ? '{{json .Name}}' : '{{json .Id}} {{json .Name}}', name]; const found = spawnSync(docker, inspect, options); @@ -103,9 +107,10 @@ export function removeRunnerObjects(runnerOwner: string, when = 'afterAll'): voi removeLabelledObjects(RUNNER_LABEL, runnerOwner, when); } -// A container's or network's full ID, as `--no-trunc` and inspect give it, and the names Docker 27.4 allows a volume. +// A container's or network's full ID, as `--no-trunc` and inspect give it, and the names Docker 27.4 allows a volume +// (two characters at least). // Every value the code passes on as an argument has one of these shapes, so none can read as an option. -const OBJECT_ID = /^[0-9a-f]{64}$/, VOLUME_NAME = /^[a-zA-Z0-9][a-zA-Z0-9_.-]*$/; +const OBJECT_ID = /^[0-9a-f]{64}$/, VOLUME_NAME = /^[a-zA-Z0-9][a-zA-Z0-9_.-]+$/; /** * Removes the containers, networks and volumes that carry `label=value`, containers first, as they hold the others in * use. Before it removes a network or a volume it also removes every container attached to it or mounting it, labelled From 0fd77f778da7952bd64b12ccb84d8c6802e72809 Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 06:03:09 -0700 Subject: [PATCH 67/75] Test objectExists; gone cases reach their inspect; per-record errors in afterAll (#189) Review round 66: a unit test loads the fixture with a stand-in docker and checks objectExists is true, false only on each kind's missing answer, and throws naming the command on a killed call or an unknown context, at the loaded daemon; removeByName's gone cases assert their inspect ran; the holder fake's missing answer names the full ID; agent-network's afterAll reports a record it cannot read and goes on. Co-Authored-By: Claude Opus 5.5 --- test/agent-network.test.ts | 7 +++++-- test/docker-sweep.test.ts | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index 3336d83d..3f9b9f03 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -53,8 +53,11 @@ afterAll(async () => { try { for (const { kind, name, file } of unlabelled) { // A record holds one name per line: should a test time out with its shim still on PATH, later creates add theirs. - for (const ref of name ? [name] : file && existsSync(file) ? readFileSync(file, 'utf8').split('\n').filter(Boolean) : []) - removeByName(kind, ref, 'afterAll'); + // A record that cannot be read is reported, and the rest still run. + let refs: string[] = []; + try { refs = name ? [name] : file && existsSync(file) ? readFileSync(file, 'utf8').split('\n').filter(Boolean) : []; } + catch (error) { console.warn('afterAll: reading a record failed:', error); } + for (const ref of refs) removeByName(kind, ref, 'afterAll'); } if (network) await removeVendorNetwork(network); } diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index 48744a12..a05060d3 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -99,6 +99,10 @@ it('removes by name only an object of exactly that name, and reports failures ot ['network', 'raced'], ['container', 'junk'], ['container', 'odd'], ['container', 'nospace'], ['container', 'optid'], ['container', '--help'], ['volume', '-v']] as const; expect(() => { for (const [kind, name] of objects) removeByName(kind, name, 'test', join(root, 'docker')); }).not.toThrow(); + // Each missing name reached its inspect, so the quiet answer was Docker's missing one. + for (const kind of ['container', 'network'] as const) + expect(readFileSync(calls, 'utf8').split('\n')).toContain(`${kind} inspect --format {{json .Id}} {{json .Name}} gone`); + expect(readFileSync(calls, 'utf8').split('\n')).toContain('volume inspect --format {{json .Name}} gone'); // Only the exact names are removed, by full ID (a volume's name is its ID). expect(readFileSync(calls, 'utf8').split('\n').filter(line => / rm |^rm /.test(line))) .toEqual([`rm --force ${id}`, `network rm ${other}`, 'volume rm --force busy', `network rm ${raced}`]); @@ -207,7 +211,7 @@ it('says nothing of a network gone since the listing, and reports any other fail }); it.each([ - ['says nothing of a holder gone since the listing', 'Error response from daemon: No such container: b', []], + ['says nothing of a holder gone since the listing', `Error response from daemon: No such container: ${'b'.repeat(64)}`, []], ['reports any other failure to read the holders', 'permission denied', [['test: reading holders: docker container inspect --format {{json .Id}} {{json .NetworkSettings.Networks}} HOLDERS failed:', 'permission denied']]], ])('%s, and still removes the holders it read', (_label, stderr, warnings) => { const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); @@ -298,3 +302,33 @@ it('reports and skips listed lines that are not a container ID, a network or a v ['test: listing volumes: docker printed a line that is not a volume name:', '-v']]); expect(readFileSync(calls, 'utf8').split('\n').filter(line => / rm |^rm |^ps /.test(line))).toEqual([]); }); + +it('counts an object gone only on Docker\'s missing answer, and throws on any other failure, at the loaded daemon', async () => { + const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); + const bin = join(root, 'bin'), seen = join(root, 'seen'); mkdirSync(bin); + symlinkSync(spawnSync('sh', ['-c', 'command -v sh'], { encoding: 'utf8' }).stdout.trim(), join(bin, 'sh')); + // Answers as Docker 27.4 does for `inspect` of an object that exists, of a missing container, network and volume, + // and for an unknown context; one call is killed by a signal. Each call records its daemon address. Only the exit + // status and stderr are read. + writeFileSync(join(bin, 'docker'), ['#!/bin/sh', `printf '%s|%s\\n' "\${DOCKER_HOST-unset}" "\${DOCKER_CONTEXT-unset}" >> '${seen}'`, + 'case "$1 $3" in', + " 'container there') echo '[]';;", + " 'container gone') echo; echo 'Error response from daemon: No such container: gone' >&2; exit 1;;", + " 'network gone') echo; echo 'Error response from daemon: network gone not found' >&2; exit 1;;", + " 'volume gone') echo; echo 'Error response from daemon: get gone: no such volume' >&2; exit 1;;", + " 'container killed') kill -9 $$;;", + " 'container lost') echo 'Failed to initialize: unable to resolve docker endpoint: context \\\"x\\\": context not found' >&2; exit 1;;", + 'esac; exit 0'].join('\n'), { mode: 0o755 }); + vi.stubEnv('PATH', bin); + vi.stubEnv('DOCKER_HOST', 'unix:///codeboost-loaded.sock'); + vi.stubEnv('DOCKER_CONTEXT', 'codeboost-loaded'); + vi.resetModules(); + const fresh = await import('./fixtures/docker-sweep.ts') as typeof Sweep; + vi.stubEnv('DOCKER_HOST', 'unix:///codeboost-changed.sock'); + vi.stubEnv('DOCKER_CONTEXT', 'codeboost-changed'); + expect(fresh.objectExists('container', 'there')).toBe(true); + for (const kind of ['container', 'network', 'volume'] as const) expect(fresh.objectExists(kind, 'gone')).toBe(false); + expect(() => fresh.objectExists('container', 'killed')).toThrow('docker container inspect killed failed: killed by SIGKILL'); + expect(() => fresh.objectExists('container', 'lost')).toThrow('docker container inspect lost failed: Failed to initialize'); + expect(new Set(readFileSync(seen, 'utf8').split('\n').filter(Boolean))).toEqual(new Set(['unix:///codeboost-loaded.sock|codeboost-loaded'])); +}); From 10b2bf8f66253bc911a283385770391f2b66e2f7 Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 06:19:33 -0700 Subject: [PATCH 68/75] Gone checks in the touched tests use objectExists; fake and comment fixes (#189) Review round 67: the keeper-replaced, foreign-name and unlabelled-agent checks use objectExists, and the holder test calls it directly; the PR says checks in untouched tests keep main's form; the objectExists fake prints the unknown context's quotes as Docker does and says its path is shortened; the supervisor's tracked containers are described as lacking the runner label. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 15 +++++++-------- test/agent-supervisor.test.ts | 9 ++++++--- test/docker-sweep.test.ts | 4 ++-- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index f5474697..98d64268 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -773,7 +773,7 @@ describe('real Docker agent isolation', () => { const data = fixture(), live = await profile(data, 'planning', 'must-not-run'); await withoutRunnerLabel([['create', '--name']], () => expect(createValidatedContainer(live)).rejects.toThrow('lockdown')); - expect(spawnSync('docker', ['container', 'inspect', live.name]).status).not.toBe(0); + expect(objectExists('container', live.name)).toBe(false); }, 60_000); // #189: the sweep removes what holds a swept network or volume, even a container created and never started, and @@ -860,12 +860,11 @@ describe('real Docker agent isolation', () => { removeByName('network', lookalike.slice(0, 12), 'test'); expect(docker('container', 'inspect', '--format', '{{.Id}}', bystanders[0]!.slice(0, 12))).toBe(bystanders[0]); removeByName('container', bystanders[0]!.slice(0, 12), 'test'); - const exists = objectExists; - for (const holder of holders) expect(exists('container', holder)).toBe(false); - expect([exists('network', network), exists('network', prefixNamed), exists('volume', volume)]).toEqual([false, false, false]); - expect(exists('container', labelled)).toBe(false); - for (const bystander of [...bystanders, holderLookalike, labelledLookalike]) expect(exists('container', bystander)).toBe(true); - expect([exists('network', bystanderNetwork), exists('volume', bystanderVolume), exists('network', lookalike)]) + for (const holder of holders) expect(objectExists('container', holder)).toBe(false); + expect([objectExists('network', network), objectExists('network', prefixNamed), objectExists('volume', volume)]).toEqual([false, false, false]); + expect(objectExists('container', labelled)).toBe(false); + for (const bystander of [...bystanders, holderLookalike, labelledLookalike]) expect(objectExists('container', bystander)).toBe(true); + expect([objectExists('network', bystanderNetwork), objectExists('volume', bystanderVolume), objectExists('network', lookalike)]) .toEqual([true, true, true]); } finally { // By exact name: some are named like the start of another object's ID. Containers first, as they hold the rest. @@ -2586,7 +2585,7 @@ describe('real Docker agent isolation', () => { '--network=none', '--entrypoint', 'sleep', imageId, 'infinity'); expect(() => removeTaskFilesystems(filesystems)).toThrow('did not settle'); expect(docker('container', 'inspect', '--format', '{{.State.Running}}', filesystems.keeper)).toBe('true'); - expect(spawnSync('docker', ['volume', 'inspect', filesystems.workVolume]).status).not.toBe(0); + expect(objectExists('volume', filesystems.workVolume)).toBe(false); passed = true; } finally { const keeperRemoved = spawnSync('docker', ['rm', '--force', filesystems.keeper], { encoding: 'utf8', ...CLIENT_LIMIT }); diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index d5abcedd..85bc3134 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -1,6 +1,6 @@ import { randomBytes, randomUUID } from 'node:crypto'; import { execFileSync, spawnSync } from 'node:child_process'; -import { CLIENT_LIMIT, removeByName, removeRunnerObjects, requireRealDocker } from './fixtures/docker-sweep.ts'; +import { CLIENT_LIMIT, objectExists, removeByName, removeRunnerObjects, requireRealDocker } from './fixtures/docker-sweep.ts'; import { fixtureGit } from './fixtures/git.ts'; import { chmodSync, mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; @@ -30,7 +30,10 @@ const testOwner = (attemptId = 'fixture') => ({ runnerOwner: TEST_RUNNER_OWNER, const roots: string[] = [], profiles: ContainerProfile[] = []; const allocations: ReturnType[] = []; -/** Label-less containers, by name, removed in afterAll should their create land after their test's finally (#189). */ +/** + * Containers without this run's runner label, by name, removed in afterAll should their create land after their test's + * finally (#189). + */ const unlabelled = new Set(); let imageId = ''; const git = fixtureGit; @@ -405,7 +408,7 @@ describe('container invocation supervisor', () => { expect(result.unreleased).toBeUndefined(); expect(isInvocationActive(attempt('foreign-name'))).toBe(false); expect(isContainerProfileAuthentic(current)).toBe(false); - expect(spawnSync('docker', ['network', 'inspect', current.network.name], { stdio: 'ignore' }).status).not.toBe(0); + expect(objectExists('network', current.network.name)).toBe(false); expect(execFileSync('docker', ['container', 'inspect', '--format', '{{index .Config.Labels "io.codeboost.invocation"}}', current.name], { encoding: 'utf8' }).trim()).toBe('foreign'); passed = true; diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index a05060d3..69e8fe4c 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -308,7 +308,7 @@ it('counts an object gone only on Docker\'s missing answer, and throws on any ot const bin = join(root, 'bin'), seen = join(root, 'seen'); mkdirSync(bin); symlinkSync(spawnSync('sh', ['-c', 'command -v sh'], { encoding: 'utf8' }).stdout.trim(), join(bin, 'sh')); // Answers as Docker 27.4 does for `inspect` of an object that exists, of a missing container, network and volume, - // and for an unknown context; one call is killed by a signal. Each call records its daemon address. Only the exit + // and for an unknown context (its meta.json path shortened); one call is killed by a signal. Each call records its daemon address. Only the exit // status and stderr are read. writeFileSync(join(bin, 'docker'), ['#!/bin/sh', `printf '%s|%s\\n' "\${DOCKER_HOST-unset}" "\${DOCKER_CONTEXT-unset}" >> '${seen}'`, 'case "$1 $3" in', @@ -317,7 +317,7 @@ it('counts an object gone only on Docker\'s missing answer, and throws on any ot " 'network gone') echo; echo 'Error response from daemon: network gone not found' >&2; exit 1;;", " 'volume gone') echo; echo 'Error response from daemon: get gone: no such volume' >&2; exit 1;;", " 'container killed') kill -9 $$;;", - " 'container lost') echo 'Failed to initialize: unable to resolve docker endpoint: context \\\"x\\\": context not found' >&2; exit 1;;", + " 'container lost') echo 'Failed to initialize: unable to resolve docker endpoint: context \"x\": context not found: open /x/meta.json: no such file or directory' >&2; exit 1;;", 'esac; exit 0'].join('\n'), { mode: 0o755 }); vi.stubEnv('PATH', bin); vi.stubEnv('DOCKER_HOST', 'unix:///codeboost-loaded.sock'); From 196a2255268485e3e7e8f72dd5ae5a43572a6f86 Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 06:34:57 -0700 Subject: [PATCH 69/75] Every gone check in a changed test uses objectExists (#189) Review round 68: the by-ID removal and start tests, the rogue-endpoint test, the recovery test's exists helper and agent-network's late-landing check use objectExists; removeByName's refusal says the name could read as an option; the objectExists fake marks its stdout as invented. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 11 +++++------ test/agent-network.test.ts | 2 +- test/docker-sweep.test.ts | 9 +++++---- test/fixtures/docker-sweep.ts | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 98d64268..44b462df 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -633,7 +633,7 @@ describe('real Docker agent isolation', () => { '--entrypoint', 'true', imageId); const replacementId = docker('container', 'inspect', '--format', '{{.Id}}', replacement); await disposeValidatedContainer(live); - expect(spawnSync('docker', ['container', 'inspect', originalId], { stdio: 'ignore' }).status).not.toBe(0); + expect(objectExists('container', originalId)).toBe(false); expect(docker('container', 'inspect', '--format', '{{.Id}}', replacement)).toBe(replacementId); expect(isContainerProfileAuthentic(live)).toBe(false); }, 60_000); @@ -673,7 +673,7 @@ describe('real Docker agent isolation', () => { expect(await startValidatedContainer(live)).toBe('stdout-marker'); // The replacement was never started, and cleanup removed only ours. expect(docker('container', 'inspect', '--format', '{{.State.StartedAt}}', replacement)).toBe('0001-01-01T00:00:00Z'); - expect(spawnSync('docker', ['container', 'inspect', originalId], { stdio: 'ignore' }).status).not.toBe(0); + expect(objectExists('container', originalId)).toBe(false); }, 60_000); it('labels every Docker object with its runner, attempt and allocation', async () => { @@ -2410,8 +2410,8 @@ describe('real Docker agent isolation', () => { // Cleanup runs in reverse: containers go before the networks and volumes they use. const undo: (() => unknown)[] = []; const quietly = (...args: string[]) => () => spawnSync('docker', args, { stdio: 'ignore' }); - const exists = (kind: 'container' | 'volume' | 'network', ref: string) => - spawnSync('docker', [kind, 'inspect', ref], { stdio: 'ignore' }).status === 0; + // Only Docker's missing answer counts as gone (#189). + const exists = objectExists; const label = (labels: Record) => Object.entries(labels).flatMap(([key, value]) => ['--label', `${key}=${value}`]); // What a crashed process leaves for the first runner, made with the labels and names D writes, so nothing in this @@ -2835,8 +2835,7 @@ describe('real Docker agent isolation', () => { docker('run', '--detach', '--name', rogue, `--network=${valid.network.name}`, '--entrypoint', 'node', imageId, '-e', 'setInterval(()=>{},1000)'); await expect(createValidatedContainer(valid)).rejects.toThrow('cleanup did not settle'); - const absent = spawnSync('docker', ['container', 'inspect', valid.name], { encoding: 'utf8' }); - expect(absent.status).not.toBe(0); + expect(objectExists('container', valid.name)).toBe(false); } finally { spawnSync('docker', ['rm', '--force', rogue], { stdio: 'ignore' }); await disposeContainerProfile(valid); diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index 3f9b9f03..4a6ea13d 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -109,7 +109,7 @@ describe('vendor-only egress', () => { execFileSync('sleep', ['3']); // A network landing later still carries this run's runner label (the shim passes production's arguments on), so // afterAll's runner sweep removes it. - const orphaned = spawnSync('docker', ['network', 'inspect', name], { stdio: 'ignore' }).status === 0; + const orphaned = objectExists('network', name); if (orphaned) docker('network', 'rm', name); expect(name).toMatch(/^codeboost-egress-/); expect(orphaned).toBe(false); diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index 69e8fe4c..22820f7a 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -119,8 +119,8 @@ it('removes by name only an object of exactly that name, and reports failures ot ['test: removing container odd: docker printed no ID and name:', '1 2\n'], ['test: removing container nospace: docker printed what is not JSON:', '"nospace"\n'], ['test: removing container optid: docker printed no ID and name:', '"--help" "/optid"\n'], - ['test: removing container --help: refused a name that is not one Docker gives'], - ['test: removing volume -v: refused a name that is not one Docker gives'], + ['test: removing container --help: refused a name that could read as an option'], + ['test: removing volume -v: refused a name that could read as an option'], ]); }); @@ -308,8 +308,9 @@ it('counts an object gone only on Docker\'s missing answer, and throws on any ot const bin = join(root, 'bin'), seen = join(root, 'seen'); mkdirSync(bin); symlinkSync(spawnSync('sh', ['-c', 'command -v sh'], { encoding: 'utf8' }).stdout.trim(), join(bin, 'sh')); // Answers as Docker 27.4 does for `inspect` of an object that exists, of a missing container, network and volume, - // and for an unknown context (its meta.json path shortened); one call is killed by a signal. Each call records its daemon address. Only the exit - // status and stderr are read. + // and for an unknown context (its meta.json path shortened); one call is killed by a signal. Each call records its + // daemon address. Only the exit status and stderr are read: the stdout is invented (27.4 prints the objects' JSON, or + // `[]` for a missing one). writeFileSync(join(bin, 'docker'), ['#!/bin/sh', `printf '%s|%s\\n' "\${DOCKER_HOST-unset}" "\${DOCKER_CONTEXT-unset}" >> '${seen}'`, 'case "$1 $3" in', " 'container there') echo '[]';;", diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index e1354448..ddd820e2 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -69,7 +69,7 @@ export function removeByName(kind: 'container' | 'network' | 'volume', name: str result.error?.message ?? (result.signal ? `killed by ${result.signal}` : result.stderr.trim())); // The name is passed on too: refused if it could read as an option (a volume's must have the shape Docker allows). if (kind === 'volume' ? !VOLUME_NAME.test(name) : name.startsWith('-') || !name) { - console.warn(`${when}: removing ${kind} ${name}: refused a name that is not one Docker gives`); return; + console.warn(`${when}: removing ${kind} ${name}: refused a name that could read as an option`); return; } // As JSON: a network's name may hold spaces. Only a container's name comes with a leading slash. const inspect = [kind, 'inspect', '--format', kind === 'volume' ? '{{json .Name}}' : '{{json .Id}} {{json .Name}}', name]; From 7ea7fc854c19923d2360b3f640834bb876043026 Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 06:49:16 -0700 Subject: [PATCH 70/75] The proxy-start test's gone check uses objectExists; assert the raced removal ran (#189) Review round 69: the last gone check in a changed test uses objectExists; the gone-network sweep test checks its removal ran before trusting the silence; the stand-in shim's note says where stderr goes. Co-Authored-By: Claude Opus 5.5 --- test/agent-network.test.ts | 5 +++-- test/docker-sweep.test.ts | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index 4a6ea13d..a58c14e7 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -171,7 +171,7 @@ describe('vendor-only egress', () => { await expect(createVendorNetwork(failing, imageId, randomUUID())).rejects.toThrow('failed to start'); proxyId = readFileSync(recorded, 'utf8'); } finally { process.env.PATH = path; rmSync(shim, { recursive: true, force: true }); } - const leaked = spawnSync('docker', ['container', 'inspect', proxyId], { stdio: 'ignore' }).status === 0; + const leaked = objectExists('container', proxyId); if (leaked) spawnSync('docker', ['rm', '--force', proxyId], { stdio: 'ignore' }); expect(proxyId).toMatch(/^[0-9a-f]{64}$/); expect(leaked).toBe(false); @@ -237,7 +237,8 @@ describe('vendor-only egress', () => { const realDocker = requireRealDocker(); // The readiness exec swaps the proxy for a same-named, same-labelled stand-in, then fails setup. Only an exec in a // proxy: any other call passes through, so the shim never makes a stand-in with an empty, random name. Its exit 1 - // with no stderr is invented, as an exec that failed; the code reads only the status. + // with no stderr is invented, as an exec that failed: the code branches only on the status, and stderr goes only into + // the error message, which the test does not check. writeFileSync(join(shim, 'docker'), ['#!/bin/sh', 'if [ "$1" = exec ]; then', ` name=$('${realDocker}' inspect -f '{{.Name}}' "$2" 2>/dev/null | sed 's#^/##')`, ' case "$name" in codeboost-proxy-*)', diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index 22820f7a..e9a95aa5 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -192,11 +192,11 @@ it('takes the daemon set at load, not the default, when one was set, for every c it('says nothing of a network gone since the listing, and reports any other failure to remove one', () => { const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); - const net = 'd'.repeat(64), answer = join(root, 'answer'); + const net = 'd'.repeat(64), answer = join(root, 'answer'), calls = join(root, 'calls'); // Answers as Docker 27.4 does for one swept network with no holder, whose removal gives the stderr in `answer`; every // other call gets no output and exit 0, as for `ls` with no match. The network line holds only the fields the sweep // reads. The second answer is invented, as a failure 27.4 might give. - writeFileSync(join(root, 'docker'), ['#!/bin/sh', + writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${calls}'`, `if [ "$1 $2" = "network ls" ]; then printf '%s\\n' '{"ID":"${net}","Name":"swept"}'; exit 0; fi`, `if [ "$1 $2" = "network rm" ]; then cat '${answer}' >&2; exit 1; fi`, 'exit 0'].join('\n'), { mode: 0o755 }); @@ -204,6 +204,8 @@ it('says nothing of a network gone since the listing, and reports any other fail const failures = () => warn.mock.calls.filter(call => String(call[0]).endsWith('failed:')); writeFileSync(answer, `Error response from daemon: network ${net} not found\n`); removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test', join(root, 'docker')); + // The removal ran and gave that answer, so the silence is the gone answer's. + expect(readFileSync(calls, 'utf8').split('\n')).toContain(`network rm ${net}`); expect(failures()).toEqual([]); writeFileSync(answer, 'Error response from daemon: permission denied\n'); removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test', join(root, 'docker')); From 2f30016de19470888cb8ca687affb9d4b3c7732f Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 07:05:49 -0700 Subject: [PATCH 71/75] Shape checks before the gone checks; objectExists refuses an option-like ref (#189) Review round 70: the late-landing network and proxy-start tests check the value their shim recorded before inspecting it, as the record tests do, so a shim that recorded something else is reported as itself; objectExists refuses an empty or dash-led ref, naming it; the proxy-start test's removal takes CLIENT_LIMIT. Co-Authored-By: Claude Opus 5.5 --- test/agent-network.test.ts | 8 +++++--- test/docker-sweep.test.ts | 2 ++ test/fixtures/docker-sweep.ts | 2 ++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index a58c14e7..57321290 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -109,9 +109,10 @@ describe('vendor-only egress', () => { execFileSync('sleep', ['3']); // A network landing later still carries this run's runner label (the shim passes production's arguments on), so // afterAll's runner sweep removes it. + // First, so a shim that recorded something else is reported as itself. + expect(name).toMatch(/^codeboost-egress-/); const orphaned = objectExists('network', name); if (orphaned) docker('network', 'rm', name); - expect(name).toMatch(/^codeboost-egress-/); expect(orphaned).toBe(false); }, 60_000); @@ -171,9 +172,10 @@ describe('vendor-only egress', () => { await expect(createVendorNetwork(failing, imageId, randomUUID())).rejects.toThrow('failed to start'); proxyId = readFileSync(recorded, 'utf8'); } finally { process.env.PATH = path; rmSync(shim, { recursive: true, force: true }); } - const leaked = objectExists('container', proxyId); - if (leaked) spawnSync('docker', ['rm', '--force', proxyId], { stdio: 'ignore' }); + // First, so a shim that recorded something else is reported as itself. expect(proxyId).toMatch(/^[0-9a-f]{64}$/); + const leaked = objectExists('container', proxyId); + if (leaked) spawnSync('docker', ['rm', '--force', proxyId], { stdio: 'ignore', ...CLIENT_LIMIT }); expect(leaked).toBe(false); }, 60_000); diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index e9a95aa5..16e17381 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -333,5 +333,7 @@ it('counts an object gone only on Docker\'s missing answer, and throws on any ot for (const kind of ['container', 'network', 'volume'] as const) expect(fresh.objectExists(kind, 'gone')).toBe(false); expect(() => fresh.objectExists('container', 'killed')).toThrow('docker container inspect killed failed: killed by SIGKILL'); expect(() => fresh.objectExists('container', 'lost')).toThrow('docker container inspect lost failed: Failed to initialize'); + expect(() => fresh.objectExists('container', '--help')).toThrow('Refused to inspect container "--help"'); + expect(() => fresh.objectExists('network', '')).toThrow('Refused to inspect network ""'); expect(new Set(readFileSync(seen, 'utf8').split('\n').filter(Boolean))).toEqual(new Set(['unix:///codeboost-loaded.sock|codeboost-loaded'])); }); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index ddd820e2..601b2ad1 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -48,6 +48,8 @@ const missingAnswer = (kind: 'container' | 'network' | 'volume', ref: string) => * gone, and any other failure (a killed call, a daemon that cannot be reached) throws, naming the command. */ export function objectExists(kind: 'container' | 'network' | 'volume', ref: string): boolean { + // The ref is passed on: one that could read as an option is refused, naming it. + if (!ref || ref.startsWith('-')) throw new Error(`Refused to inspect ${kind} ${JSON.stringify(ref)}: it could read as an option.`); const args = [kind, 'inspect', ref]; const found = spawnSync(REAL_DOCKER, args, { encoding: 'utf8', ...CLIENT_LIMIT, env: dockerEnv() }); if (found.status === 0) return true; From 59c3183bcc680168aea29ad58872973001efd3da Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 07:20:52 -0700 Subject: [PATCH 72/75] Trim the listing's stderr in its failure message (#189) Review round 71. Co-Authored-By: Claude Opus 5.5 --- test/agent-container.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 44b462df..b468a173 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -980,7 +980,7 @@ describe('real Docker agent isolation', () => { const listing = spawnSync('docker', listArgs, { encoding: 'utf8', ...CLIENT_LIMIT }); // A listing that failed fails here with the command and its cause, as main's helper did. expect(listing.status, `docker ${listArgs.join(' ')} failed: ` - + (listing.error?.message ?? (listing.signal ? `killed by ${listing.signal}` : listing.stderr))).toBe(0); + + (listing.error?.message ?? (listing.signal ? `killed by ${listing.signal}` : listing.stderr.trim()))).toBe(0); const left = listing.stdout.trim(); expect(created).toBeInstanceOf(VendorNetworkCreationCleanupError); const cleanup = (created as VendorNetworkCreationCleanupError).errors[1] as AggregateError; From 9571be5719149c8782e4b5018697b963c124fa3c Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 07:39:17 -0700 Subject: [PATCH 73/75] Pin every docker call's time limit; volume refusal reason; guarded record reads (#189) Review round 72: a unit test loads the fixture with spawnSync observed and checks every removal call is given 30 s and SIGKILL, and objectExists 60 s and SIGKILL, so the budgets built on them cannot lose their limit unseen; removeByName says a refused volume name is not one Docker allows; agent-network's finally blocks report a record they cannot read instead of throwing over the body's failure; the PR says what every changed gone check requires. Co-Authored-By: Claude Opus 5.5 --- test/agent-network.test.ts | 15 ++++++++++++--- test/docker-sweep.test.ts | 31 ++++++++++++++++++++++++++++++- test/fixtures/docker-sweep.ts | 3 ++- 3 files changed, 44 insertions(+), 5 deletions(-) diff --git a/test/agent-network.test.ts b/test/agent-network.test.ts index 57321290..39cc50c1 100644 --- a/test/agent-network.test.ts +++ b/test/agent-network.test.ts @@ -137,7 +137,10 @@ describe('vendor-only egress', () => { } finally { process.env.PATH = path; // Recorded before it is made, and removed here: the network has no labels, so no sweep finds it (#189). - const made = existsSync(recorded) ? readFileSync(recorded, 'utf8').split('\n').filter(Boolean) : []; + // A record that cannot be read is reported, not thrown, so the body's own failure stays the one reported. + let made: string[] = []; + try { made = existsSync(recorded) ? readFileSync(recorded, 'utf8').split('\n').filter(Boolean) : []; } + catch (caught) { console.warn('finally: reading the record failed:', caught); } foreign = made[0] ?? ''; // afterAll reads the record again: the object may land after this removal (#189). // Only Docker's own missing answer counts as gone; any other failure is kept and reported below, after the body's. @@ -198,7 +201,10 @@ describe('vendor-only egress', () => { } finally { process.env.PATH = path; // Recorded before it is made, and removed here: the container has no labels, so no sweep finds it (#189). - const made = existsSync(recorded) ? readFileSync(recorded, 'utf8').split('\n').filter(Boolean) : []; + // A record that cannot be read is reported, not thrown, so the body's own failure stays the one reported. + let made: string[] = []; + try { made = existsSync(recorded) ? readFileSync(recorded, 'utf8').split('\n').filter(Boolean) : []; } + catch (caught) { console.warn('finally: reading the record failed:', caught); } foreign = made[0] ?? ''; // afterAll reads the record again: the object may land after this removal (#189). // Only Docker's own missing answer counts as gone; any other failure is kept and reported below, after the body's. @@ -260,7 +266,10 @@ describe('vendor-only egress', () => { } finally { process.env.PATH = path; // The stand-in has no runner label, so no sweep finds it: remove it here, however the test ends (#189). - const made = existsSync(recorded) ? readFileSync(recorded, 'utf8').split('\n').filter(Boolean) : []; + // A record that cannot be read is reported, not thrown, so the body's own failure stays the one reported. + let made: string[] = []; + try { made = existsSync(recorded) ? readFileSync(recorded, 'utf8').split('\n').filter(Boolean) : []; } + catch (caught) { console.warn('finally: reading the record failed:', caught); } impostor = made[0] ?? ''; // afterAll reads the record again: the object may land after this removal (#189). // Only Docker's own missing answer counts as gone; any other failure is kept and reported below, after the body's. diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index 16e17381..d82ec6f7 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -120,7 +120,7 @@ it('removes by name only an object of exactly that name, and reports failures ot ['test: removing container nospace: docker printed what is not JSON:', '"nospace"\n'], ['test: removing container optid: docker printed no ID and name:', '"--help" "/optid"\n'], ['test: removing container --help: refused a name that could read as an option'], - ['test: removing volume -v: refused a name that could read as an option'], + ['test: removing volume -v: refused a name that is not a volume name Docker allows'], ]); }); @@ -337,3 +337,32 @@ it('counts an object gone only on Docker\'s missing answer, and throws on any ot expect(() => fresh.objectExists('network', '')).toThrow('Refused to inspect network ""'); expect(new Set(readFileSync(seen, 'utf8').split('\n').filter(Boolean))).toEqual(new Set(['unix:///codeboost-loaded.sock|codeboost-loaded'])); }); + +it('gives every docker call its time limit, killed with SIGKILL: 30 s for the removals, 60 s for objectExists', async () => { + const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); + // Answers as Docker 27.4 does for an `ls` with no match and an inspect of a missing container; only the options each + // call is given are checked. + writeFileSync(join(root, 'docker'), ['#!/bin/sh', + 'if [ "$2" = inspect ]; then echo; echo "Error response from daemon: No such container: x" >&2; exit 1; fi', 'exit 0'].join('\n'), + { mode: 0o755 }); + const options: { timeout?: number; killSignal?: unknown }[] = []; + vi.doMock('node:child_process', async original => { + const actual = await original(); + return { ...actual, spawnSync: (command: string, args: string[], given: { timeout?: number; killSignal?: unknown }) => { + if (!['sh'].includes(command)) options.push(given); + return actual.spawnSync(command, args, given as never); + } }; + }); + vi.stubEnv('PATH', `${root}:${process.env.PATH}`); + vi.resetModules(); + try { + const fresh = await import('./fixtures/docker-sweep.ts') as typeof Sweep; + fresh.removeByName('container', 'x', 'test', join(root, 'docker')); + fresh.removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test', join(root, 'docker')); + const removals = options.length; + expect(fresh.objectExists('container', 'x')).toBe(false); + expect(removals).toBeGreaterThan(3); + expect(options.slice(0, removals).every(given => given.timeout === 30_000 && given.killSignal === 'SIGKILL')).toBe(true); + expect(options.slice(removals)).toMatchObject([{ timeout: 60_000, killSignal: 'SIGKILL' }]); + } finally { vi.doUnmock('node:child_process'); vi.resetModules(); } +}); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index 601b2ad1..24891000 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -71,7 +71,8 @@ export function removeByName(kind: 'container' | 'network' | 'volume', name: str result.error?.message ?? (result.signal ? `killed by ${result.signal}` : result.stderr.trim())); // The name is passed on too: refused if it could read as an option (a volume's must have the shape Docker allows). if (kind === 'volume' ? !VOLUME_NAME.test(name) : name.startsWith('-') || !name) { - console.warn(`${when}: removing ${kind} ${name}: refused a name that could read as an option`); return; + console.warn(`${when}: removing ${kind} ${name}: refused a name that ${kind === 'volume' ? 'is not a volume name Docker allows' : 'could read as an option'}`); + return; } // As JSON: a network's name may hold spaces. Only a container's name comes with a leading slash. const inspect = [kind, 'inspect', '--format', kind === 'volume' ? '{{json .Name}}' : '{{json .Id}} {{json .Name}}', name]; From e11583bf1d9f9cfb380f32e4c578ef5b0af348bb Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 07:52:54 -0700 Subject: [PATCH 74/75] Time-limit test runs all thirteen calls; holder search and no-space lines tested (#189) Review round 73: the time-limit test answers so both removals make every call (13), and checks each; a holder search line that is not a container ID is reported and skipped, with a test; removeByName reports an inspect line with no space as no ID and name; the holder read's no-space line is tested. Co-Authored-By: Claude Opus 5.5 --- test/docker-sweep.test.ts | 41 +++++++++++++++++++++++++++++------ test/fixtures/docker-sweep.ts | 4 +++- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index d82ec6f7..be0e030c 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -117,7 +117,7 @@ it('removes by name only an object of exactly that name, and reports failures ot 'Failed to initialize: unable to resolve docker endpoint: context "x": context not found: open /x/meta.json: no such file or directory'], ['test: removing container junk: docker printed what is not JSON:', 'not json\n'], ['test: removing container odd: docker printed no ID and name:', '1 2\n'], - ['test: removing container nospace: docker printed what is not JSON:', '"nospace"\n'], + ['test: removing container nospace: docker printed no ID and name:', '"nospace"\n'], ['test: removing container optid: docker printed no ID and name:', '"--help" "/optid"\n'], ['test: removing container --help: refused a name that could read as an option'], ['test: removing volume -v: refused a name that is not a volume name Docker allows'], @@ -278,12 +278,13 @@ it('reports a holder line that is not an ID and networks, and removes no holder writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${calls}'`, `if [ "$1 $2" = "network ls" ]; then printf '%s\\n' '{"ID":"${net}","Name":"swept"}'; exit 0; fi`, `if [ "$1" = ps ]; then printf '%s\\n' ${'a'.repeat(64)}; exit 0; fi`, - `if [ "$1 $2" = "container inspect" ]; then printf '%s\\n' '123 null' '"--all" {"swept":{}}'; exit 0; fi`, 'exit 0'].join('\n'), + `if [ "$1 $2" = "container inspect" ]; then printf '%s\\n' '123 null' '"--all" {"swept":{}}' '"${'a'.repeat(64)}"'; exit 0; fi`, 'exit 0'].join('\n'), { mode: 0o755 }); const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test', join(root, 'docker')); expect(warn.mock.calls).toContainEqual(['test: reading holders: docker printed a line that is not an ID and networks:', '123 null']); expect(warn.mock.calls).toContainEqual(['test: reading holders: docker printed a line that is not an ID and networks:', '"--all" {"swept":{}}']); + expect(warn.mock.calls).toContainEqual(['test: reading holders: docker printed a line that is not an ID and networks:', `"${'a'.repeat(64)}"`]); expect(readFileSync(calls, 'utf8').split('\n').filter(line => line.startsWith('rm '))).toEqual([]); }); @@ -340,10 +341,19 @@ it('counts an object gone only on Docker\'s missing answer, and throws on any ot it('gives every docker call its time limit, killed with SIGKILL: 30 s for the removals, 60 s for objectExists', async () => { const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); - // Answers as Docker 27.4 does for an `ls` with no match and an inspect of a missing container; only the options each - // call is given are checked. - writeFileSync(join(root, 'docker'), ['#!/bin/sh', - 'if [ "$2" = inspect ]; then echo; echo "Error response from daemon: No such container: x" >&2; exit 1; fi', 'exit 0'].join('\n'), + const net = 'f'.repeat(64), holder = 'a'.repeat(64), named = 'b'.repeat(64), labelled = 'c'.repeat(64), volumeHolder = 'e'.repeat(64); + // The loaded-daemon test's answers, so every call the two make runs (13), plus an inspect of a missing container for + // objectExists; only the options each call is given are checked. + writeFileSync(join(root, 'docker'), ['#!/bin/sh', 'for arg; do last="$arg"; done', + `if [ "$1 $2 $3" = "container inspect x" ]; then echo; echo 'Error response from daemon: No such container: x' >&2; exit 1; fi`, + `if [ "$1 $2" = "container ls" ]; then printf '%s\\n' ${labelled}; exit 0; fi`, + `if [ "$1 $2" = "network ls" ]; then printf '%s\\n' '{"ID":"${net}","Name":"swept"}'; exit 0; fi`, + `if [ "$1 $2" = "volume ls" ]; then printf '%s\\n' swept-volume; exit 0; fi`, + `if [ "$1" = ps ] && [ "$last" = volume=swept-volume ]; then printf '%s\\n' ${volumeHolder}; exit 0; fi`, + `if [ "$1" = ps ]; then printf '%s\\n' ${holder}; exit 0; fi`, + `if [ "$1 $2" = "container inspect" ] && [ "$last" = x ]; then printf '%s\\n' '"${named}" "/x"'; exit 0; fi`, + `if [ "$1 $2" = "container inspect" ]; then printf '%s\\n' '"${holder}" {"swept":{}}'; exit 0; fi`, + 'exit 0'].join('\n'), { mode: 0o755 }); const options: { timeout?: number; killSignal?: unknown }[] = []; vi.doMock('node:child_process', async original => { @@ -357,12 +367,29 @@ it('gives every docker call its time limit, killed with SIGKILL: 30 s for the re vi.resetModules(); try { const fresh = await import('./fixtures/docker-sweep.ts') as typeof Sweep; + vi.spyOn(console, 'warn').mockImplementation(() => undefined); fresh.removeByName('container', 'x', 'test', join(root, 'docker')); fresh.removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test', join(root, 'docker')); const removals = options.length; expect(fresh.objectExists('container', 'x')).toBe(false); - expect(removals).toBeGreaterThan(3); + // removeByName's inspect and removal, and the sweep's eleven calls. + expect(removals).toBe(13); expect(options.slice(0, removals).every(given => given.timeout === 30_000 && given.killSignal === 'SIGKILL')).toBe(true); expect(options.slice(removals)).toMatchObject([{ timeout: 60_000, killSignal: 'SIGKILL' }]); } finally { vi.doUnmock('node:child_process'); vi.resetModules(); } }); + +it('reports and skips a holder search line that is not a container ID, so none becomes an option', () => { + const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); + const calls = join(root, 'calls'); + // One swept volume, whose holder search prints a line Docker 27.4 is not known to print; every other call gets no + // output and exit 0, as for `ls` with no match. + writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${calls}'`, + `if [ "$1 $2" = "volume ls" ]; then printf '%s\\n' swept-volume; exit 0; fi`, + `if [ "$1" = ps ]; then printf '%s\\n' '--all'; exit 0; fi`, 'exit 0'].join('\n'), + { mode: 0o755 }); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test', join(root, 'docker')); + expect(warn.mock.calls).toContainEqual(['test: finding holders: docker printed a line that is not a container ID:', '--all']); + expect(readFileSync(calls, 'utf8').split('\n').filter(line => line.startsWith('rm '))).toEqual([]); +}); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index 24891000..ce3e566a 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -84,7 +84,9 @@ export function removeByName(kind: 'container' | 'network' | 'volume', name: str let id: unknown, actual: unknown; try { const at = found.stdout.indexOf(' '); - if (kind !== 'volume' && at < 0) throw new Error('no space'); + if (kind !== 'volume' && at < 0) { + console.warn(`${when}: removing ${kind} ${name}: docker printed no ID and name:`, found.stdout); return; + } [id, actual] = kind === 'volume' ? [name, JSON.parse(found.stdout)] : [JSON.parse(found.stdout.slice(0, at)), JSON.parse(found.stdout.slice(at + 1))]; } catch { console.warn(`${when}: removing ${kind} ${name}: docker printed what is not JSON:`, found.stdout); return; } // A container's or network's ID is checked too, as it is passed on as an argument. From f6bf79007edda6f7b56ca2643d3d946933b6ed95 Mon Sep 17 00:00:00 2001 From: mchwang Date: Wed, 21 Oct 2026 08:08:16 -0700 Subject: [PATCH 75/75] Test the fixture's remaining guards; exact reasons in refusals and reports (#189) Review round 74: tests cover a docker that cannot start (both functions report and never throw), Docker's missing text with another exit status in objectExists, an empty name in removeByName, and a holder line that is not JSON, reported once; refusals of an empty value say so; an inspected ID that is not a full ID is reported as that; the two fakes that answer removals say their stdout is not read. Co-Authored-By: Claude Opus 5.5 --- test/docker-sweep.test.ts | 39 ++++++++++++++++++++++++++--------- test/fixtures/docker-sweep.ts | 13 +++++++----- 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/test/docker-sweep.test.ts b/test/docker-sweep.test.ts index be0e030c..313fa767 100644 --- a/test/docker-sweep.test.ts +++ b/test/docker-sweep.test.ts @@ -1,3 +1,4 @@ +import { randomUUID } from 'node:crypto'; import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, relative } from 'node:path'; @@ -97,7 +98,7 @@ it('removes by name only an object of exactly that name, and reports failures ot const objects = [['container', 'gone'], ['network', 'gone'], ['volume', 'gone'], ['container', 'prefix'], ['network', 'prefix'], ['container', 'busy'], ['network', 'busy'], ['volume', 'busy'], ['container', 'killed'], ['container', 'lost'], ['network', 'raced'], ['container', 'junk'], ['container', 'odd'], ['container', 'nospace'], ['container', 'optid'], - ['container', '--help'], ['volume', '-v']] as const; + ['container', '--help'], ['volume', '-v'], ['container', '']] as const; expect(() => { for (const [kind, name] of objects) removeByName(kind, name, 'test', join(root, 'docker')); }).not.toThrow(); // Each missing name reached its inspect, so the quiet answer was Docker's missing one. for (const kind of ['container', 'network'] as const) @@ -118,9 +119,10 @@ it('removes by name only an object of exactly that name, and reports failures ot ['test: removing container junk: docker printed what is not JSON:', 'not json\n'], ['test: removing container odd: docker printed no ID and name:', '1 2\n'], ['test: removing container nospace: docker printed no ID and name:', '"nospace"\n'], - ['test: removing container optid: docker printed no ID and name:', '"--help" "/optid"\n'], - ['test: removing container --help: refused a name that could read as an option'], + ['test: removing container optid: docker printed an ID that is not a full ID:', '"--help" "/optid"\n'], + ['test: removing container --help: refused a name that is empty or could read as an option'], ['test: removing volume -v: refused a name that is not a volume name Docker allows'], + ['test: removing container : refused a name that is empty or could read as an option'], ]); }); @@ -273,18 +275,21 @@ it('reports a holder line that is not an ID and networks, and removes no holder const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); const calls = join(root, 'calls'), net = 'd'.repeat(64); // One swept network with one candidate holder, whose inspect line parses but is not an ID and networks, which Docker - // 27.4 is not known to print. Every other call gets no output and exit 0, as for `ls` with no match. The network line - // holds only the fields the sweep reads. + // 27.4 is not known to print, one of them not JSON. Every other call gets no output and exit 0, as for `ls` with no + // match, and the removals' stdout is not read. The network line holds only the fields the sweep reads. writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${calls}'`, `if [ "$1 $2" = "network ls" ]; then printf '%s\\n' '{"ID":"${net}","Name":"swept"}'; exit 0; fi`, `if [ "$1" = ps ]; then printf '%s\\n' ${'a'.repeat(64)}; exit 0; fi`, - `if [ "$1 $2" = "container inspect" ]; then printf '%s\\n' '123 null' '"--all" {"swept":{}}' '"${'a'.repeat(64)}"'; exit 0; fi`, 'exit 0'].join('\n'), + `if [ "$1 $2" = "container inspect" ]; then printf '%s\\n' '123 null' '"--all" {"swept":{}}' '"${'a'.repeat(64)}"' 'unquoted {}'; exit 0; fi`, 'exit 0'].join('\n'), { mode: 0o755 }); const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test', join(root, 'docker')); - expect(warn.mock.calls).toContainEqual(['test: reading holders: docker printed a line that is not an ID and networks:', '123 null']); - expect(warn.mock.calls).toContainEqual(['test: reading holders: docker printed a line that is not an ID and networks:', '"--all" {"swept":{}}']); - expect(warn.mock.calls).toContainEqual(['test: reading holders: docker printed a line that is not an ID and networks:', `"${'a'.repeat(64)}"`]); + // Each line reported once: the one that is not JSON only as that. + expect(warn.mock.calls.filter(call => String(call[0]).startsWith('test: reading holders:'))).toEqual([ + ['test: reading holders: docker printed a line that is not an ID and networks:', '123 null'], + ['test: reading holders: docker printed a line that is not an ID and networks:', '"--all" {"swept":{}}'], + ['test: reading holders: docker printed a line that is not an ID and networks:', `"${'a'.repeat(64)}"`], + ['test: reading holders: docker printed a line that is not JSON:', 'unquoted']]); expect(readFileSync(calls, 'utf8').split('\n').filter(line => line.startsWith('rm '))).toEqual([]); }); @@ -322,6 +327,7 @@ it('counts an object gone only on Docker\'s missing answer, and throws on any ot " 'volume gone') echo; echo 'Error response from daemon: get gone: no such volume' >&2; exit 1;;", " 'container killed') kill -9 $$;;", " 'container lost') echo 'Failed to initialize: unable to resolve docker endpoint: context \"x\": context not found: open /x/meta.json: no such file or directory' >&2; exit 1;;", + " 'container odd') echo 'Error response from daemon: No such container: odd' >&2; exit 2;;", 'esac; exit 0'].join('\n'), { mode: 0o755 }); vi.stubEnv('PATH', bin); vi.stubEnv('DOCKER_HOST', 'unix:///codeboost-loaded.sock'); @@ -334,6 +340,8 @@ it('counts an object gone only on Docker\'s missing answer, and throws on any ot for (const kind of ['container', 'network', 'volume'] as const) expect(fresh.objectExists(kind, 'gone')).toBe(false); expect(() => fresh.objectExists('container', 'killed')).toThrow('docker container inspect killed failed: killed by SIGKILL'); expect(() => fresh.objectExists('container', 'lost')).toThrow('docker container inspect lost failed: Failed to initialize'); + // The missing answer with another exit status is not Docker's missing answer (invented: 27.4 exits 1). + expect(() => fresh.objectExists('container', 'odd')).toThrow('docker container inspect odd failed: Error response from daemon: No such container: odd'); expect(() => fresh.objectExists('container', '--help')).toThrow('Refused to inspect container "--help"'); expect(() => fresh.objectExists('network', '')).toThrow('Refused to inspect network ""'); expect(new Set(readFileSync(seen, 'utf8').split('\n').filter(Boolean))).toEqual(new Set(['unix:///codeboost-loaded.sock|codeboost-loaded'])); @@ -383,7 +391,7 @@ it('reports and skips a holder search line that is not a container ID, so none b const root = mkdtempSync(join(tmpdir(), 'codeboost-sweep-')); roots.push(root); const calls = join(root, 'calls'); // One swept volume, whose holder search prints a line Docker 27.4 is not known to print; every other call gets no - // output and exit 0, as for `ls` with no match. + // output and exit 0, as for `ls` with no match, and the removals' stdout is not read. writeFileSync(join(root, 'docker'), ['#!/bin/sh', `printf '%s\\n' "$*" >> '${calls}'`, `if [ "$1 $2" = "volume ls" ]; then printf '%s\\n' swept-volume; exit 0; fi`, `if [ "$1" = ps ]; then printf '%s\\n' '--all'; exit 0; fi`, 'exit 0'].join('\n'), @@ -393,3 +401,14 @@ it('reports and skips a holder search line that is not a container ID, so none b expect(warn.mock.calls).toContainEqual(['test: finding holders: docker printed a line that is not a container ID:', '--all']); expect(readFileSync(calls, 'utf8').split('\n').filter(line => line.startsWith('rm '))).toEqual([]); }); + +it('reports a docker that cannot start, and never throws', () => { + const missing = join(tmpdir(), `codeboost-no-docker-${randomUUID()}`, 'docker'); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + expect(() => removeByName('container', 'x', 'test', missing)).not.toThrow(); + expect(() => removeLabelledObjects(TEST_RUN_LABEL, 'x', 'test', missing)).not.toThrow(); + const reports = warn.mock.calls.map(call => `${String(call[0])} ${String(call[1])}`); + expect(reports[0]).toMatch(/^test: removing container x: docker container inspect .* x failed: .*ENOENT/); + expect(reports.slice(1).length).toBe(3); + for (const report of reports.slice(1)) expect(report).toMatch(/^test: listing (containers|networks|volumes): docker .* failed: .*ENOENT/); +}); diff --git a/test/fixtures/docker-sweep.ts b/test/fixtures/docker-sweep.ts index ce3e566a..46b76e60 100644 --- a/test/fixtures/docker-sweep.ts +++ b/test/fixtures/docker-sweep.ts @@ -48,8 +48,8 @@ const missingAnswer = (kind: 'container' | 'network' | 'volume', ref: string) => * gone, and any other failure (a killed call, a daemon that cannot be reached) throws, naming the command. */ export function objectExists(kind: 'container' | 'network' | 'volume', ref: string): boolean { - // The ref is passed on: one that could read as an option is refused, naming it. - if (!ref || ref.startsWith('-')) throw new Error(`Refused to inspect ${kind} ${JSON.stringify(ref)}: it could read as an option.`); + // The ref is passed on: one that is empty or could read as an option is refused, naming it. + if (!ref || ref.startsWith('-')) throw new Error(`Refused to inspect ${kind} ${JSON.stringify(ref)}: it is empty or could read as an option.`); const args = [kind, 'inspect', ref]; const found = spawnSync(REAL_DOCKER, args, { encoding: 'utf8', ...CLIENT_LIMIT, env: dockerEnv() }); if (found.status === 0) return true; @@ -71,7 +71,7 @@ export function removeByName(kind: 'container' | 'network' | 'volume', name: str result.error?.message ?? (result.signal ? `killed by ${result.signal}` : result.stderr.trim())); // The name is passed on too: refused if it could read as an option (a volume's must have the shape Docker allows). if (kind === 'volume' ? !VOLUME_NAME.test(name) : name.startsWith('-') || !name) { - console.warn(`${when}: removing ${kind} ${name}: refused a name that ${kind === 'volume' ? 'is not a volume name Docker allows' : 'could read as an option'}`); + console.warn(`${when}: removing ${kind} ${name}: refused a name that ${kind === 'volume' ? 'is not a volume name Docker allows' : 'is empty or could read as an option'}`); return; } // As JSON: a network's name may hold spaces. Only a container's name comes with a leading slash. @@ -89,10 +89,13 @@ export function removeByName(kind: 'container' | 'network' | 'volume', name: str } [id, actual] = kind === 'volume' ? [name, JSON.parse(found.stdout)] : [JSON.parse(found.stdout.slice(0, at)), JSON.parse(found.stdout.slice(at + 1))]; } catch { console.warn(`${when}: removing ${kind} ${name}: docker printed what is not JSON:`, found.stdout); return; } - // A container's or network's ID is checked too, as it is passed on as an argument. - if (typeof id !== 'string' || typeof actual !== 'string' || (kind !== 'volume' && !OBJECT_ID.test(id))) { + if (typeof id !== 'string' || typeof actual !== 'string') { console.warn(`${when}: removing ${kind} ${name}: docker printed no ID and name:`, found.stdout); return; } + // A container's or network's ID is checked too, as it is passed on as an argument. + if (kind !== 'volume' && !OBJECT_ID.test(id)) { + console.warn(`${when}: removing ${kind} ${name}: docker printed an ID that is not a full ID:`, found.stdout); return; + } if ((kind === 'container' ? actual.replace(/^\//, '') : actual) !== name) return; const remove = kind === 'container' ? ['rm', '--force', id] : [kind, 'rm', ...(kind === 'volume' ? ['--force'] : []), id]; const removed = spawnSync(docker, remove, options);