From 442bce08002e2ec61893e46734764a7b5c5662b2 Mon Sep 17 00:00:00 2001 From: Alex Lavaee Date: Tue, 20 Oct 2026 18:40:48 -0700 Subject: [PATCH] Keep a running build going through a Chopin restart A deploy ended every running build: connections, run credentials and the build's assignment lived only in memory, so the restarted server could not find the build again and ended it as server-restarted. A build's run credential is now signed with a key derived from SESSION_ENCRYPTION_KEY and names its document, build and kind, so a restarted server checks it again without storing it. It is accepted only for the connection that now holds its build, and refused once the build has ended. Protocol 6 connectors already name the document on every heartbeat and report, so after a restart the same paired machine and owner, on its new connection, takes the build over with the existing takeover rules; other machines and other people are still refused. The server waits 90 seconds after it starts for the machine to come back before ending a build nobody carried on, and until then answers the agent's calls with unavailable and a request to retry. Builds claimed by older connectors still end truthfully as server-restarted. The connector's build heartbeat no longer backs off past its 10-second interval while Chopin is out of reach, so a restarted server hears the build within its window. That part needs a CLI release; no protocol change. Assistant-model: Claude Opus 5.5 (fast) Assistant-workflow: inline (fast subagents) Assistant-verification: bun test passed (6670 pass, 0 fail) Assistant-verification: bun run types passed Assistant-verification: bun run ci passed Assistant-verification: e2e build restart, launcher, connector, machines and experiments passed (39) Co-authored-by: Alex Lavaee --- apps/connector/src/connection.test.ts | 4 +- apps/connector/src/connection.ts | 2 +- apps/connector/src/run.test.ts | 31 ++ apps/connector/src/run.ts | 11 +- .../connector/src/testing/fake-implementer.ts | 25 +- apps/server/src/experiments/connections.ts | 89 +++++- apps/server/src/experiments/routes.ts | 6 +- apps/server/src/tasks/builds.ts | 17 +- apps/server/src/tasks/connector.ts | 74 ++++- apps/server/src/tasks/routes.test.ts | 269 +++++++++++++++--- docs/chopin-cli.md | 5 +- docs/implementation-lifecycle.md | 7 +- docs/local-experiments.md | 30 +- docs/releasing-cli.md | 7 + e2e/build-restart.e2e.ts | 262 +++++++++++++++++ packages/protocol/implementation.d.ts | 5 + 16 files changed, 771 insertions(+), 73 deletions(-) create mode 100644 e2e/build-restart.e2e.ts diff --git a/apps/connector/src/connection.test.ts b/apps/connector/src/connection.test.ts index f995370c0..9f01e4d15 100644 --- a/apps/connector/src/connection.test.ts +++ b/apps/connector/src/connection.test.ts @@ -585,8 +585,8 @@ test("a connection failure's cause never shows credentials from the error", () = expect(text).not.toContain("abcdef0123456789xyz"); }); -test("machine, Planner and pairing key tokens are masked wherever chopin prints text", () => { - for (let prefix of ["chopin_m_", "chopin_p_", "chopin_k_"]) { +test("machine, Planner, pairing key and build run tokens are masked wherever chopin prints text", () => { + for (let prefix of ["chopin_m_", "chopin_p_", "chopin_k_", "chopin_r_"]) { let token = `${prefix}${"Zx9_-".repeat(9).slice(0, 43)}`; expect(redacted(`ran chopin connect --key ${token} -- agent`)).toBe( "ran chopin connect --key [redacted] -- agent", diff --git a/apps/connector/src/connection.ts b/apps/connector/src/connection.ts index c5dddfd05..53c40f12a 100644 --- a/apps/connector/src/connection.ts +++ b/apps/connector/src/connection.ts @@ -49,7 +49,7 @@ function stopped(reason: Exclude, error: unkn const USERINFO = /\b([a-z][a-z0-9+.-]*:\/\/)[^\s/@]+@/gi; const TOKEN = - /\b(?:gh[opsur]_|github_pat_|chopin_[kmp]_|sk-)[A-Za-z0-9_-]{8,}|\b(Bearer\s+)\S{8,}/g; + /\b(?:gh[opsur]_|github_pat_|chopin_[kmpr]_|sk-)[A-Za-z0-9_-]{8,}|\b(Bearer\s+)\S{8,}/g; /** * `text` with URL credentials, token-shaped strings and each of the `secrets` it knows, such as diff --git a/apps/connector/src/run.test.ts b/apps/connector/src/run.test.ts index 51a0cd077..53757c88e 100644 --- a/apps/connector/src/run.test.ts +++ b/apps/connector/src/run.test.ts @@ -598,6 +598,37 @@ test("a network drop mid-run keeps the agent running and delivers its completion expect(stopped).not.toBe(running); }); +test("a build keeps its agent through a server restart and its heartbeat is heard soon after", async () => { + let abort = new AbortController(); + let renewals: Array> = []; + let started = performance.now(); + let { documentId, calls } = await run( + {}, + ["--hang"], + (name, args) => { + if (name !== "renew_implementation_build") return {}; + renewals.push(args); + // Chopin is down for eight heartbeats, then forgets the connection the connector had. + if (renewals.length <= 8) throw unreachable(); + if (renewals.length === 9) { + throw new ConnectorError("connection-unavailable", "connection-unavailable"); + } + // It carries on with the build; the person then stops the agent from the terminal. + abort.abort(); + return { accepted: true }; + }, + abort.signal, + { random: () => 0.999 }, + ); + // Retries wait no longer than a heartbeat apart, rather than backing off for minutes, so a + // restarted Chopin hears the build again within its window. + expect(performance.now() - started).toBeLessThan(4000); + expect(renewals).toHaveLength(10); + // Each heartbeat names the document, so a restarted Chopin can find the build again. + expect(renewals.every(args => args.documentId === documentId)).toBe(true); + expect(calls.at(-1)?.[1]).toMatchObject({ state: "failed", error: STOPPED }); +}); + test("a run out of reach for its whole grace period stops its agent", async () => { let { id, calls } = await run( {}, diff --git a/apps/connector/src/run.ts b/apps/connector/src/run.ts index dbd3a7150..2914eaa5d 100644 --- a/apps/connector/src/run.ts +++ b/apps/connector/src/run.ts @@ -141,8 +141,11 @@ export async function runWork( abort.abort(); }, }); - // A heartbeat that fails while Chopin is out of reach retries on the reconnect backoff and - // leaves the agent running; only Chopin ending the run, or the grace period, stops it. + // A heartbeat that fails while Chopin is out of reach retries on the reconnect backoff, never + // waiting longer than a heartbeat apart, and leaves the agent running; only Chopin ending the + // run, or the grace period, stops it. A Chopin back from a restart hears the build again soon + // enough to let it carry on. + let heartbeatMs = options.heartbeatMs ?? 10_000; let beating = new AbortController(); // Ctrl-C, the run ending and the grace period running out each abandon a heartbeat in flight. let beat = AbortSignal.any([beating.signal, options.signal, reports.signal]); @@ -151,8 +154,8 @@ export async function runWork( while (!beat.aborted) { await delay( failures - ? Math.min(backoff(failures - 1, options.random), reports.remaining()) - : options.heartbeatMs ?? 10_000, + ? Math.min(backoff(failures - 1, options.random), heartbeatMs, reports.remaining()) + : heartbeatMs, beat, ); if (beat.aborted) return; diff --git a/apps/connector/src/testing/fake-implementer.ts b/apps/connector/src/testing/fake-implementer.ts index e539fafce..9171c5533 100644 --- a/apps/connector/src/testing/fake-implementer.ts +++ b/apps/connector/src/testing/fake-implementer.ts @@ -1,7 +1,7 @@ import { AgentApp, ndJsonStream, PROTOCOL_VERSION } from "@agentclientprotocol/sdk"; import type { McpServer } from "@agentclientprotocol/sdk"; import { spawn, spawnSync } from "node:child_process"; -import { writeFile } from "node:fs/promises"; +import { access, writeFile } from "node:fs/promises"; import { Readable, Writable } from "node:stream"; import { Client } from "@modelcontextprotocol/sdk/client/index.js"; import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; @@ -198,11 +198,31 @@ let app = new AgentApp() }, }), ); - let call = async (name: string, args: Record = {}) => { + let once = async (name: string, args: Record) => { let result = await client.callTool({ name, arguments: args }); if (result.isError) throw new Error(JSON.stringify(result.content)); return result; }; + // With CHOPIN_TEST_RESUME, an agent that retries a failed call for a minute, as one does + // while Chopin restarts. + let call = async (name: string, args: Record = {}) => { + for (let started = Date.now();;) { + try { + return await once(name, args); + } catch (error) { + if (!process.env.CHOPIN_TEST_RESUME || Date.now() - started > 60_000) throw error; + await new Promise(resolve => setTimeout(resolve, 500)); + } + } + }; + /** With CHOPIN_TEST_RESUME, say the build is under way and wait until told to go on. */ + let pause = async () => { + if (!process.env.CHOPIN_TEST_RESUME) return; + await writeFile(process.env.CHOPIN_TEST_PAUSED!, ""); + while (!await access(process.env.CHOPIN_TEST_RESUME).then(() => true, () => false)) { + await new Promise(resolve => setTimeout(resolve, 100)); + } + }; // A completing fixture lands a rebuild with one commit on the first pull request. if (rebuilding) { try { @@ -227,6 +247,7 @@ let app = new AgentApp() if (process.argv.includes("--complete")) { for (let [index, task] of context.graph.definition.tasks.entries()) { await call("start_task", { taskId: task.id, idempotencyKey: `start-${task.id}` }); + if (index === 0) await pause(); await call("report_pr", { taskId: task.id, idempotencyKey: `pr-${task.id}`, diff --git a/apps/server/src/experiments/connections.ts b/apps/server/src/experiments/connections.ts index 8ffc84c44..e36450a4b 100644 --- a/apps/server/src/experiments/connections.ts +++ b/apps/server/src/experiments/connections.ts @@ -1,4 +1,4 @@ -import { randomBytes } from "node:crypto"; +import { createHmac, hkdfSync, randomBytes, timingSafeEqual } from "node:crypto"; import type { Source } from "@chopin/experiment"; import { fail, fingerprint } from "./service"; @@ -79,6 +79,18 @@ const FRESH_MS = 45_000; /** The first connector protocol whose connection is the machine rather than one checkout. */ export const ANY_REPOSITORY_PROTOCOL = 5; +/** + * The first connector protocol whose build heartbeats and reports name the document, so a server + * that restarted can find the build again and its machine can carry on with it. + */ +export const RESUME_PROTOCOL = 6; + +/** A build's run, as its durable credential names it. */ +export type DurableRun = { id: string; documentId: string; kind: "implementation" | "rebuild" }; +const RUN_CONTEXT = "chopin:run-credential:v1"; +const RUN_PREFIX = "chopin_r_"; +/** An HMAC-SHA256 signature in unpadded base64url. */ +const SIGNATURE_LENGTH = 43; /** Whether a connection may take work in a repository. */ export function serves(connection: Connection, repositoryId: string) { @@ -135,7 +147,68 @@ export class Connections { #about = new Map(); /** Called once with each connection that ends: disconnected, expired, or its machine revoked. */ onEnded?: (connection: Connection) => void; - constructor(private now = () => Date.now()) {} + /** Signs build run credentials so they can be checked again after a restart; none without a key. */ + #runKey: Buffer | undefined; + constructor( + private now = () => Date.now(), + /** The deployment's SESSION_ENCRYPTION_KEY, from which build run credentials are derived. */ + key?: Uint8Array, + ) { + this.#runKey = key + ? Buffer.from(hkdfSync("sha256", key, "", RUN_CONTEXT, 32)) + : undefined; + } + /** Whether build run credentials outlive this process, so a build can resume after a restart. */ + get durable() { + return !!this.#runKey; + } + #sign(payload: string) { + return createHmac("sha256", this.#runKey!).update(payload).digest(); + } + /** + * A build's run credential: its run, signed with a key derived from the deployment's own, so a + * restarted server can check it without having stored it. The build's state still decides + * whether it is accepted: it ends with the build. + */ + #durableToken(run: DurableRun) { + let payload = Buffer.from([run.documentId, run.id, run.kind].join("\n")).toString("base64url"); + // The signature first, at its fixed length, keeps the whole token one base64url word. + return `${RUN_PREFIX}${this.#sign(payload).toString("base64url")}${payload}`; + } + /** The run a durable build credential names, when this deployment signed it. */ + durableRun(token: string): DurableRun | undefined { + if (!this.#runKey || !token.startsWith(RUN_PREFIX) || token.length > 512) return undefined; + let body = token.slice(RUN_PREFIX.length); + let signature = body.slice(0, SIGNATURE_LENGTH); + let payload = body.slice(SIGNATURE_LENGTH); + if (!payload || !/^[A-Za-z0-9_-]+$/.test(body)) return undefined; + let given = Buffer.from(signature, "base64url"); + let expected = this.#sign(payload); + if (given.length !== expected.length || !timingSafeEqual(given, expected)) return undefined; + let [documentId, id, kind, rest] = Buffer.from(payload, "base64url").toString("utf8") + .split("\n"); + if (!documentId || !id || rest !== undefined) return undefined; + if (kind !== "implementation" && kind !== "rebuild") return undefined; + return { documentId, id, kind }; + } + /** + * Accept a durable build credential this process has not seen, as after a restart, for the + * connection that now holds its build. + */ + adopt(token: string, connectionId: string, run: DurableRun) { + if (!this.#connections.has(connectionId)) return; + this.#runTokens.set(`${connectionId}:${run.id}:1:${run.kind}`, token); + this.#tokens.set(fingerprint(token), { + connectionId, + run: { id: run.id, documentId: run.documentId, generation: 1, kind: run.kind }, + }); + this.#remember(run.id, run.documentId); + } + /** Whether this process already holds a credential, or refused it. */ + holds(token: string) { + let key = fingerprint(token); + return this.#tokens.has(key) || this.#revoked.has(key); + } create(input: PairingInput) { this.sweep(); if (this.#pending.size >= 100) fail("pairing-capacity"); @@ -236,7 +309,7 @@ export class Connections { } /** Whether a token is a run's, which the coding agent itself presents. */ run(token: string) { - return !!this.#tokens.get(fingerprint(token))?.run; + return !!this.#tokens.get(fingerprint(token))?.run || !!this.durableRun(token); } lookup(token: string) { let refused = this.#revoked.get(fingerprint(token)); @@ -411,19 +484,25 @@ export class Connections { let key = `${connectionId}:${id}:${generation}:${kind ?? "experiment"}`; let existing = this.#runTokens.get(key); if (existing) return existing; - let token = randomBytes(32).toString("base64url"); + // A build's credential can be checked again after a restart; an investigation's cannot. + let token = kind && this.#runKey + ? this.#durableToken({ id, documentId, kind }) + : randomBytes(32).toString("base64url"); this.#runTokens.set(key, token); this.#tokens.set(fingerprint(token), { connectionId, run: { id, documentId, generation, ...(kind ? { kind } : {}) }, }); + this.#remember(id, documentId); + return token; + } + #remember(id: string, documentId: string) { this.#runs.delete(id); this.#runs.set(id, documentId); for (let old of this.#runs.keys()) { if (this.#runs.size <= RUNS) break; this.#runs.delete(old); } - return token; } /** The document of a run this process handed out, whichever connection holds it now. */ runDocument(id: string) { diff --git a/apps/server/src/experiments/routes.ts b/apps/server/src/experiments/routes.ts index a0b694dce..f135f6f40 100644 --- a/apps/server/src/experiments/routes.ts +++ b/apps/server/src/experiments/routes.ts @@ -257,7 +257,7 @@ function described(live: Connection[]) { } export function registerExperimentRoutes(router: Router, auth: HostedAuth, options: Options) { - let connections = new Connections(); + let connections = new Connections(undefined, auth.config.encryptionKey); /** * The stored machine choices, read before any work is routed. A read that fails is retried a * few times, then refuses the work with `machine-choice-unavailable`; it is never taken to mean @@ -1396,6 +1396,10 @@ export function registerExperimentRoutes(router: Router, auth: HostedAuth, optio * A connection that serves any repository is checked per document, as its work is claimed. */ async function connector(token: string) { + // A build's run credential from before a restart is accepted again for the connection that + // now holds its build, once its machine has come back. + let durable = !connections.holds(token) ? connections.durableRun(token) : undefined; + if (durable) await options.implementations?.()?.adopt(token, durable); let found = connections.lookup(token); let machine: StoredMachine; try { diff --git a/apps/server/src/tasks/builds.ts b/apps/server/src/tasks/builds.ts index e414f5016..e67a989c6 100644 --- a/apps/server/src/tasks/builds.ts +++ b/apps/server/src/tasks/builds.ts @@ -57,6 +57,7 @@ let buildSchema = z.object({ at: z.string().datetime(), }).strict().optional(), reportKey: z.string().uuid().optional(), + resumable: z.literal(true).optional(), kind: z.literal("rebuild").optional(), baseRevision: z.number().int().nonnegative().optional(), targetRevision: z.number().int().nonnegative().optional(), @@ -470,6 +471,8 @@ export function pickBuild( plan: Plan, user: string, connectionId: string, + /** Its run credential outlives a restart and its connector names the document as it reports. */ + resumable = false, ): Promise { return exclusive(plan, async () => { let build = plan.builds.at(-1); @@ -480,7 +483,12 @@ export function pickBuild( return; } let previous = plan.builds; - let next: BuildRequest = { ...build, state: "starting", expiresAt: Date.now() + 60_000 }; + let next: BuildRequest = { + ...build, + state: "starting", + expiresAt: Date.now() + 60_000, + ...(resumable ? { resumable: true as const } : {}), + }; plan.builds = [...previous.slice(0, -1), next]; try { await persistExclusive(plan, true); @@ -507,7 +515,12 @@ export function rebindBuild( let build = plan.builds.at(-1); if (!build || build.id !== id || build.connectionId !== from) return false; let previous = plan.builds; - plan.builds = [...previous.slice(0, -1), { ...build, connectionId: to }]; + // A build its machine carries on with after a restart may have gone longer than a heartbeat. + plan.builds = [...previous.slice(0, -1), { + ...build, + connectionId: to, + expiresAt: Math.max(build.expiresAt, Date.now() + 60_000), + }]; try { await persistExclusive(plan, true); } catch (error) { diff --git a/apps/server/src/tasks/connector.ts b/apps/server/src/tasks/connector.ts index 95a7536fa..acb3cca52 100644 --- a/apps/server/src/tasks/connector.ts +++ b/apps/server/src/tasks/connector.ts @@ -1,6 +1,6 @@ import { z } from "zod"; -import { serves } from "../experiments/connections"; -import type { Connection, Connections, Grant } from "../experiments/connections"; +import { RESUME_PROTOCOL, serves } from "../experiments/connections"; +import type { Connection, Connections, DurableRun, Grant } from "../experiments/connections"; import { ExperimentError } from "@chopin/experiment"; import { fail } from "../experiments/service"; import { LIFECYCLE_TOOLS, lifecycleCall } from "../mcp/lifecycle"; @@ -25,6 +25,14 @@ export const RESTARTED = "Chopin restarted, so this build ended."; export const DISCONNECTED = "Your local agent disconnected."; /** The build's heartbeat stopped while its machine may still be connected. */ export const UNRESPONSIVE = "Your local agent stopped responding."; +/** A run credential's build is resumable, but its machine has not come back since the restart. */ +export const RESUMING = + "Chopin restarted and is waiting for this machine's connector to come back. Retry in a few seconds."; +/** + * How long after it starts a server waits for a machine to come back and carry on with a build it + * was running before the restart: a forgotten connection's window, 90 seconds. + */ +export const RESUME_MS = 90_000; export type WithPlan = (id: string, action: (plan: Plan) => Promise) => Promise; @@ -149,6 +157,15 @@ export function implementationConnector( prototyping?: (connectionId: string) => Promise, options: { retryMs?: number; retries?: number } = {}, ) { + /** When this process started; a build it never connected may still resume until the window ends. */ + let started = Date.now(); + /** + * Whether a build was claimed before this process started, by a connection it never knew, and + * may still be resumed by its machine. + */ + let resuming = (build: BuildRequest) => + !!build.resumable && !connections.known(build.connectionId) + && ["starting", "running"].includes(build.state) && Date.now() < started + RESUME_MS; let tracked = new Set(); // Lifecycle calls of one run apply in arrival order, one at a time. let chains = new Map>(); @@ -187,6 +204,27 @@ export function implementationConnector( track(id: string) { tracked.add(id); }, + /** + * Accept a build's durable run credential again after a restart, for the connection that holds + * its build now. Until its machine comes back and takes the build over, the agent is asked to + * retry; a credential whose build has ended is refused. + */ + async adopt(token: string, run: DurableRun) { + if (!await documentExists(run.documentId)) return; + await withPlan(run.documentId, async plan => { + let build = plan.builds.find(item => item.id === run.id); + if (!build?.resumable || (build.kind === "rebuild") !== (run.kind === "rebuild")) return; + tracked.add(plan.id); + if (connections.get(build.connectionId)) { + connections.adopt(token, build.connectionId, run); + return; + } + if (build !== plan.builds.at(-1) || !["starting", "running"].includes(build.state)) { + fail("build-inactive"); + } + if (resuming(build)) fail("unavailable", RESUMING); + }); + }, async waiting(connection: Connection) { let build = await pending(connection); return build?.state === "queued" @@ -310,18 +348,27 @@ export function implementationConnector( } } let build = plan.builds.at(-1); - // A document hint survives restart; it must not revive a forgotten connection. - if (build && !grant.run && build.id === args.id && !connections.known(build.connectionId)) { + // Chopin restarted since the build's connection opened. Its machine may carry on with a + // resumable build for a while; any other is over. + let restarted = !!build && !grant.run && build.id === args.id + && !connections.known(build.connectionId); + if (restarted && !resuming(build!)) { + // One that ended since, as when someone stopped it in Chopin, says so instead. + if (build!.resumable && !["starting", "running"].includes(build!.state)) { + refuse("build-inactive", build); + } refuse("server-restarted", build, RESTARTED); } - // The connector registered its machine again, as after a dropped request, so the build's - // heartbeat and reports now come on a newer connection of the same machine and owner. + // The connector registered its machine again, as after a dropped request or a restart, so + // the build's heartbeat and reports now come on a newer connection of the same machine and + // owner. if ( build && !grant.run && build.connectionId !== connection.id && build.id === args.id && build.machineId === connection.machineId && build.user === connection.owner && serves(connection, build.repositoryId) && ["renew_implementation_build", "report_implementation_build"].includes(name) - && ["starting", "running"].includes(build.state) && build.expiresAt > Date.now() + && ["starting", "running"].includes(build.state) + && (restarted || build.expiresAt > Date.now()) // A connection already given another document's build is not this build's to take. && (connections.assigned(connection.id) ?? documentId) === documentId ) { @@ -331,7 +378,7 @@ export function implementationConnector( console.info( `chopin: build ${build.id} of document ${plan.id} moved from connection ${from} ` + `to ${connection.id} of machine ${connection.machineId}, which registered ` - + `again; ${name} continues it`, + + `again${restarted ? " after Chopin restarted" : ""}; ${name} continues it`, ); build = plan.builds.at(-1)!; } @@ -393,7 +440,12 @@ export function implementationConnector( } if (name === "claim_implementation_build") { if (build.expiresAt <= Date.now()) fail("build-expired"); - let picked = await pickBuild(plan, connection.owner, connection.id); + let picked = await pickBuild( + plan, + connection.owner, + connection.id, + connections.durable && (connection.protocol ?? 1) >= RESUME_PROTOCOL, + ); if (!picked) fail("build-already-claimed"); return { build: picked, @@ -518,12 +570,14 @@ export function implementationConnector( await renewQueued(plan, build.id); return; } + // Its machine may still come back after the restart and carry on with it. + if (resuming(build)) return; if (!connected || build.expiresAt <= Date.now()) { // Say what happened: Chopin restarted since the build's connection opened, the // machine is gone, or its agent went quiet while the machine stayed connected. let error = connected ? UNRESPONSIVE - : !connections.known(build.connectionId) + : !connections.known(build.connectionId) && !build.resumable ? RESTARTED : build.machineId && connections.machine(build.machineId).length ? UNRESPONSIVE diff --git a/apps/server/src/tasks/routes.test.ts b/apps/server/src/tasks/routes.test.ts index 9197e0b8f..368bf637e 100644 --- a/apps/server/src/tasks/routes.test.ts +++ b/apps/server/src/tasks/routes.test.ts @@ -209,6 +209,8 @@ async function setup( token?: string, origin = config.origin, cookie = issued.cookie, + /** The connector protocol the caller speaks; null sends none, as a coding agent does. */ + protocol: number | null = CONNECTOR_PROTOCOLS.max, ) => { let response = await router.handle( new Request(config.origin + url, { @@ -218,7 +220,7 @@ async function setup( cookie: cookie.split(";")[0], ...(token ? { authorization: `Bearer ${token}` } : {}), // What a current connector sends; browser routes ignore it. - [CONNECTOR_PROTOCOL_HEADER]: String(CONNECTOR_PROTOCOLS.max), + ...(protocol === null ? {} : { [CONNECTOR_PROTOCOL_HEADER]: String(protocol) }), }, ...(value ? { body: JSON.stringify(value) } : {}), }), @@ -252,6 +254,7 @@ async function paired( context: Awaited>, cookie?: string, autoRun?: boolean, + protocol?: number, ) { let pairing = await (await context.call("/api/connector/pairings", { ...checkout, label: "Laptop" })).json(); @@ -266,7 +269,7 @@ async function paired( secret: pairing.secret, })).json(); // `chopin connect` registers the checkout with the machine token. - let connection = await reconnect(context, machine.token, cookie); + let connection = await reconnect(context, machine.token, cookie, protocol); return { ...connection, machineToken: machine.token as string }; } /** The paired machine registers again with its machine token, as the connector does after a drop. */ @@ -274,6 +277,7 @@ async function reconnect( context: Awaited>, machine: string, cookie?: string, + protocol?: number, ) { return (await context.call( "/api/connector/connections", @@ -281,6 +285,7 @@ async function reconnect( machine, undefined, cookie, + protocol, )).json(); } async function tool( @@ -288,14 +293,26 @@ async function tool( token: string, name: string, args: unknown = {}, + protocol?: number | null, ) { - let response = await context.call("/connector/mcp", { - jsonrpc: "2.0", - id: 1, - method: "tools/call", - params: { name, arguments: args }, - }, token); - let result = (await response.json()).result; + let response = await context.call( + "/connector/mcp", + { + jsonrpc: "2.0", + id: 1, + method: "tools/call", + params: { name, arguments: args }, + }, + token, + undefined, + undefined, + protocol, + ); + let text = await response.text(); + let body = JSON.parse(text); + // A refusal before the tool runs, such as an unknown credential, is the HTTP answer itself. + if (!body.result) return { status: response.status, ...body }; + let result = body.result; return result.isError ? result : JSON.parse(result.content[0].text); } @@ -934,46 +951,222 @@ test.each([false, true])( }, ); -test.each([false, true])("a restart still ends a build (hint: %s)", async hinted => { - let context = await setup(); - let first = await paired(context); +/** + * A clock the test moves on, as time passes in production: the process renews its writer lease as + * it goes. + */ +function clockFor(context: Awaited>, start = Date.now()) { + let now = start; + setSystemTime(new Date(now)); + return { + get now() { + return now; + }, + async advance(ms: number) { + for (let left = ms; left > 0; left -= 10_000) { + await context.storage.leases.renew(context.lease, 60_000); + now += Math.min(left, 10_000); + setSystemTime(new Date(now)); + } + await context.storage.leases.renew(context.lease, 60_000); + }, + }; +} + +/** A build claimed and running on a newly paired machine whose connector speaks `protocol`. */ +async function running(context: Awaited>, protocol?: number) { + let connection = await paired(context, undefined, undefined, protocol); let built = await (await context.call(context.path, { planRevision: 0, graphVersion: 1, graphRevision: 1, })).json(); - await tool(context, first.token, "claim_implementation_build", { id: built.id }); - await tool(context, first.token, "report_implementation_build", { + let claim = await tool( + context, + connection.token, + "claim_implementation_build", + { id: built.id }, + protocol, + ); + await tool(context, connection.token, "report_implementation_build", { id: built.id, state: "running", session: "acp-session", - }); - context.restart(); - // Its connection is forgotten, so the connector registers the machine again. - let forgotten = await context.call("/connector/mcp", { - jsonrpc: "2.0", - id: 1, - method: "tools/call", - params: { name: "renew_implementation_build", arguments: { id: built.id } }, - }, first.token); - expect(await forgotten.text()).toContain("connection-unavailable"); - // The connector registers again and its heartbeat goes on, which Chopin can no longer place. - let second = await reconnect(context, first.machineToken); - let ended = "Chopin restarted, so this build ended."; - expect( - (await tool(context, second.token, "renew_implementation_build", { + }, protocol); + return { connection, built, claim }; +} + +test("an older connector's build still ends at a restart, saying so", async () => { + let context = await setup(); + try { + // Protocol 5 names no document on its heartbeat, so Chopin cannot find the build again. + let { connection, built } = await running(context, 5); + context.restart(); + // Its connection is forgotten, so the connector registers the machine again. + let forgotten = await tool(context, connection.token, "renew_implementation_build", { id: built.id, - ...(hinted ? { documentId: context.plan.id } : {}), - })) - .content[0].text, - ).toBe(`server-restarted: ${ended}`); - // Opening the document tracks its build again, and the sweep ends it with the same reason. - await context.call(context.path); - await context.implementations.sweep(); - expect(context.plan.builds.at(-1)).toMatchObject({ state: "failed", error: ended }); - await Plan.close(context.plan); + }, 5); + expect(forgotten.error).toBe("connection-unavailable"); + let second = await reconnect(context, connection.machineToken, undefined, 5); + let ended = "Chopin restarted, so this build ended."; + expect( + (await tool(context, second.token, "renew_implementation_build", { id: built.id }, 5)) + .content[0].text, + ).toBe(`server-restarted: ${ended}`); + // Opening the document tracks its build again, and the sweep ends it with the same reason. + await context.call(context.path); + await context.implementations.sweep(); + expect(context.plan.builds.at(-1)).toMatchObject({ state: "failed", error: ended }); + } finally { + await Plan.close(context.plan); + } }); +test("a build carries on across a server restart once its machine comes back", async () => { + let context = await setup(); + let clock = clockFor(context); + try { + let { connection, built, claim } = await running(context); + // The coding agent calls Chopin with only its run credential, and no connector protocol. + let agent = (name: string, args: Record) => + tool(context, claim.runToken, name, args, null); + expect(await agent("start_task", { taskId: "first", idempotencyKey: "start" })) + .toEqual({ state: "accepted" }); + // The deploy takes two minutes, longer than a heartbeat keeps a build alive. + await clock.advance(120_000); + context.restart(); + let pr = { + taskId: "first", + idempotencyKey: "pr", + url: "https://github.com/owner/repository/pull/7", + state: "open", + }; + // The agent reports before its connector is back: it is asked to retry, and nothing ends. + expect(await agent("report_pr", pr)).toMatchObject({ status: 503, error: "unavailable" }); + let hint = { id: built.id, documentId: context.plan.id }; + expect(await tool(context, connection.token, "renew_implementation_build", hint)) + .toMatchObject({ error: "connection-unavailable" }); + // Another machine of the same person, and another person's machine, cannot take it over. + for (let other of [await paired(context), await paired(context, context.otherCookie)]) { + let refused = await tool(context, other.token, "renew_implementation_build", hint); + expect(refused.content[0].text).toMatch(/^build-forbidden:/); + } + // The connector registers its machine again and its heartbeat carries on with the build. + let second = await reconnect(context, connection.machineToken); + expect( + await tool(context, second.token, "renew_implementation_build", { + ...hint, + progress: "Working", + }), + ).toEqual({ accepted: true }); + expect(context.plan.builds.at(-1)).toMatchObject({ + state: "running", + connectionId: second.connection.id, + }); + expect(await agent("report_pr", pr)).toEqual({ state: "accepted" }); + expect( + await agent("complete_task", { taskId: "first", idempotencyKey: "done", summary: "Done" }), + ).toEqual({ state: "accepted" }); + expect( + await tool(context, second.token, "report_implementation_build", { + ...hint, + state: "stopped", + }), + ).toEqual({ accepted: true }); + expect(context.plan.builds.at(-1)).toMatchObject({ state: "stopped" }); + expect(context.plan.builds.at(-1)!.error).toBeUndefined(); + // Its run credential ended with the build. + let late = await agent("start_task", { taskId: "first", idempotencyKey: "again" }); + expect(late.content[0].text).toMatch(/^build-inactive:/); + } finally { + setSystemTime(); + await Plan.close(context.plan); + } +}); + +test("a run credential Chopin did not issue is refused after a restart", async () => { + let context = await setup(); + try { + let { claim } = await running(context); + context.restart(); + await context.call(context.path); + let forged = claim.runToken.slice(0, -2) + (claim.runToken.endsWith("AA") ? "BB" : "AA"); + for (let protocol of [null, undefined]) { + let refused = await tool(context, forged, "start_task", { + taskId: "first", + idempotencyKey: "start", + }, protocol); + expect(refused.status).toBe(409); + expect(refused.error).toBe( + protocol === null ? "connector-update-required" : "connection-unavailable", + ); + } + let snapshot = await (await context.call(context.path)).json(); + expect(snapshot.lifecycle.activity.tasks[0].state).toBe("queued"); + } finally { + await Plan.close(context.plan); + } +}); + +test("a build that ended before a restart stays ended", async () => { + let context = await setup(); + try { + let { connection, built, claim } = await running(context); + expect((await context.call(`${context.path}/stop`, { id: built.id })).status).toBe(200); + context.restart(); + let second = await reconnect(context, connection.machineToken); + let renewed = await tool(context, second.token, "renew_implementation_build", { + id: built.id, + documentId: context.plan.id, + }); + expect(renewed.content[0].text).toMatch(/^build-inactive:/); + let agent = await tool(context, claim.runToken, "start_task", { + taskId: "first", + idempotencyKey: "start", + }, null); + expect(agent.content?.[0].text ?? agent.error).toMatch(/^build-inactive/); + expect(context.plan.builds.at(-1)).toMatchObject({ + state: "failed", + error: "Stopped in Chopin.", + }); + } finally { + await Plan.close(context.plan); + } +}); + +test.each([ + ["comes back without it", "Your local agent stopped responding."], + ["never comes back", "Your local agent disconnected."], +])( + "a build its machine does not resume after a restart waits out the window, then ends (machine %s)", + async (machine, error) => { + let context = await setup(); + let clock = clockFor(context); + try { + let { connection } = await running(context); + context.restart(); + // Someone opens the document, so Chopin looks at its build again. + await context.call(context.path); + let after = async (ms: number) => { + await clock.advance(ms); + await context.implementations.sweep(); + return context.plan.builds.at(-1)!; + }; + expect(await after(0)).toMatchObject({ state: "running" }); + expect(await after(60_000)).toMatchObject({ state: "running" }); + // A connector whose agent stopped meanwhile registers its machine but never renews the build. + if (machine === "comes back without it") { + await reconnect(context, connection.machineToken); + } + expect(await after(25_000)).toMatchObject({ state: "running" }); + expect(await after(10_000)).toMatchObject({ state: "failed", error }); + } finally { + setSystemTime(); + await Plan.close(context.plan); + } + }, +); + test("a build whose heartbeat stops while its machine stays connected says the agent stopped responding", async () => { let context = await setup(); let connection = await paired(context); diff --git a/docs/chopin-cli.md b/docs/chopin-cli.md index 6e3b4d27f..5331e30b8 100644 --- a/docs/chopin-cli.md +++ b/docs/chopin-cli.md @@ -157,7 +157,10 @@ your Chopin account in a browser: 5. Choose **Add machine**. The page says **Connected.** The pairing lasts until you [disconnect the machine](#disconnect). It survives -server restarts, deploys and signing out of the browser. +server restarts, deploys and signing out of the browser. So does a build +running on the machine: the connector keeps its agent working while Chopin +restarts, registers the machine again, and its heartbeat carries the build on +(see [Trouble connecting](local-experiments.md#trouble-connecting)). When the connection is up, you see: diff --git a/docs/implementation-lifecycle.md b/docs/implementation-lifecycle.md index d537e4540..2026183ee 100644 --- a/docs/implementation-lifecycle.md +++ b/docs/implementation-lifecycle.md @@ -290,7 +290,12 @@ press, so its trust model is narrow on purpose. connection that ran a rebuild, and whether their agent is online. A rebuild's run token reaches only `read_rebuild` and `report_rebuild`; a -build's run token cannot report a rebuild. A report repeated after the rebuild +build's run token cannot report a rebuild. A build's run token names its +document, build and kind, signed with a key derived from `SESSION_ENCRYPTION_KEY`, +so a restarted server can check it without storing it; the build's state still +decides whether it is accepted, so it ends with the build. See +[Trouble connecting](local-experiments.md#trouble-connecting) for how a build +resumes after a restart. A report repeated after the rebuild landed returns the same success. The build history keeps 100 entries; ended rebuilds give way oldest first so a long-lived document keeps rebuilding. diff --git a/docs/local-experiments.md b/docs/local-experiments.md index ed80ecc41..ac7b85dd8 100644 --- a/docs/local-experiments.md +++ b/docs/local-experiments.md @@ -177,7 +177,8 @@ when it is back. It stops instead of retrying when Chopin refuses the machine, requires a newer connector, or the owner has lost push access to the repository. A running agent keeps working through a drop. Its heartbeat retries on the same -backoff, and its session and completion reports wait in memory, in order, until +backoff, but never waits longer than its usual 10 seconds between tries, and its +session and completion reports wait in memory, in order, until Chopin answers. The connector stops the agent only when Chopin says the run is over, after 10 minutes without reaching Chopin, or after two minutes in which Chopin only said it no longer knows the connection. Ctrl-C stops the agent at @@ -206,11 +207,28 @@ on the way (a long poll cut by a proxy, a timeout, a 5xx answer); the terminal s Chopin again**. A build carries on through that: its next heartbeat comes on the new connection of the same paired machine and owner, and Chopin moves the build, and the agent's run credential, to it. A connection of any other machine, or of -another person, is still refused (`build-forbidden`). A build that was going at a -restart cannot go on, as Chopin forgot it with every connection: the connector -stops its agent with **Chopin restarted, so it ended this run.** (an older -connector says **Chopin ended this run (server-restarted).**) and waits for the -next work, and Build says **Build stopped — Chopin restarted.** Build says **your +another person, is still refused (`build-forbidden`). + +A build also goes on through a server restart or a deploy. Its heartbeat and +reports name the document (connector protocol 6, chopin 0.0.16 and later), so +the restarted Chopin finds the build in the document and hands it to the new +connection of the same paired machine and owner, as above. The agent's run +credential is signed with a key derived from the deployment's +`SESSION_ENCRYPTION_KEY` rather than kept in memory, so Chopin checks it again +after the restart, accepts it only for the connection that now holds its build, +and refuses it once the build has ended. Until the machine is back, the agent's +calls are answered `unavailable` (HTTP 503) with a request to retry. Chopin +waits 90 seconds after it starts for the machine to come back: only then does +it end a build nobody carried on, saying **your local agent stopped responding** +when the machine reconnected without it and **your local agent disconnected** +when it did not. A build that ended before the restart stays ended +(`build-inactive`). + +A build claimed by an older connector, or by a server without this change, +cannot go on, as Chopin cannot find it again: the connector stops its agent with +**Chopin restarted, so it ended this run.** (an older connector says **Chopin +ended this run (server-restarted).**) and waits for the next work, and Build +says **Build stopped — Chopin restarted.** Build says **your local agent disconnected** only when the machine has no connection left, and **your local agent stopped responding** when its heartbeat stopped while the machine stayed connected. When Chopin has answered diff --git a/docs/releasing-cli.md b/docs/releasing-cli.md index d0fd460be..3a3720bfa 100644 --- a/docs/releasing-cli.md +++ b/docs/releasing-cli.md @@ -137,6 +137,13 @@ The binaries are not code-signed or notarized. Installs through `curl` and ## Notes for the next release +- A build's heartbeat retries no longer back off past its 10-second interval + while Chopin is out of reach, so a server back from a restart hears a running + build within its 90-second resume window. This needs no new protocol: a + protocol 6 connector (chopin 0.0.16) already names the document, and the + server lets its builds resume too, though one with a long backoff may miss + the window after a long outage. + - Development builds that installed a service must run `chopin connect`, then `chopin service install` again: the service now runs its own copy of the settings, which older builds never saved. diff --git a/e2e/build-restart.e2e.ts b/e2e/build-restart.e2e.ts new file mode 100644 index 000000000..67a60c806 --- /dev/null +++ b/e2e/build-restart.e2e.ts @@ -0,0 +1,262 @@ +/** + * A build carries on through a Chopin restart: the paired connector registers its machine again, + * its heartbeat resumes the build and the agent's run credential still reports, so it ends Built. + * Restarting needs a server of its own, so this file starts one against a database of its own. + */ +import { execFileSync, spawn } from "node:child_process"; +import { access, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { SQL } from "bun"; + +import { createChannel, registerTestDatabase, seedChannel, testChannelPath } from "./database"; +import { authenticate, expect, test } from "./room"; +import { HOST, ROOT } from "./servers"; + +import type { Graph } from "../apps/server/src/tasks/graphs"; + +const PORT = 8795; +const ORIGIN = `http://${HOST}:${PORT}`; +const DATABASE = "chopin_build_restart"; + +function databaseUrls() { + let cluster = new URL( + process.env.E2E_DATABASE_URL_3 + || "postgresql://chopin:chopin@127.0.0.1:5436/chopin?sslmode=disable", + ); + let own = new URL(cluster); + own.pathname = `/${DATABASE}`; + return { cluster: cluster.href, own: own.href }; +} + +async function freshDatabase(): Promise { + let { cluster, own } = databaseUrls(); + let admin = new SQL(cluster); + try { + await admin.unsafe(`DROP DATABASE IF EXISTS ${DATABASE} WITH (FORCE)`); + await admin.unsafe(`CREATE DATABASE ${DATABASE}`); + } finally { + await admin.close(); + } + return own; +} + +const ENV = { + SERVER_HOST: HOST, + PORT: String(PORT), + APP_ORIGIN: ORIGIN, + STORAGE_DRIVER: "postgres", + JEV_API_KEY: "e2e-jev-only", + GITHUB_APP_SLUG: "chopin-e2e", + GITHUB_APP_CLIENT_ID: "e2e", + GITHUB_APP_CLIENT_SECRET: "e2e", + GITHUB_ALLOWED_USERS: "", + GITHUB_ALLOWED_ORGANIZATIONS: "githubnext", + SESSION_ENCRYPTION_KEY: process.env.SESSION_ENCRYPTION_KEY!, + AGENT: "off", + BACKGROUND_JOBS: "off", + WEB_RESEARCH: "off", + CONVERSATION_PLAN: "off", + DEV_QUESTIONS: "", + DEV_COMMENTS: "", +}; + +let processes: ReturnType[] = []; +let directories: string[] = []; + +// One disposable database and application port for the file. +test.describe.configure({ mode: "default" }); + +async function stop(child: ReturnType) { + processes = processes.filter(value => value !== child); + if (child.exitCode !== null || child.pid === undefined) return; + let exit = new Promise(resolve => child.once("close", () => resolve())); + process.kill(-child.pid, "SIGTERM"); + await exit; +} + +/** Start Chopin on the file's database; what it logs is kept. */ +async function server(database: string) { + let child = spawn(process.execPath, ["--preload", "./e2e/github.ts", "apps/server/src/main.ts"], { + cwd: ROOT, + detached: true, + env: { ...process.env, ...ENV, DATABASE_URL: database }, + stdio: ["ignore", "pipe", "pipe"], + }); + let log = { text: "" }; + let read = (chunk: Buffer) => { + log.text = (log.text + String(chunk)).slice(-20_000); + }; + child.stdout?.on("data", read); + child.stderr?.on("data", read); + processes.push(child); + for (let index = 0; index < 150; index++) { + if (child.exitCode !== null) { + throw new Error(`server exited with ${child.exitCode}: ${log.text.slice(-800)}`); + } + try { + if ((await fetch(`${ORIGIN}/api/session`)).ok) return { child, log }; + } catch { /* Not bound yet. */ } + await new Promise(resolve => setTimeout(resolve, 100)); + } + throw new Error("server did not become ready"); +} + +async function migrated(database: string) { + let migration = Bun.spawn(["bun", "apps/server/src/storage/migrate.ts"], { + cwd: ROOT, + env: { ...process.env, ...ENV, DATABASE_URL: database }, + stdout: "ignore", + stderr: "pipe", + }); + if (await migration.exited !== 0) { + throw new Error( + `migration failed: ${(await new Response(migration.stderr).text()).slice(-800)}`, + ); + } +} + +function graph(): Graph { + return { + versions: [{ + number: 1, + revision: 1, + planRevision: 0, + state: "draft", + definition: { + tasks: [{ + id: "connect", + title: "Connect the local agent", + context: "An isolated tracer.", + goal: "Report through MCP across a restart.", + acceptance: ["The agent can read the graph.", "Its reports reach Chopin."], + dependsOn: [], + }, { + id: "review", + title: "Review the connection", + context: "Wait for the connection.", + goal: "Verify progress survives the restart.", + acceptance: ["The progress persists.", "The dependency is visible."], + dependsOn: ["connect"], + }], + }, + }], + }; +} + +/** `chopin connect` of a fixture checkout, its agent the fake implementer. */ +async function connector(env: Record) { + let root = await mkdtemp(join(tmpdir(), "chopin-restart-checkout-")); + let state = await mkdtemp(join(tmpdir(), "chopin-restart-state-")); + directories.push(root, state); + let git = (...args: string[]) => execFileSync("git", ["-C", root, ...args], { stdio: "pipe" }); + git("init", "-b", "main"); + git("config", "user.name", "Fixture"); + git("config", "user.email", "fixture@example.test"); + git("remote", "add", "origin", "https://github.com/octo-org/score.git"); + await writeFile(join(root, "sample.txt"), "committed source"); + git("add", "."); + git("commit", "-m", "fixture"); + let child = spawn("bun", [ + join(ROOT, "apps/connector/src/main.ts"), + "connect", + root, + "--", + "bun", + join(ROOT, "apps/connector/src/testing/fake-implementer.ts"), + "--complete", + "--http", + ], { + cwd: ROOT, + detached: true, + env: { ...process.env, CHOPIN_URL: ORIGIN, CHOPIN_CONNECTOR_STATE_DIR: state, ...env }, + stdio: ["ignore", "pipe", "pipe"], + }); + processes.push(child); + let output = ""; + let pairing = new Promise((resolve, reject) => { + let timer = setTimeout(() => reject(new Error(`Pairing timed out: ${output}`)), 15_000); + let read = (chunk: Buffer) => { + output += String(chunk); + let match = output.match(/http:\/\/[^\s]+\/machines\?pairing=[\w-]+/); + if (match) { + clearTimeout(timer); + resolve(match[0]); + } + }; + child.stdout.on("data", read); + child.stderr.on("data", read); + child.once("exit", code => { + clearTimeout(timer); + reject(new Error(`Connector exited ${code}: ${output}`)); + }); + }); + return { pairing, output: () => output }; +} + +async function exists(path: string) { + return access(path).then(() => true, () => false); +} + +test.afterEach(async () => { + for (let child of [...processes].toReversed()) await stop(child); + for (let directory of directories.splice(0)) { + await rm(directory, { recursive: true, force: true }); + } +}); + +test("a build running across a Chopin restart finishes Built", async ({ page }) => { + test.setTimeout(150_000); + let database = await freshDatabase(); + await migrated(database); + let chopin = await server(database); + registerTestDatabase(PORT, database); + let room = crypto.randomUUID(); + await createChannel(PORT, room); + await seedChannel(PORT, room, "# Restart tracer\n\nBuild through a deploy.\n", { + graph: graph(), + }); + + let signals = await mkdtemp(join(tmpdir(), "chopin-restart-signals-")); + directories.push(signals); + let paused = join(signals, "paused"); + let resume = join(signals, "resume"); + let local = await connector({ CHOPIN_TEST_PAUSED: paused, CHOPIN_TEST_RESUME: resume }); + + await authenticate(page, `ana-${crypto.randomUUID().slice(0, 8)}`, ORIGIN); + await page.goto(await local.pairing); + await page.getByRole("button", { name: "Add machine", exact: true }).click(); + await expect(page.getByRole("status").filter({ hasText: "Connected" })).toHaveText("Connected"); + await page.goto(`${ORIGIN}${testChannelPath(room)}`); + await page.getByRole("button", { name: "Build plan", exact: true }).click(); + + // The agent has started its first task when Chopin is deployed again. + await expect.poll(() => exists(paused), { timeout: 30_000 }).toBe(true); + let snapshot = + async () => (await (await page.request.get(`${ORIGIN}/api/channels/${room}/implementation`)) + .json()); + expect((await snapshot()).build.state).toBe("running"); + await stop(chopin.child); + // Longer than a heartbeat apart, so the connector sees Chopin gone. + await new Promise(resolve => setTimeout(resolve, 12_000)); + chopin = await server(database); + // The agent goes on at once; its calls are retried until its machine is back. + await writeFile(resume, ""); + + await expect.poll(async () => (await snapshot()).build?.state, { timeout: 90_000 }) + .toMatch(/^(stopped|failed)$/); + let built = await snapshot(); + expect(built.build.state, `${local.output()}\n${chopin.log.text}`).toBe("stopped"); + expect(built.build.error).toBeUndefined(); + expect(chopin.log.text).toContain("after Chopin restarted"); + expect(local.output()).not.toContain("ended this run"); + + await page.reload(); + await expect(page.getByRole("button", { name: /^Built/ })).toBeVisible(); + await page.getByRole("group", { name: "Document view" }) + .getByRole("button", { name: /^Build/ }).click(); + let view = page.getByRole("region", { name: "Build", exact: true }); + await expect(view.getByText("Built", { exact: true })).toBeVisible(); + await expect(view.getByText("2 pull requests", { exact: true })).toBeVisible(); +}); diff --git a/packages/protocol/implementation.d.ts b/packages/protocol/implementation.d.ts index 846d9d1b5..d3049f866 100644 --- a/packages/protocol/implementation.d.ts +++ b/packages/protocol/implementation.d.ts @@ -31,6 +31,11 @@ export type BuildRequest = { stoppedAtRevision?: number; /** The connector's key for the report that set `state`, so a retry of it is recognized. */ reportKey?: string; + /** + * Claimed with a run credential Chopin can check again after it restarts, by a connector that + * names the document as it reports, so its machine can carry on with it after a restart. + */ + resumable?: true; /** A rebuild carries edits since `baseRevision` onto a live build's pull requests. */ kind?: "rebuild"; baseRevision?: number;