From 37f76043c6d02a9b4ee1f87ff6c2cb997c83bec9 Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Fri, 23 Oct 2026 08:54:30 -0400 Subject: [PATCH 1/2] fix(engine): subagents survive shared-quota 429s and a dead child reports itself resumable (#1849) Two compounding bugs killed subagents on Baseten 429s (retry-after: 1): - retry.ts: the header was the whole wait and RETRY_MAX_RETRIES=5 applied, so a rate limit got 5 x 1s retries (~6s) and the turn died. Rate-limited errors (429 / rate-limit message, excluding the free/Go usage-limit upsells) now back off exponentially with jitter, capped at 60s/attempt, with the provider hint as a floor, for a 10-minute budget. Other errors keep the 5-attempt cap. The header parse is factored into hint(). - task.ts: runTask returned the last text part (or "") without checking the child message for an error, so the parent saw an empty completed result and started a fresh subagent. An errored child (other than a user abort) now fails the task with the error, the preserved session id, an explicit task_id resume instruction, and the child's last output. --- packages/app-bundle/manifest.json | 14 +- .../packages/opencode/src/session/retry.ts | 235 +++++++ .../packages/opencode/src/tool/task.ts | 23 +- .../opencode/test/session/retry.test.ts | 581 ++++++++++++++++++ .../packages/opencode/test/tool/task.test.ts | 56 +- 5 files changed, 902 insertions(+), 7 deletions(-) create mode 100644 packages/app-bundle/overlay/packages/opencode/src/session/retry.ts create mode 100644 packages/app-bundle/overlay/packages/opencode/test/session/retry.test.ts diff --git a/packages/app-bundle/manifest.json b/packages/app-bundle/manifest.json index fccfd657a..b1bd470f2 100644 --- a/packages/app-bundle/manifest.json +++ b/packages/app-bundle/manifest.json @@ -374,6 +374,7 @@ "packages/opencode/src/session/receipt-privacy.ts": "92ba8a92d4ddc0d18249efa5efdf600a32ebc0453d605b8c747cd4bcbd6d19a9", "packages/opencode/src/session/receipt.ts": "06fd30326424112b8dacd756e4bba5424b3a7a2d10318464a5670aeabb0c4808", "packages/opencode/src/session/reminders.ts": "abcd602de7f8ab8ab1a0364d30d28f1eb04fe81bf7cd837db9e926ffecf2dc6c", + "packages/opencode/src/session/retry.ts": "36fc688497d53226b97a7157328ae3ad477064e37d4a3e7c69d25e8df75bc303", "packages/opencode/src/session/rollout.ts": "55518da37d6d4d1298eeab1c7ee208f59b80b7246f3370438de98ad774e4b02e", "packages/opencode/src/session/run-state.ts": "b6d6270424fc6f1837075ab25847b58a42a774cb0433d8fbc0456c1d71d4cfa2", "packages/opencode/src/session/session.ts": "d3d8b9e7603aa9e40113a361f5bdbd7a48924250274071f5c07e0c13e1e62d4d", @@ -389,7 +390,7 @@ "packages/opencode/src/tool/registry.ts": "0097ab8a7904559d0354d165245caff6bf0e887c99ce267be77fa4ee4b66ec3c", "packages/opencode/src/tool/shell-output-throttle.ts": "00933a5c28bf4f3026541b1ca226678778177146b5b0b56bb5f189d837be9067", "packages/opencode/src/tool/shell.ts": "09808bb3865574853d162bfd7e1492a618c230828b95ec3989e9e0afa4acb97c", - "packages/opencode/src/tool/task.ts": "ddeb07a1cf009dcafc1d1d1c86b9d507676a8991db33a4632c8f29ae0e70e654", + "packages/opencode/src/tool/task.ts": "96a41e9ad58a77dca4df1445f562943a0b3a1fdfc1441bb51d66f9117a1afc94", "packages/opencode/src/tool/truncate.ts": "92c59d83c4a660116d6ecc48e900aeb16bde93962bfae9f520193fda18273379", "packages/opencode/src/tool/widget-author.ts": "912be8636b162adca8796a501e927019c2449393c14bb5f3ff478b02083b73a9", "packages/opencode/src/tool/widget-author.txt": "a2572913debbbbd2d4a644dcdf4576accbc7efdcc8217c0fc9f5a011e7a15ec4", @@ -460,6 +461,7 @@ "packages/opencode/test/session/prompt.test.ts": "e3d0748edb833640ef284857ae725e4b974843480d4128f56239e3fd1e6d481f", "packages/opencode/test/session/prose-guard.test.ts": "7239a758cbf27df4c3879fa9ee0374964559c1461773c2f5a85196f64632e6c4", "packages/opencode/test/session/receipt-privacy.test.ts": "6c8777edc280a061fec1a4d16e2504aec18bdf9cd6dd2cedc7a09ef8fc6c2ca0", + "packages/opencode/test/session/retry.test.ts": "8136de9fed8d493c9703098b5f701674681ecf8a3f823268c039b819753d00e2", "packages/opencode/test/session/rollout.test.ts": "3153fad45639db647d98d7fd96b3b5c2f0d938fca3a40598a4f88b8f62dda397", "packages/opencode/test/session/run-state-store.test.ts": "412ad2f9829bc60a3552f3e116783ab5b40d327947cd7ae31235486954be8434", "packages/opencode/test/session/session.test.ts": "c182f3e5311324175c9ba44a7164737a75ee6d2f22e4a024d66518b5ae91f5f2", @@ -472,7 +474,7 @@ "packages/opencode/test/tool/question.test.ts": "75760ba6ed98f2fdfd75945c3b492342dfe91026f99cde86db19cf52444260c2", "packages/opencode/test/tool/registry-schema.test.ts": "053cb9732299ea6554db007bad873d8d61cd3769aa9514d66c70c3bf0d405b90", "packages/opencode/test/tool/shell-output-throttle.test.ts": "9fed6f4ea8f7c62941280ab4f973848ea4184274830a153fbea83aae2c028576", - "packages/opencode/test/tool/task.test.ts": "fb0276cc33add808c669a4f43649d189d8f62c75974f3092a5f05e45cf176486", + "packages/opencode/test/tool/task.test.ts": "db9e368fadf22ab0dd216b096834f97176a56126ee5d113f1ba2e772a72fce09", "packages/opencode/test/tool/truncation.test.ts": "582ef7199b3f294b1c1588fadda8a1c0ea43ca9af0424a97f48747ade1654ad1", "packages/opencode/test/tool/write.test.ts": "820d36fde2314abbd939ce43ca5bcc4cfa0139d12a4c669716249060d9a955ff", "packages/schema/src/v1/question.ts": "3ba37742dbd5ef322b7edb42fea0dacf43a1322747dfe997af909bf564bffd35", @@ -1011,7 +1013,7 @@ "packages/ui/src/v2/components/text-shimmer-v2.tsx" ], "exceptions": [], - "extracted_at": "2026-10-23T12:21:18.499Z", + "extracted_at": "2026-10-23T13:24:06.506Z", "per_package": { "packages/app": { "A": 254, @@ -1030,7 +1032,7 @@ }, "packages/opencode": { "A": 98, - "M": 98, + "M": 100, "D": 0 }, "packages/schema": { @@ -1060,7 +1062,7 @@ } }, "counts": { - "overlay_total": 989, + "overlay_total": 991, "deletions": 7, "server_coupled": 0 }, @@ -1431,6 +1433,7 @@ "packages/opencode/src/session/receipt-privacy.ts": "A", "packages/opencode/src/session/receipt.ts": "A", "packages/opencode/src/session/reminders.ts": "M", + "packages/opencode/src/session/retry.ts": "M", "packages/opencode/src/session/rollout.ts": "A", "packages/opencode/src/session/run-state.ts": "M", "packages/opencode/src/session/session.ts": "M", @@ -1517,6 +1520,7 @@ "packages/opencode/test/session/prompt.test.ts": "M", "packages/opencode/test/session/prose-guard.test.ts": "A", "packages/opencode/test/session/receipt-privacy.test.ts": "A", + "packages/opencode/test/session/retry.test.ts": "M", "packages/opencode/test/session/rollout.test.ts": "A", "packages/opencode/test/session/run-state-store.test.ts": "A", "packages/opencode/test/session/session.test.ts": "M", diff --git a/packages/app-bundle/overlay/packages/opencode/src/session/retry.ts b/packages/app-bundle/overlay/packages/opencode/src/session/retry.ts new file mode 100644 index 000000000..a74f8ad57 --- /dev/null +++ b/packages/app-bundle/overlay/packages/opencode/src/session/retry.ts @@ -0,0 +1,235 @@ +import type { NamedError } from "@opencode-ai/core/util/error" +import { SessionV1 } from "@opencode-ai/core/v1/session" +import { Cause, Clock, Duration, Effect, Schedule } from "effect" +import { MessageV2 } from "./message-v2" +import { iife } from "@/util/iife" +import { isRecord } from "@/util/record" + +export type Err = ReturnType + +export const GO_UPSELL_MESSAGE = "Free usage exceeded, subscribe to Go" +export const GO_UPSELL_URL = "https://opencode.ai/go" +export type RetryReason = "free_tier_limit" | "account_rate_limit" | (string & {}) + +export type Retryable = { + message: string + action?: { + reason: RetryReason + provider: string + title: string + message: string + label: string + link?: string + } +} + +export const RETRY_INITIAL_DELAY = 2000 +export const RETRY_BACKOFF_FACTOR = 2 +export const RETRY_JITTER_FACTOR = 0.25 +export const RETRY_MAX_DELAY_NO_HEADERS = 30_000 // 30 seconds +export const RETRY_MAX_DELAY = 2_147_483_647 // max 32-bit signed integer for setTimeout +export const RETRY_MAX_RETRIES = 5 +// #1849: rate limits are a shared-quota condition, not a broken request. A +// provider answering `429, retry-after: 1` to every session at once used to +// get five 1-second retries (~6s) and then killed the turn — and with it any +// subagent. Rate-limited turns instead back off exponentially (the header is +// a floor, never the whole wait) and keep retrying for a time budget. +export const RETRY_RATE_LIMIT_BUDGET = 10 * 60_000 // 10 minutes +export const RETRY_RATE_LIMIT_MAX_DELAY = 60_000 // 1 minute between attempts + +const RATE_LIMIT_PATTERN = /rate increased too quickly|rate limit|rate-limit|rate_limit|too many requests|too_many_requests/i + +const RETRYABLE_MESSAGE_PATTERNS = [ + /429|500|502|503|504|524/i, + /rate increased too quickly|rate limit|rate-limit|rate_limit|too many requests/i, + /overloaded|service unavailable|service_unavailable|service-unavailable|internal error|internal_error|internal server error|server error|server_error|server-error|provider returned error|provider_returned_error|provider-returned-error/i, + /terminated|fetch failed|failed to fetch|network[-_\s]error|upstream connect|connection error|connection refused|connection lost|socket connection was closed|socket hang up|reset before headers|getaddrinfo|enotfound|eai_again|econnrefused|econnreset|etimedout/i, + /^timeout$|\b(?:request|response|connection|network|stream|read) (?:timeout|timed out|time out)\b/i, + /try your request again|retry your request|resource exhausted|resource_exhausted/i, + /\btry again (?:later|in\b)|\b(?:currently|temporarily) at capacity\b/i, +] + +function cap(ms: number) { + return Math.min(ms, RETRY_MAX_DELAY) +} + +/** The provider's retry hint in ms (retry-after-ms, retry-after seconds or + * HTTP date), or undefined when absent/unparseable/past. */ +export function hint(error?: SessionV1.APIError) { + const headers = error?.data.responseHeaders + if (!headers) return undefined + const retryAfterMs = headers["retry-after-ms"] + if (retryAfterMs) { + const parsedMs = Number.parseFloat(retryAfterMs) + if (!Number.isNaN(parsedMs)) return cap(parsedMs) + } + const retryAfter = headers["retry-after"] + if (retryAfter) { + const parsedSeconds = Number.parseFloat(retryAfter) + if (!Number.isNaN(parsedSeconds)) return cap(Math.ceil(parsedSeconds * 1000)) + const parsed = Date.parse(retryAfter) - Date.now() + if (!Number.isNaN(parsed) && parsed > 0) return cap(Math.ceil(parsed)) + } + return undefined +} + +export function delay(attempt: number, error?: SessionV1.APIError, random = Math.random()) { + const hinted = hint(error) + if (hinted !== undefined) return hinted + if (error?.data.responseHeaders) return cap(exponential(attempt, random)) + return cap(Math.min(exponential(attempt, random), RETRY_MAX_DELAY_NO_HEADERS)) +} + +function exponential(attempt: number, random: number) { + const base = RETRY_INITIAL_DELAY * Math.pow(RETRY_BACKOFF_FACTOR, attempt - 1) + return Math.ceil(base + base * RETRY_JITTER_FACTOR * random) +} + +/** True for rate-limit failures (HTTP 429 or a rate-limit message/body). */ +export function isRateLimit(error: Err) { + if (SessionV1.APIError.isInstance(error)) { + if (error.data.statusCode === 429) return true + return RATE_LIMIT_PATTERN.test(error.data.message) || RATE_LIMIT_PATTERN.test(error.data.responseBody ?? "") + } + const message = isRecord(error.data) ? error.data.message : undefined + return typeof message === "string" && RATE_LIMIT_PATTERN.test(message) +} + +/** The wait before a rate-limited retry: exponential with jitter (capped at + * RETRY_RATE_LIMIT_MAX_DELAY), never shorter than the provider's hint. */ +export function rateLimitDelay(attempt: number, error?: SessionV1.APIError, random = Math.random()) { + const backoff = Math.min(exponential(attempt, random), RETRY_RATE_LIMIT_MAX_DELAY) + return cap(Math.max(backoff, hint(error) ?? 0)) +} + +export function retryable(error: Err, provider: string) { + // context overflow errors should not be retried + if (SessionV1.ContextOverflowError.isInstance(error)) return undefined + if (SessionV1.APIError.isInstance(error)) { + const status = error.data.statusCode + // 5xx errors are transient server failures and should always be retried, + // even when the provider SDK doesn't explicitly mark them as retryable. + if ( + !error.data.isRetryable && + !(status !== undefined && status >= 500) && + !matchesRetryableMessage(error.data.message) && + !matchesRetryableMessage(error.data.responseBody) + ) + return undefined + if (error.data.responseBody?.includes("FreeUsageLimitError")) { + return { + message: GO_UPSELL_MESSAGE, + action: { + reason: "free_tier_limit", + provider, + title: "Free limit reached", + message: "Subscribe to OpenCode Go for reliable access to the best open-source models for $10/month.", + label: "subscribe", + link: GO_UPSELL_URL, + }, + } + } + if (error.data.responseBody?.includes("GoUsageLimitError")) { + const body = parseJSON(error.data.responseBody) + const workspace = str(body?.metadata?.workspace) + const limitName = str(body?.metadata?.limitName) + const retryAfter = num(error.data.responseHeaders?.["retry-after"]) + const resetIn = iife(() => { + if (retryAfter === undefined) return "" + const seconds = Math.max(0, Math.ceil(retryAfter)) + const days = Math.floor(seconds / 86_400) + const hours = Math.floor((seconds % 86_400) / 3_600) + const minutes = Math.ceil((seconds % 3_600) / 60) + const unit = (value: number, name: string) => `${value} ${name}${value === 1 ? "" : "s"}` + + if (days > 0) return hours > 0 ? `${unit(days, "day")} ${unit(hours, "hour")}` : unit(days, "day") + if (hours > 0) return minutes > 0 ? `${unit(hours, "hour")} ${unit(minutes, "minute")}` : unit(hours, "hour") + return minutes > 0 ? unit(minutes, "minute") : "less than a minute" + }) + + const message = `${limitName ? `${limitName} usage limit` : "Usage limit"} reached. It will reset in ${resetIn}. To continue using this model now, enable usage from your available balance` + + const link = `https://opencode.ai/workspace/${workspace}/go` + return { + message: `${message} - ${link}`, + action: { + reason: "account_rate_limit", + provider, + title: "Go limit reached", + message, + label: "open settings", + link, + }, + } + } + return { message: error.data.message.includes("Overloaded") ? "Provider is overloaded" : error.data.message } + } + + const message = isRecord(error.data) ? error.data.message : undefined + if (typeof message !== "string") return undefined + const lower = message.toLowerCase() + if (lower.includes("too_many_requests")) return { message: "Too Many Requests" } + if (lower.includes("exhausted") || lower.includes("unavailable")) return { message: "Provider is overloaded" } + if (matchesRetryableMessage(message)) return { message } + return undefined +} + +function matchesRetryableMessage(value: unknown) { + return typeof value === "string" && RETRYABLE_MESSAGE_PATTERNS.some((pattern) => pattern.test(value)) +} + +function str(value: unknown) { + if (value === undefined || value === null) return "" + return String(value) +} + +function num(value: unknown) { + const parsed = Number.parseFloat(str(value)) + if (Number.isNaN(parsed)) return undefined + return parsed +} + +function parseJSON(value: unknown) { + return iife(() => { + try { + if (typeof value !== "string") return undefined + return JSON.parse(value) + } catch { + return undefined + } + }) +} + +export function policy(opts: { + provider: string + parse: (error: unknown) => Err + set: (input: { attempt: number; message: string; action?: Retryable["action"]; next: number }) => Effect.Effect +}) { + return Schedule.fromStepWithMetadata( + Effect.succeed((meta: Schedule.InputMetadata) => { + const error = opts.parse(meta.input) + const retry = retryable(error, opts.provider) + if (!retry) return Cause.done(meta.attempt) + const api = SessionV1.APIError.isInstance(error) ? error : undefined + // usage-limit upsells (free tier / Go) reset on hours-scale clocks; they + // keep the attempt cap. Plain rate limits get the time budget. + const rateLimited = !retry.action && isRateLimit(error) + const wait = rateLimited ? rateLimitDelay(meta.attempt, api) : delay(meta.attempt, api) + if (rateLimited) { + if (meta.elapsed >= RETRY_RATE_LIMIT_BUDGET) return Cause.done(meta.attempt) + } else if (meta.attempt > RETRY_MAX_RETRIES) return Cause.done(meta.attempt) + return Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis + yield* opts.set({ + attempt: meta.attempt, + message: retry.message, + action: retry.action, + next: now + wait, + }) + return [meta.attempt, Duration.millis(wait)] as [number, Duration.Duration] + }) + }), + ) +} + +export * as SessionRetry from "./retry" diff --git a/packages/app-bundle/overlay/packages/opencode/src/tool/task.ts b/packages/app-bundle/overlay/packages/opencode/src/tool/task.ts index f6dbf29ab..6f17c9009 100644 --- a/packages/app-bundle/overlay/packages/opencode/src/tool/task.ts +++ b/packages/app-bundle/overlay/packages/opencode/src/tool/task.ts @@ -78,6 +78,17 @@ function renderOutput(input: { ].join("\n") } +export function subagentFailure(sessionID: SessionID, error: { name: string; data?: unknown }, partial: string) { + const data = error.data as { message?: unknown } | undefined + const detail = typeof data?.message === "string" && data.message ? `${error.name}: ${data.message}` : error.name + return [ + `The subagent stopped before finishing (${detail}).`, + `Its work so far is preserved in session ${sessionID}.`, + `To continue it, call the task tool again with task_id="${sessionID}" and a prompt such as "continue where you left off" — do not start a new task for the same work.`, + ...(partial ? ["", "Last output before it stopped:", partial] : []), + ].join("\n") +} + export const TaskTool = Tool.define( id, Effect.gen(function* () { @@ -211,7 +222,17 @@ export const TaskTool = Tool.define( agent: next.name, parts, }) - return result.parts.findLast((item) => item.type === "text")?.text ?? "" + const text = result.parts.findLast((item) => item.type === "text")?.text ?? "" + // #1849: a child whose final turn errored (a provider 429 that outlived + // its retries, an auth failure, ...) used to come back as an EMPTY + // success — the parent saw "completed" with no result and started a + // fresh subagent, discarding all the child's work. Report it as a + // failure that names the resumable session instead. + const error = result.info.role === "assistant" ? result.info.error : undefined + if (error && error.name !== "MessageAbortedError") { + return yield* Effect.fail(new Error(subagentFailure(nextSession.id, error, text))) + } + return text }) const inject = Effect.fn("TaskTool.injectBackgroundResult")(function* ( diff --git a/packages/app-bundle/overlay/packages/opencode/test/session/retry.test.ts b/packages/app-bundle/overlay/packages/opencode/test/session/retry.test.ts new file mode 100644 index 000000000..eab31be6f --- /dev/null +++ b/packages/app-bundle/overlay/packages/opencode/test/session/retry.test.ts @@ -0,0 +1,581 @@ +import { describe, expect, test } from "bun:test" +import { LayerNode } from "@opencode-ai/core/effect/layer-node" +import { SessionV1 } from "@opencode-ai/core/v1/session" +import type { NamedError } from "@opencode-ai/core/util/error" +import { APICallError } from "ai" +import { setTimeout as sleep } from "node:timers/promises" +import { Effect, Schedule, Schema } from "effect" +import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner" +import { SessionRetry } from "../../src/session/retry" +import { MessageV2 } from "../../src/session/message-v2" +import { ProviderError } from "../../src/provider/error" +import { SessionID } from "../../src/session/schema" +import { SessionStatus } from "../../src/session/status" +import { testEffect } from "../lib/effect" +import { ProviderV2 } from "@opencode-ai/core/provider" + +const providerID = ProviderV2.ID.make("test") +const retryProvider = "test" +const it = testEffect(LayerNode.compile(LayerNode.group([SessionStatus.node, CrossSpawnSpawner.node]))) + +function apiError(headers?: Record): SessionV1.APIError { + return Schema.decodeUnknownSync(SessionV1.APIError.Schema)( + new SessionV1.APIError({ + message: "boom", + isRetryable: true, + responseHeaders: headers, + }).toObject(), + ) +} + +function wrap(message: unknown): ReturnType { + return { name: "", data: { message } } +} + +describe("session.retry.delay", () => { + test("caps delay at 30 seconds when headers missing", () => { + const error = apiError() + const delays = Array.from({ length: 10 }, (_, index) => SessionRetry.delay(index + 1, error, 0)) + expect(delays).toStrictEqual([2000, 4000, 8000, 16000, 30000, 30000, 30000, 30000, 30000, 30000]) + }) + + test("adds jitter to exponential delays", () => { + const error = apiError() + expect(SessionRetry.delay(1, error, 0)).toBe(2000) + expect(SessionRetry.delay(1, error, 1)).toBe(2500) + expect(SessionRetry.delay(4, error, 1)).toBe(20000) + expect(SessionRetry.delay(5, error, 1)).toBe(30000) + }) + + test("prefers retry-after-ms when shorter than exponential", () => { + const error = apiError({ "retry-after-ms": "1500" }) + expect(SessionRetry.delay(4, error)).toBe(1500) + }) + + test("uses retry-after seconds when reasonable", () => { + const error = apiError({ "retry-after": "30" }) + expect(SessionRetry.delay(3, error)).toBe(30000) + }) + + test("accepts http-date retry-after values", () => { + const date = new Date(Date.now() + 20000).toUTCString() + const error = apiError({ "retry-after": date }) + const d = SessionRetry.delay(1, error) + expect(d).toBeGreaterThanOrEqual(19000) + expect(d).toBeLessThanOrEqual(20000) + }) + + test("ignores invalid retry hints", () => { + const error = apiError({ "retry-after": "not-a-number" }) + expect(SessionRetry.delay(1, error, 0)).toBe(2000) + }) + + test("ignores malformed date retry hints", () => { + const error = apiError({ "retry-after": "Invalid Date String" }) + expect(SessionRetry.delay(1, error, 0)).toBe(2000) + }) + + test("ignores past date retry hints", () => { + const pastDate = new Date(Date.now() - 5000).toUTCString() + const error = apiError({ "retry-after": pastDate }) + expect(SessionRetry.delay(1, error, 0)).toBe(2000) + }) + + test("uses retry-after values even when exceeding 10 minutes with headers", () => { + const error = apiError({ "retry-after": "50" }) + expect(SessionRetry.delay(1, error)).toBe(50000) + + const longError = apiError({ "retry-after-ms": "700000" }) + expect(SessionRetry.delay(1, longError)).toBe(700000) + }) + + test("caps oversized header delays to the runtime timer limit", () => { + const error = apiError({ "retry-after-ms": "999999999999" }) + expect(SessionRetry.delay(1, error)).toBe(SessionRetry.RETRY_MAX_DELAY) + }) + + it.instance("policy updates retry status and increments attempts", () => + Effect.gen(function* () { + const sessionID = SessionID.make("session-retry-test") + const error = apiError({ "retry-after-ms": "0" }) + const status = yield* SessionStatus.Service + + const step = yield* Schedule.toStepWithMetadata( + SessionRetry.policy({ + provider: "test", + parse: Schema.decodeUnknownSync(SessionV1.APIError.Schema), + set: (info) => + status.set(sessionID, { + type: "retry", + attempt: info.attempt, + message: info.message, + next: info.next, + }), + }), + ) + yield* step(error) + yield* step(error) + + expect(yield* status.get(sessionID)).toMatchObject({ + type: "retry", + attempt: 2, + message: "boom", + }) + }), + ) + + it.instance("policy stops after five retries", () => + Effect.gen(function* () { + const attempts: number[] = [] + const error = apiError({ "retry-after-ms": "0" }) + const step = yield* Schedule.toStepWithMetadata( + SessionRetry.policy({ + provider: "test", + parse: Schema.decodeUnknownSync(SessionV1.APIError.Schema), + set: (info) => + Effect.sync(() => { + attempts.push(info.attempt) + }), + }), + ) + + yield* Effect.forEach(Array.from({ length: SessionRetry.RETRY_MAX_RETRIES + 1 }), () => + Effect.ignore(step(error)), + ) + + expect(attempts).toStrictEqual([1, 2, 3, 4, 5]) + }), + ) +}) + +// #1849: a shared-quota 429 with `retry-after: 1` (Baseten's shape) must back +// off and keep going, not die after five 1-second retries. +describe("session.retry rate limits", () => { + function rateLimited(headers: Record = { "retry-after": "1" }): SessionV1.APIError { + return Schema.decodeUnknownSync(SessionV1.APIError.Schema)( + new SessionV1.APIError({ + message: "Too Many Requests: Too Many Requests", + statusCode: 429, + isRetryable: true, + responseHeaders: headers, + }).toObject(), + ) + } + + test("isRateLimit recognizes 429s and rate-limit messages, not other errors", () => { + expect(SessionRetry.isRateLimit(rateLimited())).toBe(true) + expect(SessionRetry.isRateLimit(wrap("Rate limit exceeded"))).toBe(true) + expect(SessionRetry.isRateLimit(apiError())).toBe(false) + }) + + test("the retry-after hint is a floor under exponential backoff, capped per attempt", () => { + const error = rateLimited() + const delays = Array.from({ length: 8 }, (_, index) => SessionRetry.rateLimitDelay(index + 1, error, 0)) + expect(delays).toStrictEqual([2000, 4000, 8000, 16000, 32000, 60000, 60000, 60000]) + }) + + test("a hint longer than the backoff wins", () => { + expect(SessionRetry.rateLimitDelay(1, rateLimited({ "retry-after": "90" }), 0)).toBe(90000) + }) + + it.effect("policy keeps retrying a rate limit past the attempt cap, until the time budget", () => + Effect.gen(function* () { + const attempts: number[] = [] + const error = rateLimited() + // explicit timestamps: the schedule's elapsed clock is the `now` we pass + const step = yield* Schedule.toStep( + SessionRetry.policy({ + provider: "test", + parse: Schema.decodeUnknownSync(SessionV1.APIError.Schema), + set: (info) => + Effect.sync(() => { + attempts.push(info.attempt) + }), + }), + ) + + // well past the generic five-attempt cap, all inside the budget + const start = 1_000_000 + const n = SessionRetry.RETRY_MAX_RETRIES * 2 + for (let i = 0; i < n; i++) yield* step(start + i * 10_000, error) + expect(attempts).toHaveLength(n) + + // once the budget is spent the schedule ends and the error surfaces + const exit = yield* Effect.exit(step(start + SessionRetry.RETRY_RATE_LIMIT_BUDGET, error)) + expect(exit._tag).toBe("Failure") + expect(attempts).toHaveLength(n) + }), + ) +}) + +describe("session.retry.retryable", () => { + test("retries serialized too_many_requests messages", () => { + const error = wrap(JSON.stringify({ type: "error", error: { type: "too_many_requests" } })) + expect(SessionRetry.retryable(error, retryProvider)).toEqual({ message: "Too Many Requests" }) + }) + + test("retries serialized overloaded provider codes", () => { + const error = wrap(JSON.stringify({ code: "resource_exhausted" })) + expect(SessionRetry.retryable(error, retryProvider)).toEqual({ message: "Provider is overloaded" }) + }) + + test("retries serialized rate_limit messages", () => { + const message = JSON.stringify({ type: "error", error: { code: "rate_limit_exceeded" } }) + expect(SessionRetry.retryable(wrap(message), retryProvider)).toEqual({ message }) + }) + + test("does not retry unknown json messages", () => { + const error = wrap(JSON.stringify({ error: { message: "no_kv_space" } })) + expect(SessionRetry.retryable(error, retryProvider)).toBeUndefined() + }) + + test("does not throw on numeric error codes", () => { + const error = wrap(JSON.stringify({ type: "error", error: { code: 123 } })) + const result = SessionRetry.retryable(error, retryProvider) + expect(result).toBeUndefined() + }) + + test("returns undefined for non-json message", () => { + const error = wrap("not-json") + expect(SessionRetry.retryable(error, retryProvider)).toBeUndefined() + }) + + test("retries plain text rate limit errors from Alibaba", () => { + const msg = + "Upstream error from Alibaba: Request rate increased too quickly. To ensure system stability, please adjust your client logic to scale requests more smoothly over time." + const error = wrap(msg) + expect(SessionRetry.retryable(error, retryProvider)).toEqual({ message: msg }) + }) + + test("retries plain text rate limit errors", () => { + const msg = "Rate limit exceeded, please try again later" + const error = wrap(msg) + expect(SessionRetry.retryable(error, retryProvider)).toEqual({ message: msg }) + }) + + test("retries too many requests in plain text", () => { + const msg = "Too many requests, please slow down" + const error = wrap(msg) + expect(SessionRetry.retryable(error, retryProvider)).toEqual({ message: msg }) + }) + + test.each([ + "Internal server error", + "internal error", + "server-error", + "Provider returned error", + "provider-returned-error", + "terminated", + "fetch failed", + "network error", + "network-error", + "network_error", + "connection refused", + "connect ECONNREFUSED", + "request ETIMEDOUT", + "failed to fetch", + "EAI_AGAIN", + "response timed out", + "Please retry your request", + "try your request again", + "Please try again in a few minutes", + "The model is currently at capacity due to high demand", + "The service is temporarily at capacity", + "upstream returned status 524", + ])("retries matching API error text: %s", (message) => { + expect(SessionRetry.retryable(wrap(message), retryProvider)).toEqual({ message }) + }) + + test("retries hyphenated service-unavailable errors", () => { + expect(SessionRetry.retryable(wrap("service-unavailable"), retryProvider)).toEqual({ + message: "Provider is overloaded", + }) + }) + + test("matches retryable API response bodies", () => { + const error = Schema.decodeUnknownSync(SessionV1.APIError.Schema)( + new SessionV1.APIError({ + message: "Request failed", + isRetryable: false, + statusCode: 400, + responseBody: JSON.stringify({ error: { message: "upstream connection refused" } }), + }).toObject(), + ) + expect(SessionRetry.retryable(error, retryProvider)).toEqual({ message: "Request failed" }) + }) + + test("retries transport timeout errors", () => { + const request = MessageV2.fromError(new ProviderError.HeaderTimeoutError(10000), { providerID }) + expect(SessionV1.APIError.isInstance(request)).toBe(true) + expect(SessionRetry.retryable(request, retryProvider)).toEqual({ + message: "Provider response headers timed out after 10000ms", + }) + }) + + test("retries websocket stream transport errors", () => { + const request = MessageV2.fromError( + new ProviderError.ResponseStreamError("WebSocket closed before response.completed (code 1006: Connection ended)"), + { providerID }, + ) + expect(SessionV1.APIError.isInstance(request)).toBe(true) + expect(SessionRetry.retryable(request, retryProvider)).toEqual({ + message: "WebSocket closed before response.completed (code 1006: Connection ended)", + }) + }) + + test("does not retry context overflow errors", () => { + const error = new SessionV1.ContextOverflowError({ + message: "Input exceeds context window of this model", + responseBody: '{"error":{"code":"context_length_exceeded"}}', + }).toObject() + + expect(SessionRetry.retryable(error, retryProvider)).toBeUndefined() + }) + + test("retries 500 errors even when isRetryable is false", () => { + const error = Schema.decodeUnknownSync(SessionV1.APIError.Schema)( + new SessionV1.APIError({ + message: "Internal server error", + isRetryable: false, + statusCode: 500, + responseBody: '{"type":"api_error","message":"Internal server error"}', + }).toObject(), + ) + + expect(SessionRetry.retryable(error, retryProvider)).toEqual({ message: "Internal server error" }) + }) + + test("retries 502 bad gateway errors", () => { + const error = Schema.decodeUnknownSync(SessionV1.APIError.Schema)( + new SessionV1.APIError({ + message: "Bad gateway", + isRetryable: false, + statusCode: 502, + }).toObject(), + ) + + expect(SessionRetry.retryable(error, retryProvider)).toEqual({ message: "Bad gateway" }) + }) + + test("retries 503 service unavailable errors", () => { + const error = Schema.decodeUnknownSync(SessionV1.APIError.Schema)( + new SessionV1.APIError({ + message: "Service unavailable", + isRetryable: false, + statusCode: 503, + }).toObject(), + ) + + expect(SessionRetry.retryable(error, retryProvider)).toEqual({ message: "Service unavailable" }) + }) + + test("does not retry 4xx errors when isRetryable is false", () => { + const error = Schema.decodeUnknownSync(SessionV1.APIError.Schema)( + new SessionV1.APIError({ + message: "Bad request", + isRetryable: false, + statusCode: 400, + }).toObject(), + ) + + expect(SessionRetry.retryable(error, retryProvider)).toBeUndefined() + }) + + test("retries ZlibError decompression failures", () => { + const error = Schema.decodeUnknownSync(SessionV1.APIError.Schema)( + new SessionV1.APIError({ + message: "Response decompression failed", + isRetryable: true, + metadata: { code: "ZlibError" }, + }).toObject(), + ) + + const retryable = SessionRetry.retryable(error, retryProvider) + expect(retryable).toBeDefined() + expect(retryable).toEqual({ message: "Response decompression failed" }) + }) + + test("maps free limits to Go upsell action", () => { + const error = Schema.decodeUnknownSync(SessionV1.APIError.Schema)( + new SessionV1.APIError({ + message: "Free usage exceeded", + isRetryable: true, + statusCode: 429, + responseBody: JSON.stringify({ + type: "error", + error: { type: "FreeUsageLimitError", message: "Free usage exceeded" }, + }), + }).toObject(), + ) + + expect(SessionRetry.retryable(error, "opencode")).toEqual({ + message: SessionRetry.GO_UPSELL_MESSAGE, + action: { + reason: "free_tier_limit", + provider: "opencode", + title: "Free limit reached", + message: "Subscribe to OpenCode Go for reliable access to the best open-source models for $10/month.", + label: "subscribe", + link: SessionRetry.GO_UPSELL_URL, + }, + }) + }) + + test("maps Go subscription limits to workspace PAYG upsell", () => { + const error = Schema.decodeUnknownSync(SessionV1.APIError.Schema)( + new SessionV1.APIError({ + message: "Subscription quota exceeded. You can continue using free models.", + isRetryable: true, + statusCode: 429, + responseHeaders: { + "retry-after": "19380", + }, + responseBody: JSON.stringify({ + type: "error", + error: { + type: "GoUsageLimitError", + message: "Subscription quota exceeded. You can continue using free models.", + }, + metadata: { + workspace: "wrk_01K6XGM22R6FM8JVABE9XDQXGH", + limitName: "5 hour", + }, + }), + }).toObject(), + ) + + expect(SessionRetry.retryable(error, "opencode-go")).toEqual({ + message: + "5 hour usage limit reached. It will reset in 5 hours 23 minutes. To continue using this model now, enable usage from your available balance - https://opencode.ai/workspace/wrk_01K6XGM22R6FM8JVABE9XDQXGH/go", + action: { + reason: "account_rate_limit", + provider: "opencode-go", + title: "Go limit reached", + message: + "5 hour usage limit reached. It will reset in 5 hours 23 minutes. To continue using this model now, enable usage from your available balance", + label: "open settings", + link: "https://opencode.ai/workspace/wrk_01K6XGM22R6FM8JVABE9XDQXGH/go", + }, + }) + }) + + test("maps Go subscription limits without limit metadata", () => { + const error = Schema.decodeUnknownSync(SessionV1.APIError.Schema)( + new SessionV1.APIError({ + message: "Subscription quota exceeded. You can continue using free models.", + isRetryable: true, + statusCode: 429, + responseHeaders: { + "retry-after": "900", + }, + responseBody: JSON.stringify({ + type: "error", + error: { + type: "GoUsageLimitError", + message: "Subscription quota exceeded. You can continue using free models.", + }, + metadata: { + workspace: "wrk_01K6XGM22R6FM8JVABE9XDQXGH", + }, + }), + }).toObject(), + ) + + expect(SessionRetry.retryable(error, "opencode-go")?.action?.message).toBe( + "Usage limit reached. It will reset in 15 minutes. To continue using this model now, enable usage from your available balance", + ) + }) +}) + +describe("session.message-v2.fromError", () => { + test.concurrent( + "converts ECONNRESET socket errors to retryable APIError", + async () => { + using server = Bun.serve({ + port: 0, + idleTimeout: 8, + async fetch(_req) { + return new Response( + new ReadableStream({ + async pull(controller) { + controller.enqueue("Hello,") + await sleep(10000) + controller.enqueue(" World!") + controller.close() + }, + }), + { headers: { "Content-Type": "text/plain" } }, + ) + }, + }) + + const error = await fetch(new URL("/", server.url.origin)) + .then((res) => res.text()) + .catch((e) => e) + + const result = MessageV2.fromError(error, { providerID }) + + expect(SessionV1.APIError.isInstance(result)).toBe(true) + if (!SessionV1.APIError.isInstance(result)) throw new Error("expected APIError") + expect(result.data.isRetryable).toBe(true) + expect(result.data.message).toBe("Connection reset by server") + expect(result.data.metadata?.code).toBe("ECONNRESET") + expect(result.data.metadata?.message).toInclude("socket connection") + }, + 15_000, + ) + + test("ECONNRESET socket error is retryable", () => { + const error = Schema.decodeUnknownSync(SessionV1.APIError.Schema)( + new SessionV1.APIError({ + message: "Connection reset by server", + isRetryable: true, + metadata: { code: "ECONNRESET", message: "The socket connection was closed unexpectedly" }, + }).toObject(), + ) + + const retryable = SessionRetry.retryable(error, retryProvider) + expect(retryable).toBeDefined() + expect(retryable).toEqual({ message: "Connection reset by server" }) + }) + + test("marks OpenAI 404 status codes as retryable", () => { + const error = new APICallError({ + message: "boom", + url: "https://api.openai.com/v1/chat/completions", + requestBodyValues: {}, + statusCode: 404, + responseHeaders: { "content-type": "application/json" }, + responseBody: '{"error":"boom"}', + isRetryable: false, + }) + const result = MessageV2.fromError(error, { providerID: ProviderV2.ID.make("openai") }) + if (!SessionV1.APIError.isInstance(result)) throw new Error("expected APIError") + expect(result.data.isRetryable).toBe(true) + }) + + test("converts OpenAI server_error stream chunks to retryable APIError", () => { + const result = MessageV2.fromError( + { + message: JSON.stringify({ + type: "error", + sequence_number: 2, + error: { + type: "server_error", + code: "server_error", + message: "An error occurred while processing your request.", + param: null, + }, + }), + }, + { providerID: ProviderV2.ID.make("openai") }, + ) + + expect(SessionV1.APIError.isInstance(result)).toBe(true) + if (!SessionV1.APIError.isInstance(result)) throw new Error("expected APIError") + expect(result.data.isRetryable).toBe(true) + expect(SessionRetry.retryable(result, retryProvider)).toEqual({ + message: "An error occurred while processing your request.", + }) + }) +}) diff --git a/packages/app-bundle/overlay/packages/opencode/test/tool/task.test.ts b/packages/app-bundle/overlay/packages/opencode/test/tool/task.test.ts index c879ec35e..8e90eaebd 100644 --- a/packages/app-bundle/overlay/packages/opencode/test/tool/task.test.ts +++ b/packages/app-bundle/overlay/packages/opencode/test/tool/task.test.ts @@ -3,7 +3,7 @@ import { SessionV1 } from "@opencode-ai/core/v1/session" import { Database } from "@opencode-ai/core/database/database" import { LayerNode } from "@opencode-ai/core/effect/layer-node" import { SessionProjector } from "@opencode-ai/core/session/projector" -import { Deferred, Effect, Exit, Fiber, Layer } from "effect" +import { Cause, Deferred, Effect, Exit, Fiber, Layer } from "effect" import { Agent } from "../../src/agent/agent" import { BackgroundJob } from "@/background/job" import { EventV2Bridge } from "@/event-v2-bridge" @@ -216,6 +216,60 @@ describe("tool.task", () => { }, ) + // #1849: the child's last turn died on a provider 429 — the parent must get + // a failure naming the resumable session, not an empty "completed". + it.instance("execute reports an errored child as a failure carrying its task_id", () => + Effect.gen(function* () { + const { chat, assistant } = yield* seed() + const tool = yield* TaskTool + const def = yield* tool.init() + const promptOps: TaskPromptOps = { + ...stubOps(), + prompt: (input) => + Effect.sync(() => { + const out = reply(input, "partial analysis") + return { + ...out, + info: { + ...out.info, + finish: undefined, + error: new SessionV1.APIError({ + message: "Too Many Requests: Too Many Requests", + statusCode: 429, + isRetryable: true, + }).toObject(), + } as SessionV1.WithParts["info"], + } + }), + } + + const exit = yield* Effect.exit( + def.execute( + { description: "analyze artifacts", prompt: "analyze", subagent_type: "general" }, + { + sessionID: chat.id, + messageID: assistant.id, + agent: "build", + abort: new AbortController().signal, + extra: { promptOps }, + messages: [], + metadata: () => Effect.void, + ask: () => Effect.void, + }, + ), + ) + + expect(Exit.isFailure(exit)).toBe(true) + const sessions = yield* Session.Service + const kids = yield* sessions.children(chat.id) + expect(kids).toHaveLength(1) + const text = Exit.isFailure(exit) ? String(Cause.squash(exit.cause)) : "" + expect(text).toContain("Too Many Requests") + expect(text).toContain(`task_id="${kids[0]!.id}"`) + expect(text).toContain("partial analysis") + }), + ) + it.instance("execute resumes an existing task session from task_id", () => Effect.gen(function* () { const sessions = yield* Session.Service From 2d36dccfd5f40fcab88deb64cec212a7c1eac536 Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Fri, 23 Oct 2026 09:16:08 -0400 Subject: [PATCH 2/2] chore(app-bundle): refresh the manifest hash for retry.test.ts (#1849) --- packages/app-bundle/manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/app-bundle/manifest.json b/packages/app-bundle/manifest.json index b1bd470f2..542167f12 100644 --- a/packages/app-bundle/manifest.json +++ b/packages/app-bundle/manifest.json @@ -1013,7 +1013,7 @@ "packages/ui/src/v2/components/text-shimmer-v2.tsx" ], "exceptions": [], - "extracted_at": "2026-10-23T13:24:06.506Z", + "extracted_at": "2026-10-23T13:24:11.640Z", "per_package": { "packages/app": { "A": 254,