From 671cbc03669ab6a3ab7af0b64c1f4b474c2420d8 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Sat, 24 Oct 2026 13:26:38 +0200 Subject: [PATCH 1/2] mason: hold sticky OC1 and Pi routes across quota windows Use the latest known future reset across every exhausted window blocking the requested model. Preserve retain decisions and deterministic jitter, and report the window responsible for each hold or migration. Generalize OC1 and Pi messages and add red-first per-window coverage. Parameterizing the existing OC1 test preserves its original five-hour hold, transient-failure retention, and far-reset migration assertions while adding weekly and model-scoped cases. OC2 remains unchanged pending the common-auth retry-delay contract. --- ARCHITECTURE.md | 4 +- AUDITOR.md | 2 +- CHANGELOG.md | 2 + docs/accounts-and-routing.md | 2 + docs/configuration.md | 2 +- packages/core/src/sticky-routing.ts | 68 ++-- .../src/tests/sticky-quota-failure.test.ts | 142 ++++++++ packages/opencode/src/cache-diagnostics.ts | 2 +- packages/opencode/src/index.ts | 2 +- packages/opencode/src/tests/index.test.ts | 335 ++++++++++-------- packages/pi/src/stream.ts | 4 +- packages/pi/src/tests/stream.test.ts | 183 ++++++++++ 12 files changed, 554 insertions(+), 194 deletions(-) create mode 100644 packages/core/src/tests/sticky-quota-failure.test.ts diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 7e24de50c..84e3a6b59 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -49,7 +49,7 @@ 4. **Request interception** — OpenCode's messages hook observes messages to mark effort transitions and assign user-turn prompt UUIDs, `chat.headers` injects internal correlation headers (`x-cortexkit-effort-plan`, `x-cortexkit-billing-lineage`), and OpenCode's fetch wrapper intercepts Anthropic fetch calls, resolving and stripping internal headers before dispatch — `packages/opencode/src/index.ts`, `packages/opencode/src/billing-lineage.ts` (experimental fetch wrapping) 5. **URL rewrite** — `rewriteUrl()` adds `?beta=true` to `/v1/messages`, normalizes versioned base paths without duplicating version segments, and overrides base URL when `ANTHROPIC_BASE_URL` or a proxy route `baseURL` is set — `packages/opencode/src/transform.ts` 6. **Request body rewrite** — `rewriteRequestBody()` removes only provably empty trailing assistant messages, refuses meaningful or unknown assistant tails before cache/billing/signing, and strips trailing whitespace after tool_use, normalizes Fable/Mythos 5/5.1, Sonnet 5/5.5, Haiku 5.5, and Opus 5/5.5 adaptive thinking (mapping Sonnet 5.5 opt-outs to `between_tools` and removing its unsupported forced tool choice, enforcing always-on adaptive thinking on Opus 5.5 and normalizing `[1m]` context-window qualifiers via `normalizeAnthropicModelId`), injects the Claude Code 2.1.302 billing header with a bounded process-local suffix pin derived from the initial visible user text and optional request lineage (`cc_prev_req` and `cc_prompt_id`), sanitizes the system prompt (removes OpenCode identity), prepends Claude Code identity, applies cache strategy (explicit/automatic/hybrid), adds fast mode only to Opus 4.8/5/5.5, remaps model names on proxy routes using `ANTHROPIC_DEFAULT_*_MODEL` tier-specific or `ANTHROPIC_MODEL` generic overrides, prefixes tool names and named-tool choices with `mcp_`, removes forced `tool_choice` on Opus 5.5 and Fable/Mythos 5.1 (OpenCode still rejects missing structured output), opts eligible OAuth Fable 5/5.1, Opus 5, and Opus 5.5 requests into Anthropic server-side safety fallback, restores stored fallback boundary markers, conditionally applies the configured Fable 5.1 thinking-prefix behavior only when signed or redacted thinking remains in the outgoing history, keeps top-level Fable 5.1 effort anchored to the earliest retained turn while inserting later effort changes as empty system markers, applies cache diagnostics opt-in (`diagnostics.previous_message_id`), shapes a correlated OAuth lane-start request to one output token without thinking, strips request lineage before reusing bodies in source cache warming or prewarming, and creates `cch` over the final serialized body — `packages/opencode/src/transform.ts`, `packages/opencode/src/server-fallback.ts`, `packages/core/src/thinking-binding.ts`, `packages/core/src/model-remap.ts`, `packages/opencode/src/cache-diagnostics.ts`, `packages/core/src/cch.ts` -7. **Routing** — `shouldFallbackStatus()` checks if response should trigger fallback; native account admission iterates enabled accounts in ordered modes, while `StickySessionRouter` assigns cold sessions by reset-normalized spendable OAuth quota and weighted initial-prompt deficit, then persists hashed session affinity across processes/restarts. Sticky routes retain transient failures, including DNS-related refresh backoffs on a fixed five-minute retry interval while the fallback's current access token remains valid, hold confirmed 5h exhaustion when reset is within 15 minutes, and migrate for longer confirmed exhaustion/permanent account failure. All modes respect model-scoped quotas and killswitch thresholds (including per-model scoped thresholds). API-key routes remain excluded until fresh general OAuth exhaustion is confirmed; model-scoped exhaustion can move only among OAuth accounts. A complete sticky pool with no eligible route returns an explicit 401 for a permanent main-account authentication failure, an explicit 401 naming fallback OAuth accounts that require re-login, or a quota-policy 429 rather than falling through to an excluded account; incomplete pools fail retryably. Killswitch blocks are classified as scoped-driven (matching a specific model's weekly limit) or account-level (5h/7d limits) with a model-specific or generic retry hint — `packages/core/src/routing.ts`, `packages/core/src/sticky-routing.ts`, `packages/core/src/accounts.ts`, `packages/opencode/src/index.ts` +7. **Routing** — `shouldFallbackStatus()` checks if response should trigger fallback; native account admission iterates enabled accounts in ordered modes, while `StickySessionRouter` assigns cold sessions by reset-normalized spendable OAuth quota and weighted initial-prompt deficit, then persists hashed session affinity across processes/restarts. Sticky routes retain transient failures, including DNS-related refresh backoffs on a fixed five-minute retry interval while the fallback's current access token remains valid, and migrate for confirmed exhaustion/permanent account failure. In OpenCode 1 and Pi, sticky-balanced instead holds when all exhausted windows blocking the requested model have known future resets and the latest reset is within 15 minutes; five-hour, seven-day and matching model-scoped limits are treated alike, and other models' scopes do not count. An unknown, invalid or past reset on any exhausted blocking window, or a latest reset beyond 15 minutes, permits migration. OpenCode 2 does not implement this hold yet and can reroute instead. All modes respect model-scoped quotas and killswitch thresholds (including per-model scoped thresholds). API-key routes remain excluded until fresh general OAuth exhaustion is confirmed; model-scoped exhaustion can move only among OAuth accounts. A complete sticky pool with no eligible route returns an explicit 401 for a permanent main-account authentication failure, an explicit 401 naming fallback OAuth accounts that require re-login, or a quota-policy 429 rather than falling through to an excluded account; incomplete pools fail retryably. Killswitch blocks are classified as scoped-driven (matching a specific model's weekly limit) or account-level (5h/7d limits) with a model-specific or generic retry hint — `packages/core/src/routing.ts`, `packages/core/src/sticky-routing.ts`, `packages/core/src/accounts.ts`, `packages/opencode/src/index.ts` 8. **Relay** — `sendViaRelay()` sends full or patched body to Cloudflare Worker, which streams Anthropic response back. HTTP relay responses and WebSocket `response_start` control messages deliver genuine upstream headers to the OpenCode account-bound quota harvester and billing lineage tracker (committing upstream `req_*` request IDs for subsequent session turns); synthetic optimistic response headers are never harvest input, and relay-to-direct fallback stays owned by the direct path — `packages/core/src/relay.ts`, `packages/opencode/src/index.ts` 9. **SSE stream and content-filter safety fallback** — By default, OAuth Fable 5/5.1 and Opus 5 requests send `fallbacks: "default"` with Anthropic's `server-side-fallback-2026-07-01` beta. The stream wrapper detects initial `fallback` blocks, sticky `fallback_message` usage iterations, and restoration to the requested model. Because OpenCode does not natively preserve `fallback` content blocks, the wrapper rewrites each block into a hidden signed thinking marker for storage, then the next eligible request restores the exact Anthropic fallback boundary before signing and sending. If an eligible OAuth response still terminates in refusal, client recovery switches to Opus 4.8 for 10 completed responses, then 20 on a re-refusal, then 40 for each further re-refusal. The source is tried again after every round; ten consecutive successful source responses reset the backoff. No recovery source-model prewarms are sent; ordinary CacheKeep follows the active model. If a served fallback completes a `tool_use` before its terminal refusal, the wrapper preserves that completed call, changes the terminal finish to `tool_use`, and lets OpenCode continue on Opus 4.8 with the existing `tool_result` instead of replaying the tool. The active Anthropic-selected target and later restoration are written per session to the TUI sidebar state; when no matching TUI is connected, OpenCode Desktop receives an ignored/no-reply `promptAsync` notice that the plugin schedules delivery as soon as `session.idle` is observed instead of waiting for a later `session.updated` event, which OpenCode 1.18 no longer emits after idle; delivery still escapes the awaited event handler with `setImmediate` because OpenCode evaluates the run-loop exit condition only after awaiting event handlers. Intermediate `tool-calls` completions do not flush notices. When a refusal arms client recovery, the Desktop switch notice is not queued until the first downgraded Opus response completes successfully, so a transient idle state between the refused source response and OpenCode's internal retry cannot insert a user message or consume the retry response. The plugin probes `session.status()` outside that critical section and revalidates its idle-delivery lease after both the asynchronous status lookup and prompt-context lookup so a new prompt cannot adopt the ignored notice as its retry parent. A newly created user-message event revokes that lease before OpenCode's later busy-status event, while repeated updates for the user message that produced the current idle event do not suppress delivery; if multiple fallback state changes overtake delivery, only the newest notice is retained. Delivered notice message IDs are tracked per session (bounded to four IDs across at most 128 recently used sessions) and excluded from user-message lease revocation so the plugin's own notice cannot cancel its successor; delivery additionally requires the notice still head its queue, with a superseded queue looping to the newest instead of dropping it. Inconclusive probes re-arm for at most four attempts, with attempt 0 on `setImmediate` and later attempts delayed by `25` ms multiplied by the attempt number, and requires message-ID placement before the active assistant; consecutive notices use strictly increasing IDs bounded by that assistant, taking both host history and the latest locally prepared notice into account. If no safe ID can be allocated, delivery stays queued rather than letting the host mint an untracked ID. IDs are registered before dispatch so the notice's own user-message event cannot revoke its delivery lease. This removes the unsafe host-assigned-ID fallback when placement fails. Custom API-key routes strip the beta and `fallbacks` field. Setting `OPENCODE_ANTHROPIC_AUTH_FALLBACK_MODE=legacy` bypasses the server policy and uses the same deterministic Opus 4.8 recovery from the first refusal, keyed by session and source-model family with account-bound source-model prewarming and standby cache bridges beyond Anthropic's 20-block lookback — `packages/opencode/src/transform.ts`, `packages/opencode/src/server-fallback.ts`, `packages/opencode/src/fable-fallback.ts`, `packages/opencode/src/prompt-context.ts`, `packages/opencode/src/index.ts` 10. **Sidebar update** — `writeSidebarState()` writes quota/routing/cache state plus bounded per-session server-side fallback and deterministic recovery status to a JSON file read by the TUI sidebar widget (separate process via RPC). Routing-authoritative writes (e.g. active routing decisions) are distinguished from display-only/metadata writes (e.g. quota refreshes or command paths). Display-only writes re-read the file and merge state to preserve any live routing session's `activeId` and route. Cross-process writes are synchronized using an atomic `mkdir` directory lock with jittered retries, rename-claim eviction (tolerating `ENOENT` and `EINVAL` race conditions during marker eviction), and lock-budget exhaustion skips. The write is fenced: ownership is verified before and after the rename, triggering one bounded locked repair of routing-authoritative fields on post-rename loss. — `packages/opencode/src/sidebar-state.ts`, `packages/opencode/src/index.ts` @@ -238,7 +238,7 @@ ## Error Handling -**Strategy:** Parse failures preserve the original request body; invalid trailing-assistant history and uncorrelated effort markers refuse locally before cache/billing/signing/model dispatch; 429 backoff with exponential retry for token refresh; process-wide main-refresh coalescing across OpenCode project plugin instances; stale-token 401 retries adopt a concurrently persisted replacement credential; retryable stream errors detected and bubbled as synthetic `ECONNRESET` for retry by the caller. Sticky-balanced routing fails retryably while its quota pool is incomplete instead of silently selecting a non-sticky account; once the pool is complete but has no eligible route, it returns an explicit 429 quota block, or a 401 re-login error when the main refresh token is permanently invalid and no fallback can serve the requested model, rather than falling through to the excluded main account. Confirmed five-hour exhaustion with at most 15 minutes until reset returns a deterministic jittered `Retry-After` while retaining affinity. Killswitch blocks requests before they hit the API when quota drops below configured thresholds (including per-model scoped thresholds evaluated against the requested model). Blocks are classified as scoped-driven (when a specific model's weekly limit is reached) or account-level (5h/7d limits), generating detailed user-facing error messages. Fallback routing triggers on confirmed standard quota exhaustion or model-scoped quota exhaustion (stale cached quota never triggers API-key fallback routes). Content-filter handling tries Anthropic's server-side safety fallback first for eligible OAuth Fable 5/5.1 and Opus 5 requests, then activates deterministic client recovery if the response still refuses. Fable 5.1 signed-thinking prefix mismatches caused by host compaction are handled explicitly with Anthropic's `drop_block` control rather than retried as opaque 400 errors. A refusal after a completed tool call preserves that call and continues with its existing result rather than replaying the original turn. Malformed or unrecognized fallback metadata passes through without fabricated state; `legacy` mode bypasses the server policy and uses deterministic client recovery exclusively. Claustrum transport failures back off per handle and fall through to available sidecar credentials; provider-bound custody tombstones fail locally before network refresh, while closing a reconnecting client rejects rather than installing a late replacement. +**Strategy:** Parse failures preserve the original request body; invalid trailing-assistant history and uncorrelated effort markers refuse locally before cache/billing/signing/model dispatch; 429 backoff with exponential retry for token refresh; process-wide main-refresh coalescing across OpenCode project plugin instances; stale-token 401 retries adopt a concurrently persisted replacement credential; retryable stream errors detected and bubbled as synthetic `ECONNRESET` for retry by the caller. Sticky-balanced routing fails retryably while its quota pool is incomplete instead of silently selecting a non-sticky account; once the pool is complete but has no eligible route, it returns an explicit 429 quota block, or a 401 re-login error when the main refresh token is permanently invalid and no fallback can serve the requested model, rather than falling through to the excluded main account. In OpenCode 1 and Pi, exhausted five-hour, seven-day and matching model-scoped windows hold affinity only when every blocking reset is known and in the future and the latest is at most 15 minutes away. The wait uses that latest reset plus deterministic session jitter; OpenCode 1 returns `Retry-After`, while Pi emits an error event. Any exhausted blocking window with an unknown, invalid or past reset permits migration, as does a latest reset beyond 15 minutes. OpenCode 2 does not implement this hold yet. Killswitch blocks requests before they hit the API when quota drops below configured thresholds (including per-model scoped thresholds evaluated against the requested model). Blocks are classified as scoped-driven (when a specific model's weekly limit is reached) or account-level (5h/7d limits), generating detailed user-facing error messages. Fallback routing triggers on confirmed standard quota exhaustion or model-scoped quota exhaustion (stale cached quota never triggers API-key fallback routes). Content-filter handling tries Anthropic's server-side safety fallback first for eligible OAuth Fable 5/5.1 and Opus 5 requests, then activates deterministic client recovery if the response still refuses. Fable 5.1 signed-thinking prefix mismatches caused by host compaction are handled explicitly with Anthropic's `drop_block` control rather than retried as opaque 400 errors. A refusal after a completed tool call preserves that call and continues with its existing result rather than replaying the original turn. Malformed or unrecognized fallback metadata passes through without fabricated state; `legacy` mode bypasses the server policy and uses deterministic client recovery exclusively. Claustrum transport failures back off per handle and fall through to available sidecar credentials; provider-bound custody tombstones fail locally before network refresh, while closing a reconnecting client rejects rather than installing a late replacement. ## Cross-Cutting Concerns diff --git a/AUDITOR.md b/AUDITOR.md index 98de1b602..424467bee 100644 --- a/AUDITOR.md +++ b/AUDITOR.md @@ -29,7 +29,7 @@ Guidance for future audits of intentional tradeoffs in this repository. ## Sticky-balanced routing - **Session affinity intentionally outranks moment-to-moment quota ranking.** Once assigned, a session remains on its OAuth account across transient transport/provider/quota-probe failures and relative quota changes so its large prompt cache is not rewritten on another account. -- **A confirmed five-hour exhaustion does not migrate when reset is within 15 minutes.** The route returns `Retry-After` and remains assigned; longer 5h exhaustion, 7d/model-scoped exhaustion, killswitch blocks, removed/disabled accounts, and permanent re-login failures may migrate. +- **OpenCode 1 and Pi hold when all exhausted windows blocking the requested model reset within 15 minutes.** Five-hour, seven-day and the requested model's scoped window are treated alike: every exhausted blocking window must have a known future reset, and the latest reset determines the wait, with deterministic session jitter. Another model's scope does not count. OpenCode 1 returns `Retry-After`; Pi emits an error event. Unknown, invalid or past resets, longer waits, killswitch blocks, removed/disabled accounts, and permanent re-login failures may migrate. OpenCode 2 does not implement the hold yet and can reroute instead. - **API-key routes are not candidates for quota-balanced first assignment.** Their existing confirmed-main-exhaustion gate remains authoritative. ## CI test-count floor gate diff --git a/CHANGELOG.md b/CHANGELOG.md index e680dc8a2..ff1f67b1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ This repo is a CortexKit-maintained Anthropic auth monorepo for OpenCode and Pi. ## Unreleased +- In sticky-balanced mode on OpenCode 1 and Pi, a session now waits on its account when whichever quota window blocks it resets within 15 minutes, including 7-day and model limits, instead of moving and rebuilding its cache. OpenCode 2 does not hold yet. + - OpenCode 2: finishing an OAuth sign-in from `/claude` → Accounts no longer fails with "Action failed"; the Accounts menu explains how to connect OpenCode when no main account exists. Thanks to @colinmollenhour. Addresses #464. ## 2.2.6 diff --git a/docs/accounts-and-routing.md b/docs/accounts-and-routing.md index 65ae04673..67f8f33db 100644 --- a/docs/accounts-and-routing.md +++ b/docs/accounts-and-routing.md @@ -36,6 +36,8 @@ OAuth login must resolve a verified Claude account UUID. Credentials and refresh Change mode in `/claude` → **Routing**. **Reset routing session** clears the current session's pin; it does not remove accounts. Pins are quota-aware session-affinity hints, not credentials. Changing the selected model can require a fresh assignment. Model-specific exhaustion can move among OAuth accounts, but paid API-key fallback requires confirmed general OAuth exhaustion, not just an exhausted model-specific limit or a display percentage rounded up to 100%. +In OpenCode 1 and Pi, sticky-balanced waits on the assigned account when all exhausted windows blocking the requested model have known future resets and the latest reset is within 15 minutes. This includes five-hour, seven-day and matching model-scoped limits, but not another model's scope. The wait uses the latest reset plus deterministic session jitter; OpenCode 1 returns `Retry-After` and Pi emits an error event. An unknown, invalid or past reset on any exhausted blocking window, or a latest reset beyond 15 minutes, allows migration. OpenCode 2 does not implement this hold yet and can reroute instead. + ## Quota and limits Quota shows five-hour, seven-day and model-scoped windows when available. OpenCode's **Refresh quota** action requests fresh readings; Pi shows the summary without a refresh action. Passive response headers and quota API polling have different coverage: headers can update standard windows without supplying model-scoped limits. diff --git a/docs/configuration.md b/docs/configuration.md index 22d82e289..e32c1e5a8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -40,7 +40,7 @@ Fallbacks are separate OAuth accounts or Anthropic-compatible API-key routes in The balanced allocator normalizes headroom by time until reset and accounts for the initial prompt before the next quota refresh. It persists the assignment by hashed session ID across processes and restarts. **Reset routing session** clears only the current session. A user-selected model change discards its old assignment; transparent safety recovery retains the account. Existing hybrid CacheKeep sessions can seed their warm route instead of changing accounts unnecessarily. -Affinity survives network/relay failures, 5xx responses, unconfirmed 429s, quota-probe failures and relative weight changes. Confirmed seven-day or matching model-scoped exhaustion can move the session. Confirmed five-hour exhaustion moves it only when more than 15 minutes remain before reset; within 15 minutes the route stays and returns `Retry-After`. Disabled/removed accounts, permanent re-login failures and killswitch blocks can also cause reselection. Direct Opus sessions prefer usable accounts whose Fable quota is exhausted, preserving Fable-capable quota. +Affinity survives network/relay failures, 5xx responses, unconfirmed 429s, quota-probe failures and relative weight changes. In OpenCode 1 and Pi, sticky-balanced holds the session on its account when every exhausted window blocking the requested model has a known future reset and the latest reset is within 15 minutes. Five-hour, seven-day and the requested model's scoped limits are treated alike; another model's scoped limit does not count. A hold uses the latest reset plus deterministic per-session jitter. OpenCode 1 returns `Retry-After`; Pi reports the hold as an error event. If any exhausted blocking window has an unknown, invalid or past reset, or the latest reset is more than 15 minutes away, the session can move. OpenCode 2 does not implement this hold yet and can reroute instead. Disabled/removed accounts, permanent re-login failures and killswitch blocks can also cause reselection. Direct Opus sessions prefer usable accounts whose Fable quota is exhausted, preserving Fable-capable quota. ### API-key fallback diff --git a/packages/core/src/sticky-routing.ts b/packages/core/src/sticky-routing.ts index e64f1ac26..717726e6e 100644 --- a/packages/core/src/sticky-routing.ts +++ b/packages/core/src/sticky-routing.ts @@ -86,7 +86,10 @@ export type StickyQuotaFailureDecision = | { action: 'retain'; reason: 'not-exhausted' | 'unknown' } | { action: 'hold' - reason: 'five-hour-short-reset' + reason: + | 'five-hour-short-reset' + | 'seven-day-short-reset' + | 'model-scoped-short-reset' retryAfterSeconds: number } | { @@ -488,43 +491,38 @@ export function decideStickyQuotaFailure(input: { const now = input.now ?? Date.now() const quota = knownQuota(input.quota) if (!quota) return { action: 'retain', reason: 'unknown' } - const scoped = getScopedQuotaWindowForModel(quota, input.modelId) - if ( - scoped && - Number.isFinite(scoped.remainingPercent) && - scoped.remainingPercent <= 0 - ) { - return { action: 'migrate', reason: 'model-scoped' } - } - const sevenDay = quota.seven_day - if ( - sevenDay && - Number.isFinite(sevenDay.remainingPercent) && - sevenDay.remainingPercent <= 0 - ) { - return { action: 'migrate', reason: 'seven-day' } - } - const fiveHour = quota.five_hour - if ( - fiveHour && - Number.isFinite(fiveHour.remainingPercent) && - fiveHour.remainingPercent <= 0 - ) { - const resetAt = fiveHour.resetsAt ? Date.parse(fiveHour.resetsAt) : NaN + let blocking: + | { resetAt: number; reason: 'five-hour' | 'seven-day' | 'model-scoped' } + | undefined + for (const [reason, window] of [ + ['model-scoped', getScopedQuotaWindowForModel(quota, input.modelId)], + ['seven-day', quota.seven_day], + ['five-hour', quota.five_hour], + ] as const) { if ( - Number.isFinite(resetAt) && - resetAt > now && - resetAt - now <= STICKY_ROUTING_SHORT_RESET_GRACE_MS - ) { - return { - action: 'hold', - reason: 'five-hour-short-reset', - retryAfterSeconds: Math.max(1, Math.ceil((resetAt - now) / 1000)), - } + !window || + !Number.isFinite(window.remainingPercent) || + window.remainingPercent > 0 + ) + continue + const resetAt = window.resetsAt ? Date.parse(window.resetsAt) : NaN + if (!Number.isFinite(resetAt) || resetAt <= now) + return { action: 'migrate', reason } + // A request can resume only after every exhausted window blocking its model resets. + if (!blocking || resetAt > blocking.resetAt) blocking = { resetAt, reason } + } + if (!blocking) return { action: 'retain', reason: 'not-exhausted' } + if (blocking.resetAt - now <= STICKY_ROUTING_SHORT_RESET_GRACE_MS) { + return { + action: 'hold', + reason: `${blocking.reason}-short-reset`, + retryAfterSeconds: Math.max( + 1, + Math.ceil((blocking.resetAt - now) / 1000), + ), } - return { action: 'migrate', reason: 'five-hour' } } - return { action: 'retain', reason: 'not-exhausted' } + return { action: 'migrate', reason: blocking.reason } } export function getStickyRoutingStatePath(accountStoragePath: string) { diff --git a/packages/core/src/tests/sticky-quota-failure.test.ts b/packages/core/src/tests/sticky-quota-failure.test.ts new file mode 100644 index 000000000..36154c950 --- /dev/null +++ b/packages/core/src/tests/sticky-quota-failure.test.ts @@ -0,0 +1,142 @@ +import { describe, expect, test } from 'bun:test' +import type { OAuthQuotaSnapshot } from '../accounts.ts' +import { + decideStickyQuotaFailure, + stickyRetryAfterWithJitter, +} from '../sticky-routing.ts' + +const NOW = Date.parse('2026-10-24T08:00:00Z') +const MODEL = 'claude-fable-5' +const kinds = ['five-hour', 'seven-day', 'model-scoped'] as const +type Kind = (typeof kinds)[number] + +function exhausted(kind: Kind, reset: string | undefined): OAuthQuotaSnapshot { + const window = { + usedPercent: 100, + remainingPercent: 0, + resetsAt: reset, + checkedAt: NOW, + } + if (kind === 'five-hour') return { five_hour: window } + if (kind === 'seven-day') return { seven_day: window } + return { + scoped: [ + { + ...window, + id: 'fable-weekly', + title: 'Fable only', + modelName: 'Fable', + }, + ], + } +} + +function decide(quota: OAuthQuotaSnapshot, modelId = MODEL) { + return decideStickyQuotaFailure({ quota, modelId, now: NOW }) +} + +const resetIn = (ms: number) => new Date(NOW + ms).toISOString() + +describe('sticky quota hold', () => { + for (const kind of kinds) { + test(`${kind} holds within fifteen minutes including the boundary`, () => { + for (const ms of [1, 10 * 60_000, 15 * 60_000]) { + expect(decide(exhausted(kind, resetIn(ms)))).toEqual({ + action: 'hold', + reason: `${kind}-short-reset`, + retryAfterSeconds: Math.max(1, Math.ceil(ms / 1000)), + }) + } + }) + + test(`${kind} moves beyond fifteen minutes`, () => { + expect(decide(exhausted(kind, resetIn(15 * 60_000 + 1)))).toEqual({ + action: 'migrate', + reason: kind, + }) + }) + + test(`${kind} moves for unknown invalid or past resets even with another short reset`, () => { + for (const reset of [undefined, '', 'invalid', resetIn(0), resetIn(-1)]) { + const other = kind === 'five-hour' ? 'seven-day' : 'five-hour' + expect( + decide({ + ...exhausted(other, resetIn(60_000)), + ...exhausted(kind, reset), + }), + ).toEqual({ action: 'migrate', reason: kind }) + } + }) + } + + test('uses the latest blocking reset for both the hold and its duration', () => { + for (const first of kinds) { + for (const second of kinds.filter((kind) => kind !== first)) { + const quota = { + ...exhausted(first, resetIn(5 * 60_000)), + ...exhausted(second, resetIn(12 * 60_000)), + } + expect(decide(quota)).toEqual({ + action: 'hold', + reason: `${second}-short-reset`, + retryAfterSeconds: 12 * 60, + }) + expect( + decide({ ...quota, ...exhausted(second, resetIn(16 * 60_000)) }), + ).toEqual({ action: 'migrate', reason: second }) + } + } + expect( + decide({ + ...exhausted('five-hour', resetIn(14 * 60_000)), + ...exhausted('seven-day', resetIn(10 * 60_000)), + ...exhausted('model-scoped', resetIn(2 * 60_000)), + }), + ).toEqual({ + action: 'hold', + reason: 'five-hour-short-reset', + retryAfterSeconds: 14 * 60, + }) + }) + + test('ignores an exhausted scope for a different model including an unknown reset', () => { + for (const reset of [undefined, resetIn(60 * 60_000)]) { + const scoped = exhausted('model-scoped', reset) + expect(decide(scoped, 'claude-opus-5')).toEqual({ + action: 'retain', + reason: 'not-exhausted', + }) + expect( + decide( + { ...scoped, ...exhausted('five-hour', resetIn(10 * 60_000)) }, + 'claude-opus-5', + ), + ).toEqual({ + action: 'hold', + reason: 'five-hour-short-reset', + retryAfterSeconds: 600, + }) + } + }) + + test('retains unknown and non-exhausted quotas unchanged', () => { + for (const quota of [undefined, { kind: 'unknown' as const }]) { + expect(decideStickyQuotaFailure({ quota, now: NOW })).toEqual({ + action: 'retain', + reason: 'unknown', + }) + } + expect(decide({})).toEqual({ action: 'retain', reason: 'not-exhausted' }) + expect( + decide({ + five_hour: { usedPercent: 99, remainingPercent: 1, checkedAt: NOW }, + }), + ).toEqual({ action: 'retain', reason: 'not-exhausted' }) + }) + + test('keeps the existing deterministic SHA256 session jitter', () => { + expect(stickyRetryAfterWithJitter('session-a', 840)).toBe(856) + expect(stickyRetryAfterWithJitter('session-a', 0)).toBe(17) + expect(stickyRetryAfterWithJitter('session-a', 1.1)).toBe(18) + }) +}) diff --git a/packages/opencode/src/cache-diagnostics.ts b/packages/opencode/src/cache-diagnostics.ts index 0236c6453..be72adeca 100644 --- a/packages/opencode/src/cache-diagnostics.ts +++ b/packages/opencode/src/cache-diagnostics.ts @@ -118,7 +118,7 @@ export async function withStickyRetryAfter( error: { type: 'rate_limit_error', message: - 'Sticky OAuth account five-hour quota resets shortly; retaining session affinity.', + 'Sticky OAuth account quota resets shortly; retaining session affinity.', }, }), { status: 429, headers }, diff --git a/packages/opencode/src/index.ts b/packages/opencode/src/index.ts index d8c257a89..8399f053f 100644 --- a/packages/opencode/src/index.ts +++ b/packages/opencode/src/index.ts @@ -9220,7 +9220,7 @@ const anthropicAuthPlugin = async ( error: { type: 'rate_limit_error', message: - 'Sticky OAuth account five-hour quota resets shortly; retaining session affinity.', + 'Sticky OAuth account quota resets shortly; retaining session affinity.', }, }), { diff --git a/packages/opencode/src/tests/index.test.ts b/packages/opencode/src/tests/index.test.ts index fc49f2d08..65d2aa930 100644 --- a/packages/opencode/src/tests/index.test.ts +++ b/packages/opencode/src/tests/index.test.ts @@ -16828,169 +16828,202 @@ describe('auth.loader', () => { ]) }) - test('sticky-balanced retains the assigned account across transient errors and a short 5h reset', async () => { - const checkedAt = Date.now() - const shortResetAt = new Date(checkedAt + 14 * 60_000).toISOString() - const longResetAt = new Date(checkedAt + 2 * 60 * 60_000).toISOString() - let useLongReset = false - let now = checkedAt - const quota = (fableRemaining: number) => ({ - checkedAt, - five_hour: { - usedPercent: 0, - remainingPercent: 100, - resetsAt: new Date(checkedAt + 5 * 60 * 60_000).toISOString(), - checkedAt, - }, - seven_day: { - usedPercent: 0, - remainingPercent: 100, - resetsAt: new Date(checkedAt + 4 * 24 * 60 * 60_000).toISOString(), + for (const window of ['5h', '7d', 'model-scoped'] as const) { + test(`sticky-balanced retains the assigned account across transient errors and a short ${window} reset`, async () => { + const checkedAt = Date.now() + const holdMinutes = window === '5h' ? 14 : 10 + const shortResetAt = new Date( + checkedAt + holdMinutes * 60_000, + ).toISOString() + const longResetAt = new Date(checkedAt + 2 * 60 * 60_000).toISOString() + let useLongReset = false + let now = checkedAt + const quota = (fableRemaining: number) => ({ checkedAt, - }, - scoped: [ - { - id: 'claude-weekly-scoped-fable', - title: 'Fable only', - modelName: 'Fable', - usedPercent: 100 - fableRemaining, - remainingPercent: fableRemaining, + five_hour: { + usedPercent: 0, + remainingPercent: 100, + resetsAt: new Date(checkedAt + 5 * 60 * 60_000).toISOString(), + checkedAt, + }, + seven_day: { + usedPercent: 0, + remainingPercent: 100, resetsAt: new Date(checkedAt + 4 * 24 * 60 * 60_000).toISOString(), checkedAt, }, - ], - }) - await useTempAccountFile( - bindPoolAccounts( - createFallbackStorage({ - routing: { mode: 'sticky-balanced' }, - quota: { - enabled: true, - checkIntervalMinutes: 5, - minimumRemaining: { five_hour: 1, seven_day: 1 }, - failClosedOnUnknownQuota: true, - mainQuota: quota(0), - mainQuotaCheckedAt: checkedAt, - mainQuotaToken: tokenFingerprint('sk-ant-oat01-main-access'), + scoped: [ + { + id: 'claude-weekly-scoped-fable', + title: 'Fable only', + modelName: 'Fable', + usedPercent: 100 - fableRemaining, + remainingPercent: fableRemaining, + resetsAt: new Date(checkedAt + 4 * 24 * 60 * 60_000).toISOString(), + checkedAt, }, - accounts: [ - { - id: 'yiyi', - type: 'oauth', - access: 'sk-ant-oat01-scarce-access', - refresh: 'scarce-refresh', - expires: checkedAt + 5 * 60 * 60_000, - quota: quota(13), - }, - { - id: 'ufuk2', - type: 'oauth', - access: 'sk-ant-oat01-abundant-access', - refresh: 'abundant-refresh', - expires: checkedAt + 5 * 60 * 60_000, - quota: quota(98), + ], + }) + await useTempAccountFile( + bindPoolAccounts( + createFallbackStorage({ + routing: { mode: 'sticky-balanced' }, + quota: { + enabled: true, + checkIntervalMinutes: 5, + minimumRemaining: { five_hour: 1, seven_day: 1 }, + failClosedOnUnknownQuota: true, + mainQuota: quota(0), + mainQuotaCheckedAt: checkedAt, + mainQuotaToken: tokenFingerprint('sk-ant-oat01-main-access'), }, - ], - }), - ), - ) - const authorizations: string[] = [] - let modelRequest = 0 - globalThis.fetch = mock( - withNativeAdmission((input: any, init: any) => { - const url = extractUrl(input) - if (url.includes('/api/oauth/usage')) { - const authorization = new Headers(init?.headers).get('authorization') - const abundant = - authorization === 'Bearer sk-ant-oat01-abundant-access' - const main = authorization === 'Bearer sk-ant-oat01-main-access' - return Promise.resolve( - new Response( - JSON.stringify({ - five_hour: { - utilization: abundant ? 100 : 0, - resets_at: useLongReset ? longResetAt : shortResetAt, - }, - seven_day: { utilization: 0 }, - limits: [ - { - kind: 'weekly_scoped', - group: 'weekly', - percent: main ? 100 : abundant ? 2 : 87, - resets_at: new Date( - checkedAt + 4 * 24 * 60 * 60_000, - ).toISOString(), - scope: { model: { display_name: 'Fable' } }, + accounts: [ + { + id: 'yiyi', + type: 'oauth', + access: 'sk-ant-oat01-scarce-access', + refresh: 'scarce-refresh', + expires: checkedAt + 5 * 60 * 60_000, + quota: quota(13), + }, + { + id: 'ufuk2', + type: 'oauth', + access: 'sk-ant-oat01-abundant-access', + refresh: 'abundant-refresh', + expires: checkedAt + 5 * 60 * 60_000, + quota: quota(98), + }, + ], + }), + ), + ) + const authorizations: string[] = [] + let modelRequest = 0 + globalThis.fetch = mock( + withNativeAdmission((input: any, init: any) => { + const url = extractUrl(input) + if (url.includes('/api/oauth/usage')) { + const authorization = new Headers(init?.headers).get( + 'authorization', + ) + const abundant = + authorization === 'Bearer sk-ant-oat01-abundant-access' + const main = authorization === 'Bearer sk-ant-oat01-main-access' + return Promise.resolve( + new Response( + JSON.stringify({ + five_hour: { + utilization: abundant && window === '5h' ? 100 : 0, + resets_at: useLongReset ? longResetAt : shortResetAt, }, - ], - }), - { status: 200 }, - ), - ) - } - if (!url.includes('/v1/messages')) { - return Promise.resolve(new Response('{}', { status: 200 })) - } - authorizations.push( - new Headers(init?.headers).get('authorization') ?? '', - ) - modelRequest += 1 - if (modelRequest === 1) { - return Promise.resolve(new Response('temporary', { status: 500 })) - } - if (modelRequest === 2) { - return Promise.resolve( - new Response( - 'event: error\ndata: {"type":"error","error":{"type":"rate_limit_error","message":"five-hour"}}\n\n', - { status: 200 }, - ), + seven_day: { + utilization: abundant && window === '7d' ? 100 : 0, + resets_at: useLongReset ? longResetAt : shortResetAt, + }, + limits: [ + { + kind: 'weekly_scoped', + group: 'weekly', + percent: + main || (abundant && window === 'model-scoped') + ? 100 + : abundant + ? 2 + : 87, + resets_at: + abundant && window === 'model-scoped' + ? useLongReset + ? longResetAt + : shortResetAt + : new Date( + checkedAt + 4 * 24 * 60 * 60_000, + ).toISOString(), + scope: { model: { display_name: 'Fable' } }, + }, + ], + }), + { status: 200 }, + ), + ) + } + if (!url.includes('/v1/messages')) { + return Promise.resolve(new Response('{}', { status: 200 })) + } + authorizations.push( + new Headers(init?.headers).get('authorization') ?? '', ) - } - return Promise.resolve(new Response('ok', { status: 200 })) - }), - ) as unknown as typeof fetch + modelRequest += 1 + if (modelRequest === 1) { + return Promise.resolve(new Response('temporary', { status: 500 })) + } + if (modelRequest === 2) { + return Promise.resolve( + new Response( + 'event: error\ndata: {"type":"error","error":{"type":"rate_limit_error","message":"quota"}}\n\n', + { status: 200 }, + ), + ) + } + return Promise.resolve(new Response('ok', { status: 200 })) + }), + ) as unknown as typeof fetch - Date.now = mock(() => now) as unknown as typeof Date.now - const plugin = await getPlugin() - const result = await plugin.auth.loader( - () => - Promise.resolve({ - type: 'oauth', - access: 'sk-ant-oat01-main-access', - refresh: 'main-refresh', - expires: checkedAt + 10 * 60 * 60_000, + Date.now = mock(() => now) as unknown as typeof Date.now + const plugin = await getPlugin() + const result = await plugin.auth.loader( + () => + Promise.resolve({ + type: 'oauth', + access: 'sk-ant-oat01-main-access', + refresh: 'main-refresh', + expires: checkedAt + 10 * 60 * 60_000, + }), + { models: {} }, + ) + const request = { + method: 'POST', + headers: { 'x-session-affinity': 'ses_sticky_hold' }, + body: JSON.stringify({ + model: 'claude-fable-5', + stream: true, + messages: [{ role: 'user', content: 'hello' }], }), - { models: {} }, - ) - const request = { - method: 'POST', - headers: { 'x-session-affinity': 'ses_sticky_hold' }, - body: JSON.stringify({ - model: 'claude-fable-5', - stream: true, - messages: [{ role: 'user', content: 'hello' }], - }), - } + } - expect((await result.fetch(MESSAGES_URL, request)).status).toBe(500) - const held = await result.fetch(MESSAGES_URL, request) - expect(held.status).toBe(429) - expect(Number(held.headers.get('retry-after'))).toBeGreaterThanOrEqual( - 13 * 60, - ) - expect(Number(held.headers.get('retry-after'))).toBeLessThanOrEqual(15 * 60) - expect((await result.fetch(MESSAGES_URL, request)).status).toBe(429) + expect((await result.fetch(MESSAGES_URL, request)).status).toBe(500) + const held = await result.fetch(MESSAGES_URL, request) + expect(held.status).toBe(429) + expect(Number(held.headers.get('retry-after'))).toBeGreaterThanOrEqual( + holdMinutes * 60, + ) + expect(Number(held.headers.get('retry-after'))).toBeLessThanOrEqual( + holdMinutes * 60 + 20, + ) + expect(await held.text()).toContain( + 'Sticky OAuth account quota resets shortly; retaining session affinity.', + ) + const proactive = await result.fetch(MESSAGES_URL, request) + expect(proactive.status).toBe(429) + expect(await proactive.text()).toContain( + 'Sticky OAuth account quota resets shortly; retaining session affinity.', + ) + expect(authorizations).toEqual([ + 'Bearer sk-ant-oat01-abundant-access', + 'Bearer sk-ant-oat01-abundant-access', + ]) - useLongReset = true - now += 6 * 60_000 - const migrated = await result.fetch(MESSAGES_URL, request) - expect(migrated.status).toBe(200) - expect(authorizations).toEqual([ - 'Bearer sk-ant-oat01-abundant-access', - 'Bearer sk-ant-oat01-abundant-access', - 'Bearer sk-ant-oat01-scarce-access', - ]) - }) + useLongReset = true + now += 6 * 60_000 + const migrated = await result.fetch(MESSAGES_URL, request) + expect(migrated.status).toBe(200) + expect(authorizations).toEqual([ + 'Bearer sk-ant-oat01-abundant-access', + 'Bearer sk-ant-oat01-abundant-access', + 'Bearer sk-ant-oat01-scarce-access', + ]) + }) + } test('normalizes a server fallback response even when no session-affinity header is available', async () => { delete process.env.OPENCODE_ANTHROPIC_AUTH_FALLBACK_MODE diff --git a/packages/pi/src/stream.ts b/packages/pi/src/stream.ts index a1785431c..dee2b086b 100644 --- a/packages/pi/src/stream.ts +++ b/packages/pi/src/stream.ts @@ -1434,7 +1434,7 @@ async function executeWithFallback(options: { error: { type: 'rate_limit_error', message: - 'Sticky OAuth account five-hour quota resets shortly; retaining session affinity.', + 'Sticky OAuth account quota resets shortly; retaining session affinity.', }, }), { @@ -1514,7 +1514,7 @@ async function executeWithFallback(options: { error: { type: 'rate_limit_error', message: - 'Sticky OAuth account five-hour quota resets shortly; retaining session affinity.', + 'Sticky OAuth account quota resets shortly; retaining session affinity.', }, }), { status: 429, headers }, diff --git a/packages/pi/src/tests/stream.test.ts b/packages/pi/src/tests/stream.test.ts index 223429d8a..e3d68b01b 100644 --- a/packages/pi/src/tests/stream.test.ts +++ b/packages/pi/src/tests/stream.test.ts @@ -10,6 +10,7 @@ import { signRequestBody, } from '@cortexkit/anthropic-auth-core' import { nativeQuotaCodec } from '../../../core/src/native-quota-codec.ts' +import { getPiNativeRuntime } from '../native.ts' import { buildExplicitBaseMessagesUrl, configureApiRouteHeaders, @@ -886,6 +887,188 @@ describe('Pi API fallback routing helpers', () => { } }) + for (const window of ['5h', '7d', 'model-scoped'] as const) { + test(`sticky-balanced holds the assigned Pi account for a 10-minute ${window} reset and moves for a far reset`, async () => { + tempDir = await mkdtemp(join(tmpdir(), 'pi-sticky-hold-')) + const storagePath = join(tempDir, 'anthropic-auth.json') + const routingPath = join(tempDir, 'sticky-routes.json') + process.env.PI_ANTHROPIC_AUTH_FILE = storagePath + process.env.PI_ANTHROPIC_AUTH_ROUTING_STATE_FILE = routingPath + const checkedAt = Date.now() + const quota = (remaining: number) => ({ + checkedAt, + five_hour: { usedPercent: 0, remainingPercent: 100, checkedAt }, + seven_day: { usedPercent: 0, remainingPercent: 100, checkedAt }, + scoped: [ + { + id: 'claude-weekly-scoped-fable', + title: 'Fable only', + modelName: 'Fable', + usedPercent: 100 - remaining, + remainingPercent: remaining, + resetsAt: new Date(checkedAt + 4 * 24 * 60 * 60_000).toISOString(), + checkedAt, + }, + ], + }) + await saveNativePiFixture( + { + version: 1, + mainAccountId: 'main-account', + main: { type: 'opencode', provider: 'anthropic' }, + fallbackOn: [401, 403, 429], + quota: { + enabled: true, + checkIntervalMinutes: 5, + minimumRemaining: { five_hour: 1, seven_day: 1 }, + failClosedOnUnknownQuota: true, + mainQuota: { ...quota(0), accountIdentity: 'main-account' }, + mainQuotaCheckedAt: checkedAt, + }, + routing: { mode: 'sticky-balanced' }, + accounts: [ + { + id: 'yiyi', + type: 'oauth', + access: 'sk-ant-oat01-scarce-access', + refresh: 'scarce-refresh', + expires: checkedAt + 5 * 60 * 60_000, + quota: quota(13), + }, + { + id: 'ufuk2', + type: 'oauth', + access: 'sk-ant-oat01-abundant-access', + refresh: 'abundant-refresh', + expires: checkedAt + 5 * 60 * 60_000, + quota: quota(98), + }, + ], + }, + storagePath, + ) + + let now = checkedAt + let useLongReset = false + const authorizations: string[] = [] + globalThis.fetch = mock( + (input: string | URL | Request, init?: RequestInit) => { + const url = input instanceof Request ? input.url : input.toString() + const authorization = + new Headers(init?.headers).get('authorization') ?? '' + const abundant = + authorization === 'Bearer sk-ant-oat01-abundant-access' + if (url.includes('/api/oauth/usage')) { + const reset = new Date( + checkedAt + (useLongReset ? 120 : 10) * 60_000, + ).toISOString() + return Promise.resolve( + new Response( + JSON.stringify({ + five_hour: { + utilization: abundant && window === '5h' ? 100 : 0, + resets_at: reset, + }, + seven_day: { + utilization: abundant && window === '7d' ? 100 : 0, + resets_at: reset, + }, + limits: [ + { + kind: 'weekly_scoped', + group: 'weekly', + percent: + authorization === 'Bearer sk-ant-oat01-main-access' || + (abundant && window === 'model-scoped') + ? 100 + : abundant + ? 2 + : 87, + resets_at: reset, + scope: { model: { display_name: 'Fable' } }, + }, + ], + }), + { status: 200 }, + ), + ) + } + if (!url.includes('/v1/messages')) + return Promise.resolve(new Response('{}')) + authorizations.push(authorization) + return Promise.resolve( + new Response( + abundant + ? 'event: error\ndata: {"type":"error","error":{"type":"rate_limit_error","message":"quota"}}\n\n' + : [ + 'event: message_start\ndata: {"type":"message_start","message":{"usage":{"input_tokens":1,"output_tokens":0}}}\n\n', + 'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":1}}\n\n', + 'event: message_stop\ndata: {"type":"message_stop"}\n\n', + ].join(''), + { status: 200 }, + ), + ) + }, + ) as unknown as typeof fetch + + const originalNow = Date.now + Date.now = () => now + try { + for (let request = 0; request < 2; request++) { + const events = [] + const stream = streamCortexKitAnthropic( + anthropicModel, + anthropicContext, + { + apiKey: 'sk-ant-oat01-main-access', + sessionId: 'ses_pi_sticky_hold', + }, + ) + for await (const event of stream) events.push(event) + const error = events.find((event) => event.type === 'error') + expect(error?.error.errorMessage).toContain('HTTP 429') + expect(error?.error.errorMessage).toContain( + 'Sticky OAuth account quota resets shortly; retaining session affinity.', + ) + await getPiNativeRuntime(storagePath).drainRoutingPublications() + expect( + Object.values( + JSON.parse(await readFile(routingPath, 'utf8')).assignments, + ), + ).toMatchObject([{ accountId: 'ufuk2' }]) + } + expect(authorizations).toEqual(['Bearer sk-ant-oat01-abundant-access']) + + useLongReset = true + now += 6 * 60_000 + const moved = streamCortexKitAnthropic( + anthropicModel, + anthropicContext, + { + apiKey: 'sk-ant-oat01-main-access', + sessionId: 'ses_pi_sticky_hold', + }, + ) + for await (const _event of moved) { + // Drain the successful retry so its final result is available. + } + expect((await moved.result()).errorMessage).toBeUndefined() + expect(authorizations).toEqual([ + 'Bearer sk-ant-oat01-abundant-access', + 'Bearer sk-ant-oat01-scarce-access', + ]) + await getPiNativeRuntime(storagePath).drainRoutingPublications() + expect( + Object.values( + JSON.parse(await readFile(routingPath, 'utf8')).assignments, + ), + ).toMatchObject([{ accountId: 'yiyi' }]) + } finally { + Date.now = originalNow + } + }) + } + test('sticky-balanced keeps repeated Pi session requests on the quota-selected account', async () => { tempDir = await mkdtemp(join(tmpdir(), 'pi-sticky-routing-')) const storagePath = join(tempDir, 'anthropic-auth.json') From fd42b793ad73fd9d6e47ce4fcc2c1f455df2d1ba Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Sat, 24 Oct 2026 13:58:01 +0200 Subject: [PATCH 2/2] mason: register sticky quota hold mutation controls Add seven-day-always-migrate and first-reset-only controls beside the Core routing rows, including full-package broad audit settings. Both exact named tests are CAUGHT through the pinned ckdev-mutate 0.12.5 runner and Bun wrapper. --- mutations.toml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/mutations.toml b/mutations.toml index 14a130748..1f06d9a0c 100644 --- a/mutations.toml +++ b/mutations.toml @@ -635,6 +635,39 @@ broad_command = ["bun", "scripts/run-mutation-bun.ts", "--broad", "packages/core broad_report = "tmp/mutations/core.xml" broad_id = "{name}" +[[control]] +id = "sticky-quota-seven-day-always-migrate" +guards = "An exhausted seven-day window with a known reset within fifteen minutes must hold instead of migrating immediately." +file = "packages/core/src/sticky-routing.ts" +old = " const resetAt = window.resetsAt ? Date.parse(window.resetsAt) : NaN" +new = """ if (reason === 'seven-day') return { action: 'migrate', reason } // NON-VACUITY BREAK: migrate on seven-day exhaustion without checking its reset. + const resetAt = window.resetsAt ? Date.parse(window.resetsAt) : NaN""" +test_file = "packages/core/src/tests/sticky-quota-failure.test.ts" +runner = "command" +command = ["bun", "scripts/run-mutation-bun.ts", "packages/core/src/tests/sticky-quota-failure.test.ts", "{test}"] +test_count_pattern = "CKDEV executed {count} tests" +expect_red = ["sticky quota hold > seven-day holds within fifteen minutes including the boundary"] +expect_message = "error: expect(received).toEqual(expected)" +broad_command = ["bun", "scripts/run-mutation-bun.ts", "--broad", "packages/core", "tmp/mutations/core.xml"] +broad_report = "tmp/mutations/core.xml" +broad_id = "{name}" + +[[control]] +id = "sticky-quota-drop-latest-reset-keep-first" +guards = "The latest reset among exhausted windows blocking the requested model must determine whether to hold and how long to wait." +file = "packages/core/src/sticky-routing.ts" +old = " if (!blocking || resetAt > blocking.resetAt) blocking = { resetAt, reason }" +new = " if (!blocking) blocking = { resetAt, reason } // NON-VACUITY BREAK: keep the first reset instead of the latest." +test_file = "packages/core/src/tests/sticky-quota-failure.test.ts" +runner = "command" +command = ["bun", "scripts/run-mutation-bun.ts", "packages/core/src/tests/sticky-quota-failure.test.ts", "{test}"] +test_count_pattern = "CKDEV executed {count} tests" +expect_red = ["sticky quota hold > uses the latest blocking reset for both the hold and its duration"] +expect_message = "error: expect(received).toEqual(expected)" +broad_command = ["bun", "scripts/run-mutation-bun.ts", "--broad", "packages/core", "tmp/mutations/core.xml"] +broad_report = "tmp/mutations/core.xml" +broad_id = "{name}" + [[control]] id = "pi-auth-slot-uses-sdk-directory" guards = "Offline vault setup protects Pi's actual auth file, not an extension-only directory override."