diff --git a/src/providers/opencode-zen-rate-limit.ts b/src/providers/opencode-zen-rate-limit.ts index c4dbb10319e..01a84cae05a 100644 --- a/src/providers/opencode-zen-rate-limit.ts +++ b/src/providers/opencode-zen-rate-limit.ts @@ -175,3 +175,81 @@ export function enrichOpenCodeZenUpstreamMessage( ): string { return enrichOpenCodeZenFreeTierMessage(enrichOpenCodeZenRateLimitMessage(message, opts), opts); } + +/** + * Console Go transient upload rejection. + * + * Both Go and Zen hand the request to OpenCode's Console gateway, which intermittently answers + * a body it accepts moments later with a 400 invalid_request_error / "Invalid upload request." — + * a rejection naming an upload step the caller never invoked (the request is plain JSON). + * + * Live evidence (2026-09-12, usage ledger, conversation a016a2a76ee921d9300342e56bf5824d): + * request ocx-ce696fa4a554e4f3e375cf120904b0da failed 400, while ocx-ed78f47b96f7d1bb7c — + * sent 22s later with the same model, the same inline-image history and the same tool output — + * answered 200. The rejection is therefore an upstream flap, not a verdict on the payload, so + * callers replay the byte-identical request once instead of failing the turn. + * Upstream tracker: anomalyco/opencode#47237. + */ +/** Canonical Console destinations: the Zen/Go provider rows and any opencode.ai /zen base URL. */ +const CONSOLE_GO_PROVIDER_IDS = new Set(["opencode-go", "opencode-zen", "opencode-free"]); + +export function isConsoleGoDestination(opts: { + providerName?: string; + baseUrl?: string; +}): boolean { + const name = opts.providerName?.trim(); + if (name && CONSOLE_GO_PROVIDER_IDS.has(name)) return true; + const baseUrl = opts.baseUrl?.trim(); + if (!baseUrl) return false; + try { + const url = new URL(baseUrl); + if (url.hostname !== "opencode.ai") return false; + const path = url.pathname.replace(/\/+$/, ""); + return path === "/zen" || path.startsWith("/zen/"); + } catch { + return false; + } +} + +/** + * The canonical refusal envelope, as served on both Console routes: + * {"error":{"param":null,"type":"invalid_request_error","message":"Error from provider + * (Console Go): Upstream request failed: [invalid_request_error] Invalid upload request."}} + * + * The gateway names itself Console on the Zen key route and Console Go on the Go route, so the + * anchor is the shared product name plus the refusal sentence. The message is matched whole: a + * bare string, a suffix, or any other envelope is a different refusal and must not be replayed. + * Being stricter than necessary is the safe direction: a missed match leaves the turn failing + * exactly as it does today, while a loose match spends an extra request on unrelated 400s. + */ +const CONSOLE_UPLOAD_REFUSAL_RE = + /^Error from provider \(Console(?: Go)?\): Upstream request failed: \[invalid_request_error\] Invalid upload request\.$/; + +/** + * True only for the canonical Console upload refusal on a canonical Console destination. + * Route-gated on purpose: the message alone would let any other upstream that happens to answer + * with this English sentence trigger a second send from an unrelated provider. + */ +export function isTransientConsoleGoUploadRejection(opts: { + status: number; + errorBody: string | undefined; + providerName?: string; + baseUrl?: string; +}): boolean { + if (opts.status !== 400 || !opts.errorBody) return false; + if (!isConsoleGoDestination({ providerName: opts.providerName, baseUrl: opts.baseUrl })) return false; + let payload: unknown; + try { + payload = JSON.parse(opts.errorBody); + } catch { + return false; + } + const error = (payload as { error?: unknown } | null)?.error; + if (!error || typeof error !== "object" || Array.isArray(error)) return false; + // The whole envelope, not just the sentence: Console always answers this refusal as + // invalid_request_error with a null param, so a partial envelope is a different error. + const envelope = error as { param?: unknown; type?: unknown; message?: unknown }; + if (envelope.type !== "invalid_request_error" || envelope.param !== null) return false; + // Matched without trimming: padding means the gateway wrapped or appended something. + return typeof envelope.message === "string" && CONSOLE_UPLOAD_REFUSAL_RE.test(envelope.message); +} diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 1f78570eae7..7180c442e0c 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -102,7 +102,10 @@ import { } from "../../lib/errors"; import { injectionDebugLog } from "../../lib/injection-debug-log"; import { resolveClientRetryAfter } from "../../lib/retry-after"; -import { enrichOpenCodeZenUpstreamMessage } from "../../providers/opencode-zen-rate-limit"; +import { + enrichOpenCodeZenUpstreamMessage, + isTransientConsoleGoUploadRejection, +} from "../../providers/opencode-zen-rate-limit"; import { CODE_MODE_EXEC_TOOL_NAME, modelInList, namespacedToolName } from "../../types"; import type { AdapterEvent, @@ -213,6 +216,7 @@ import { fetchWithTransientRetry, isNonReplayableResponse, prepareSameTarget429Wait, + sleepWithAbort, } from "../../lib/upstream-retry"; import { ForwardAdmissionCredentialError, @@ -848,6 +852,33 @@ async function opaqueBlobRejectionBodyForRecovery( } } +/** + * Console Go transient-400 retry delay. A short pause lets a momentary gateway state clear + * before the single byte-identical replay; it is imperceptible inside a turn that already + * waits seconds for the model. + */ +const CONSOLE_GO_UPLOAD_RETRY_DELAY_MS = 800; + +/** + * Peek the upstream error body for the Console Go transient-400 recovery. Only a complete, + * display-safe body may drive a retry decision (same contract as + * opaqueBlobRejectionBodyForRecovery), and reading a clone leaves the original response intact + * for the caller's own error surface when no retry is taken. + */ +async function consoleGoUploadRejectionBody( + response: Response, + alreadyAttempted: boolean, + signal: AbortSignal, +): Promise { + if (response.status !== 400 || alreadyAttempted) return undefined; + try { + const body = await readBoundedResponseBody(response.clone(), { signal }); + return body.displaySafe && !body.truncated ? body.text : undefined; + } catch { + return undefined; + } +} + /** * Materialize an upstream error body only when the bounded reader observed a complete, * display-safe payload. Partial timeout and over-limit prefixes are attacker-controlled, @@ -5231,6 +5262,9 @@ async function handleResponsesInner( const opaqueBlobRecoveryGuard: OpaqueBlobRecoveryGuard = { attempted: false }; let oauth401ReplayAttempted = false; let codex401ReplayKind: "main" | "stored" | null = null; + // Console Go answers a transient 400 "Invalid upload request." for bodies it accepts + // moments later; at most one byte-identical replay is allowed per request. + const consoleGoUploadRetryGuard: { attempted: boolean } = { attempted: false }; const rateLimitPolicy = rateLimitRetryPolicyFor(route.provider); let rateLimitRetries = 0; const rebuildAndRefetch = async ( @@ -5778,6 +5812,38 @@ async function handleResponsesInner( logCtx.terminalIncompleteReason = preflightLog.terminalIncompleteReason; } } + // Console Go (opencode-zen / opencode-go) intermittently rejects a body it accepts seconds + // later with 400 invalid_request_error / "Invalid upload request." Replay the byte-identical + // request once: the same conversation re-sent 22s later answered 200, so surfacing this flap + // would fail a turn the upstream would have served. Single-shot guard. + // No same-target cache invalidation here: the passthrough rebuild calls buildRequest on every + // attempt, and the cache guarded by invalidateSameTargetRequest() belongs to the generic loop. + if (!consoleGoUploadRetryGuard.attempted) { + const uploadRejectionBody = await consoleGoUploadRejectionBody( + upstreamResponse, + consoleGoUploadRetryGuard.attempted, + upstream.signal, + ); + if (uploadRejectionBody !== undefined + && isTransientConsoleGoUploadRejection({ + status: upstreamResponse.status, + errorBody: uploadRejectionBody, + providerName: route.providerName, + baseUrl: route.provider.baseUrl, + })) { + consoleGoUploadRetryGuard.attempted = true; + try { void upstreamResponse.body?.cancel().catch(() => {}); } catch { /* already consumed/closed */ } + if (!upstream.signal.aborted) { + try { + await sleepWithAbort(CONSOLE_GO_UPLOAD_RETRY_DELAY_MS, upstream.signal); + } catch { /* client aborted: the replay below settles through the normal abort path */ } + } + const result = await rebuildAndRefetch("console-go-upload-retry"); + if ("failed" in result) return result.failed; + upstreamResponse = result; + continue passthroughRecovery; + } + } break; } const headers = sanitizePassthroughHeaders(upstreamResponse.headers); @@ -7267,6 +7333,9 @@ async function handleResponsesInner( // 413→429 rotation cannot silently undo the tightening. let imageRetryAttempted = false; const opaqueBlobRecoveryGuard: OpaqueBlobRecoveryGuard = { attempted: false }; + // Console Go answers a transient 400 "Invalid upload request." for bodies it accepts + // moments later; at most one byte-identical replay is allowed per request. + const consoleGoUploadRetryGuard: { attempted: boolean } = { attempted: false }; let oauth401ReplayAttempted = false; /** * Rebuild the request from the current parsed input (and any image-tier bias) and refetch @@ -7651,6 +7720,37 @@ async function handleResponsesInner( upstreamResponse = result; continue recovery; } + // Console Go (opencode-zen / opencode-go) intermittently rejects a body it accepts seconds + // later with 400 invalid_request_error / "Invalid upload request." Replay the + // byte-identical request once: the same conversation re-sent 22s later answered 200, so + // surfacing this flap would fail a turn the upstream would have served. Single-shot guard. + if (!consoleGoUploadRetryGuard.attempted) { + const uploadRejectionBody = await consoleGoUploadRejectionBody( + upstreamResponse, + consoleGoUploadRetryGuard.attempted, + upstream.signal, + ); + if (uploadRejectionBody !== undefined + && isTransientConsoleGoUploadRejection({ + status: upstreamResponse.status, + errorBody: uploadRejectionBody, + providerName: route.providerName, + baseUrl: route.provider.baseUrl, + })) { + consoleGoUploadRetryGuard.attempted = true; + invalidateSameTargetRequest(); + try { void upstreamResponse.body?.cancel().catch(() => {}); } catch { /* already consumed/closed */ } + if (!upstream.signal.aborted) { + try { + await sleepWithAbort(CONSOLE_GO_UPLOAD_RETRY_DELAY_MS, upstream.signal); + } catch { /* client aborted: the replay below settles through the normal abort path */ } + } + const result = await rebuildAndRefetch("console-go-upload-retry"); + if ("failed" in result) return result.failed; + upstreamResponse = result; + continue recovery; + } + } break; } if (!upstreamResponse.ok) { diff --git a/src/usage/log.ts b/src/usage/log.ts index 2944c22f9ae..953195b7cae 100644 --- a/src/usage/log.ts +++ b/src/usage/log.ts @@ -70,6 +70,7 @@ export type AttemptRecoveryKind = | "anthropic-oauth-429" | "oauth-account-429" | "image-413" + | "console-go-upload-retry" | "opaque-blob-rejection" | "empty-completion"; @@ -309,6 +310,7 @@ const ATTEMPT_RECOVERY_KINDS = new Set([ "anthropic-oauth-429", "oauth-account-429", "image-413", + "console-go-upload-retry", "opaque-blob-rejection", "empty-completion", ]); diff --git a/tests/providers/opencode-zen-rate-limit.test.ts b/tests/providers/opencode-zen-rate-limit.test.ts index 8751082c2ab..3bf4c4db52a 100644 --- a/tests/providers/opencode-zen-rate-limit.test.ts +++ b/tests/providers/opencode-zen-rate-limit.test.ts @@ -6,6 +6,7 @@ import { enrichOpenCodeZenFreeTierMessage, enrichOpenCodeZenRateLimitMessage, enrichOpenCodeZenUpstreamMessage, + isTransientConsoleGoUploadRejection, isOpenCodeZenFreeTierLockIn, isOpenCodeZenRateLimitProvider, } from "../../src/providers/opencode-zen-rate-limit"; @@ -230,3 +231,64 @@ describe("opencode-free keyless tier lock-in (#4121)", () => { expect(lockedOut).not.toContain(OPENCODE_ZEN_OBSERVED_RPM_HINT); }); }); + +describe("Console Go transient upload refusal", () => { + // The observed Go-route refusal, byte-for-byte as Console serves it. + const GO_MESSAGE = "Error from provider (Console Go): Upstream request failed: [invalid_request_error] Invalid upload request."; + // The Zen key route names the same gateway without the Go suffix. + const ZEN_MESSAGE = "Error from provider (Console): Upstream request failed: [invalid_request_error] Invalid upload request."; + const envelope = (message: string) => JSON.stringify({ model: "muse-spark-1.3-contributor", error: { param: null, type: "invalid_request_error", message } }); + const GO_ROUTE = { providerName: "opencode-go", baseUrl: "https://opencode.ai/zen/go/v1" }; + const ZEN_ROUTE = { providerName: "opencode-zen", baseUrl: "https://opencode.ai/zen/v1" }; + + test("accepts the canonical refusal on both canonical Console routes", () => { + expect(isTransientConsoleGoUploadRejection({ status: 400, errorBody: envelope(GO_MESSAGE), ...GO_ROUTE })).toBe(true); + expect(isTransientConsoleGoUploadRejection({ status: 400, errorBody: envelope(ZEN_MESSAGE), ...ZEN_ROUTE })).toBe(true); + // A custom row pointed at the same destination is still Console: the base URL decides. + expect(isTransientConsoleGoUploadRejection({ status: 400, errorBody: envelope(GO_MESSAGE), providerName: "my-go-row", baseUrl: "https://opencode.ai/zen/go/v1" })).toBe(true); + }); + + test("rejects the refusal text from a non-Console route", () => { + expect(isTransientConsoleGoUploadRejection({ status: 400, errorBody: envelope(GO_MESSAGE), providerName: "deepseek", baseUrl: "https://api.deepseek.com/v1" })).toBe(false); + // opencode.ai without the /zen segment is not the Console gateway. + expect(isTransientConsoleGoUploadRejection({ status: 400, errorBody: envelope(GO_MESSAGE), providerName: "other", baseUrl: "https://opencode.ai/v1" })).toBe(false); + expect(isTransientConsoleGoUploadRejection({ status: 400, errorBody: envelope(GO_MESSAGE) })).toBe(false); + }); + + test("rejects noncanonical envelopes, suffixes, and other statuses", () => { + // A bare string is not the structured envelope. + expect(isTransientConsoleGoUploadRejection({ status: 400, errorBody: JSON.stringify({ error: "Invalid upload request." }), ...GO_ROUTE })).toBe(false); + // A suffix means the gateway said something else; do not guess. + expect(isTransientConsoleGoUploadRejection({ status: 400, errorBody: envelope(GO_MESSAGE + " Please retry."), ...GO_ROUTE })).toBe(false); + // Different status: only the gateway 400 is the flap. + expect(isTransientConsoleGoUploadRejection({ status: 500, errorBody: envelope(GO_MESSAGE), ...GO_ROUTE })).toBe(false); + expect(isTransientConsoleGoUploadRejection({ status: 400, errorBody: undefined, ...GO_ROUTE })).toBe(false); + // Other 400s on the same wire are verdicts on the request, not flaps. + expect(isTransientConsoleGoUploadRejection({ + status: 400, + errorBody: envelope("Error from provider (Console Go): Upstream request failed: [invalid_request_error] reasoning_effort max requires an active Muse Code subscription for model muse-spark-1.3-contributor."), + ...GO_ROUTE, + })).toBe(false); + expect(isTransientConsoleGoUploadRejection({ + status: 400, + errorBody: JSON.stringify({ type: "error", error: { type: "MissingSessionID", message: "Request is missing x-opencode-session" } }), + ...GO_ROUTE, + })).toBe(false); + }); + + test("rejects partial envelopes and padded messages", () => { + const withError = (error: unknown) => JSON.stringify({ model: "muse-spark-1.3-contributor", error }); + // type carries the refusal identity; a partial envelope is a different error. + expect(isTransientConsoleGoUploadRejection({ status: 400, errorBody: withError({ param: null, message: GO_MESSAGE }), ...GO_ROUTE })).toBe(false); + expect(isTransientConsoleGoUploadRejection({ status: 400, errorBody: withError({ param: null, type: "server_error", message: GO_MESSAGE }), ...GO_ROUTE })).toBe(false); + // param must be present and null. + expect(isTransientConsoleGoUploadRejection({ status: 400, errorBody: withError({ type: "invalid_request_error", message: GO_MESSAGE }), ...GO_ROUTE })).toBe(false); + expect(isTransientConsoleGoUploadRejection({ status: 400, errorBody: withError({ param: "input", type: "invalid_request_error", message: GO_MESSAGE }), ...GO_ROUTE })).toBe(false); + // Padding means the gateway wrapped or appended something. + expect(isTransientConsoleGoUploadRejection({ status: 400, errorBody: withError({ param: null, type: "invalid_request_error", message: " " + GO_MESSAGE }), ...GO_ROUTE })).toBe(false); + expect(isTransientConsoleGoUploadRejection({ status: 400, errorBody: withError({ param: null, type: "invalid_request_error", message: GO_MESSAGE + "\n" }), ...GO_ROUTE })).toBe(false); + // The exact canonical envelope still matches. + expect(isTransientConsoleGoUploadRejection({ status: 400, errorBody: envelope(GO_MESSAGE), ...GO_ROUTE })).toBe(true); + }); +}); + diff --git a/tests/responses/responses-console-go-upload-retry.test.ts b/tests/responses/responses-console-go-upload-retry.test.ts new file mode 100644 index 00000000000..a21f333882f --- /dev/null +++ b/tests/responses/responses-console-go-upload-retry.test.ts @@ -0,0 +1,153 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { handleResponses } from "../../src/server/responses/core"; +import type { RequestLogContext } from "../../src/server/request-log"; +import type { OcxConfig } from "../../src/types"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +const originalFetch = globalThis.fetch; +const originalOpenCodexHome = process.env.OPENCODEX_HOME; + +/** The exact Console Go rejection for a payload it accepts moments later. */ +const UPLOAD_REFUSAL = JSON.stringify({ + model: "muse-spark-1.3-contributor", + error: { + param: null, + type: "invalid_request_error", + message: "Error from provider (Console Go): Upstream request failed: [invalid_request_error] Invalid upload request.", + }, +}); + +/** A deterministic 400 on the same wire: a verdict on the request, never a flap. */ +const EFFORT_REFUSAL = JSON.stringify({ + model: "muse-spark-1.3-contributor", + error: { + param: "reasoning.effort", + type: "invalid_request_error", + message: "Error from provider (Console Go): Upstream request failed: [invalid_request_error] reasoning_effort max requires an active Muse Code subscription for model muse-spark-1.3-contributor.", + }, +}); + +let testDir = ""; + +beforeEach(() => { + testDir = mkdtempSync(join(tmpdir(), "ocx-console-go-upload-retry-")); + process.env.OPENCODEX_HOME = testDir; +}); + +afterEach(() => { + globalThis.fetch = originalFetch; + if (originalOpenCodexHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = originalOpenCodexHome; + removeTreeWithRetry(testDir); +}); + +function config(): OcxConfig { + return { + defaultProvider: "go", + providers: { + go: { + adapter: "openai-responses", + baseUrl: "https://opencode.ai/zen/go/v1", + authMode: "key", + apiKey: "go-test-key", + }, + other: { + adapter: "openai-responses", + baseUrl: "https://other.example.test/v1", + authMode: "key", + apiKey: "other-test-key", + }, + }, + } as OcxConfig; +} + +function request(stream = false, provider = "go"): Request { + return new Request("http://localhost/v1/responses", { + method: "POST", + headers: { + "content-type": "application/json", + "session_id": "thread-console-go-upload-retry", + }, + body: JSON.stringify({ + model: provider + "/muse-spark-1.3-contributor", + stream, + store: false, + input: [ + { type: "message", role: "user", content: [{ type: "input_text", text: "hi" }] }, + ], + }), + }); +} + +function refusal(status = 400, body = UPLOAD_REFUSAL): Response { + return new Response(body, { status, headers: { "content-type": "application/json" } }); +} + +function success(id: string): Response { + return Response.json({ id, object: "response", status: "completed", model: "muse-spark-1.3-contributor", output: [] }); +} + +describe("Console Go transient upload refusal recovery", () => { + test("replays the refusal once and serves the retry with a byte-identical body", async () => { + const outbound: string[] = []; + globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { + outbound.push(String(init?.body)); + return outbound.length === 1 ? refusal() : success("resp-upload-retry-recovered"); + }) as typeof fetch; + const logCtx: RequestLogContext = { model: "", provider: "" }; + + const response = await handleResponses(request(), config(), logCtx); + + expect(response.status).toBe(200); + expect(outbound).toHaveLength(2); + // The replay must be the same request, not a reshaped one. Evidence: usage ledger + // ocx-ce696fa4a554e4f3e375cf120904b0da (400) vs ocx-ed78f47b96f7d1bb7c (200, +22s). + expect(outbound[1]).toBe(outbound[0]); + expect(logCtx.activeAttempt?.recoveryKinds).toEqual(["console-go-upload-retry"]); + }); + + test("does not replay a different 400 from the same wire", async () => { + let sends = 0; + globalThis.fetch = (async () => { + sends += 1; + return refusal(400, EFFORT_REFUSAL); + }) as typeof fetch; + + const response = await handleResponses(request(), config(), { model: "", provider: "" }); + + expect(response.status).toBe(400); + expect(sends).toBe(1); + }); + + test("keeps a repeated refusal visible after the single bounded replay", async () => { + let sends = 0; + globalThis.fetch = (async () => { + sends += 1; + return refusal(); + }) as typeof fetch; + const logCtx: RequestLogContext = { model: "", provider: "" }; + + const response = await handleResponses(request(), config(), logCtx); + + expect(response.status).toBe(400); + expect(sends).toBe(2); + expect(logCtx.activeAttempt?.recoveryKinds).toEqual(["console-go-upload-retry"]); + }); + + test("does not replay the same refusal text from a non-Console provider", async () => { + let sends = 0; + globalThis.fetch = (async () => { + sends += 1; + return refusal(); + }) as typeof fetch; + + const response = await handleResponses(request(false, "other"), config(), { model: "", provider: "" }); + + expect(response.status).toBe(400); + expect(sends).toBe(1); + }); +}); +