Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions src/providers/opencode-zen-rate-limit.ts
Original file line number Diff line number Diff line change
Expand Up @@ -175,3 +175,25 @@ export function enrichOpenCodeZenUpstreamMessage(
): string {
return enrichOpenCodeZenFreeTierMessage(enrichOpenCodeZenRateLimitMessage(message, opts), opts);
}

/**
* Console Go transient upload rejection.
*
* Both Go and Zen hand the request to OpenCode's Console gateway, which intermittently answers
* a body it accepts moments later with a 400 invalid_request_error / "Invalid upload request." —
* a rejection naming an upload step the caller never invoked (the request is plain JSON).
*
* Live evidence (2026-09-12, usage ledger, conversation a016a2a76ee921d9300342e56bf5824d):
* request ocx-ce696fa4a554e4f3e375cf120904b0da failed 400, while ocx-ed78f47b96f7d1bb7c —
* sent 22s later with the same model, the same inline-image history and the same tool output —
* answered 200. The rejection is therefore an upstream flap, not a verdict on the payload, so
* callers replay the byte-identical request once instead of failing the turn.
* Upstream tracker: anomalyco/opencode#47237.
*/
export function isTransientConsoleGoUploadRejection(opts: {
status: number;
errorBody: string | undefined;
}): boolean {
if (opts.status !== 400 || !opts.errorBody) return false;
return /invalid upload request/i.test(opts.errorBody);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}
96 changes: 95 additions & 1 deletion src/server/responses/core.ts
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,10 @@ import {
} from "../../lib/errors";
import { injectionDebugLog } from "../../lib/injection-debug-log";
import { resolveClientRetryAfter } from "../../lib/retry-after";
import { enrichOpenCodeZenUpstreamMessage } from "../../providers/opencode-zen-rate-limit";
import {
enrichOpenCodeZenUpstreamMessage,
isTransientConsoleGoUploadRejection,
} from "../../providers/opencode-zen-rate-limit";
import { CODE_MODE_EXEC_TOOL_NAME, modelInList, namespacedToolName } from "../../types";
import type {
AdapterEvent,
Expand Down Expand Up @@ -213,6 +216,7 @@ import {
fetchWithTransientRetry,
isNonReplayableResponse,
prepareSameTarget429Wait,
sleepWithAbort,
} from "../../lib/upstream-retry";
import {
ForwardAdmissionCredentialError,
Expand Down Expand Up @@ -848,6 +852,33 @@ async function opaqueBlobRejectionBodyForRecovery(
}
}

/**
* Console Go transient-400 retry delay. A short pause lets a momentary gateway state clear
* before the single byte-identical replay; it is imperceptible inside a turn that already
* waits seconds for the model.
*/
const CONSOLE_GO_UPLOAD_RETRY_DELAY_MS = 800;

/**
* Peek the upstream error body for the Console Go transient-400 recovery. Only a complete,
* display-safe body may drive a retry decision (same contract as
* opaqueBlobRejectionBodyForRecovery), and reading a clone leaves the original response intact
* for the caller's own error surface when no retry is taken.
*/
async function consoleGoUploadRejectionBody(
response: Response,
alreadyAttempted: boolean,
signal: AbortSignal,
): Promise<string | undefined> {
if (response.status !== 400 || alreadyAttempted) return undefined;
try {
const body = await readBoundedResponseBody(response.clone(), { signal });
return body.displaySafe && !body.truncated ? body.text : undefined;
} catch {
return undefined;
}
}

/**
* Materialize an upstream error body only when the bounded reader observed a complete,
* display-safe payload. Partial timeout and over-limit prefixes are attacker-controlled,
Expand Down Expand Up @@ -5231,6 +5262,9 @@ async function handleResponsesInner(
const opaqueBlobRecoveryGuard: OpaqueBlobRecoveryGuard = { attempted: false };
let oauth401ReplayAttempted = false;
let codex401ReplayKind: "main" | "stored" | null = null;
// Console Go answers a transient 400 "Invalid upload request." for bodies it accepts
// moments later; at most one byte-identical replay is allowed per request.
const consoleGoUploadRetryGuard: { attempted: boolean } = { attempted: false };
const rateLimitPolicy = rateLimitRetryPolicyFor(route.provider);
let rateLimitRetries = 0;
const rebuildAndRefetch = async (
Expand Down Expand Up @@ -5778,6 +5812,34 @@ async function handleResponsesInner(
logCtx.terminalIncompleteReason = preflightLog.terminalIncompleteReason;
}
}
// Console Go (opencode-zen / opencode-go) intermittently rejects a body it accepts seconds
// later with 400 invalid_request_error / "Invalid upload request." Replay the byte-identical
// request once: the same conversation re-sent 22s later answered 200, so surfacing this flap
// would fail a turn the upstream would have served. Single-shot guard.
if (!consoleGoUploadRetryGuard.attempted) {
const uploadRejectionBody = await consoleGoUploadRejectionBody(
upstreamResponse,
consoleGoUploadRetryGuard.attempted,
upstream.signal,
);
if (uploadRejectionBody !== undefined
&& isTransientConsoleGoUploadRejection({
status: upstreamResponse.status,
errorBody: uploadRejectionBody,
})) {
consoleGoUploadRetryGuard.attempted = true;
try { void upstreamResponse.body?.cancel().catch(() => {}); } catch { /* already consumed/closed */ }
if (!upstream.signal.aborted) {
try {
await sleepWithAbort(CONSOLE_GO_UPLOAD_RETRY_DELAY_MS, upstream.signal);
} catch { /* client aborted: the replay below settles through the normal abort path */ }
}
const result = await rebuildAndRefetch("console-go-upload-retry");
if ("failed" in result) return result.failed;
upstreamResponse = result;
continue passthroughRecovery;
}
}
break;
}
const headers = sanitizePassthroughHeaders(upstreamResponse.headers);
Expand Down Expand Up @@ -7267,6 +7329,9 @@ async function handleResponsesInner(
// 413→429 rotation cannot silently undo the tightening.
let imageRetryAttempted = false;
const opaqueBlobRecoveryGuard: OpaqueBlobRecoveryGuard = { attempted: false };
// Console Go answers a transient 400 "Invalid upload request." for bodies it accepts
// moments later; at most one byte-identical replay is allowed per request.
const consoleGoUploadRetryGuard: { attempted: boolean } = { attempted: false };
let oauth401ReplayAttempted = false;
/**
* Rebuild the request from the current parsed input (and any image-tier bias) and refetch
Expand Down Expand Up @@ -7651,6 +7716,35 @@ async function handleResponsesInner(
upstreamResponse = result;
continue recovery;
}
// Console Go (opencode-zen / opencode-go) intermittently rejects a body it accepts seconds
// later with 400 invalid_request_error / "Invalid upload request." Replay the
// byte-identical request once: the same conversation re-sent 22s later answered 200, so
// surfacing this flap would fail a turn the upstream would have served. Single-shot guard.
if (!consoleGoUploadRetryGuard.attempted) {
const uploadRejectionBody = await consoleGoUploadRejectionBody(
upstreamResponse,
consoleGoUploadRetryGuard.attempted,
upstream.signal,
);
if (uploadRejectionBody !== undefined
&& isTransientConsoleGoUploadRejection({
status: upstreamResponse.status,
errorBody: uploadRejectionBody,
})) {
consoleGoUploadRetryGuard.attempted = true;
invalidateSameTargetRequest();
try { void upstreamResponse.body?.cancel().catch(() => {}); } catch { /* already consumed/closed */ }
if (!upstream.signal.aborted) {
try {
await sleepWithAbort(CONSOLE_GO_UPLOAD_RETRY_DELAY_MS, upstream.signal);
} catch { /* client aborted: the replay below settles through the normal abort path */ }
}
const result = await rebuildAndRefetch("console-go-upload-retry");
if ("failed" in result) return result.failed;
upstreamResponse = result;
continue recovery;
}
}
break;
}
if (!upstreamResponse.ok) {
Expand Down
2 changes: 2 additions & 0 deletions src/usage/log.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ export type AttemptRecoveryKind =
| "anthropic-oauth-429"
| "oauth-account-429"
| "image-413"
| "console-go-upload-retry"
| "opaque-blob-rejection"
| "empty-completion";

Expand Down Expand Up @@ -309,6 +310,7 @@ const ATTEMPT_RECOVERY_KINDS = new Set<AttemptRecoveryKind>([
"anthropic-oauth-429",
"oauth-account-429",
"image-413",
"console-go-upload-retry",
"opaque-blob-rejection",
"empty-completion",
]);
Expand Down
39 changes: 39 additions & 0 deletions tests/providers/opencode-zen-rate-limit.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import {
enrichOpenCodeZenFreeTierMessage,
enrichOpenCodeZenRateLimitMessage,
enrichOpenCodeZenUpstreamMessage,
isTransientConsoleGoUploadRejection,
isOpenCodeZenFreeTierLockIn,
isOpenCodeZenRateLimitProvider,
} from "../../src/providers/opencode-zen-rate-limit";
Expand Down Expand Up @@ -230,3 +231,41 @@ describe("opencode-free keyless tier lock-in (#4121)", () => {
expect(lockedOut).not.toContain(OPENCODE_ZEN_OBSERVED_RPM_HINT);
});
});

describe("Console Go transient upload refusal", () => {
// The exact body Console Go returns for a payload it accepts moments later.
const UPLOAD_REFUSAL = JSON.stringify({
model: "muse-spark-1.3-contributor",
error: {
param: null,
type: "invalid_request_error",
message: "Error from provider (Console Go): Upstream request failed: [invalid_request_error] Invalid upload request.",
},
});

test("accepts the 400 upload refusal regardless of casing", () => {
expect(isTransientConsoleGoUploadRejection({ status: 400, errorBody: UPLOAD_REFUSAL })).toBe(true);
expect(isTransientConsoleGoUploadRejection({
status: 400,
errorBody: '{"error":"INVALID UPLOAD REQUEST."}',
})).toBe(true);
});

test("rejects every other status and error body", () => {
// Status is part of the identity: only the gateway's 400 is the flap.
expect(isTransientConsoleGoUploadRejection({ status: 200, errorBody: UPLOAD_REFUSAL })).toBe(false);
expect(isTransientConsoleGoUploadRejection({ status: 500, errorBody: UPLOAD_REFUSAL })).toBe(false);
expect(isTransientConsoleGoUploadRejection({ status: 400, errorBody: undefined })).toBe(false);
expect(isTransientConsoleGoUploadRejection({ status: 400, errorBody: "" })).toBe(false);
// Other 400s on the same wire are deterministic verdicts on the request, not flaps.
expect(isTransientConsoleGoUploadRejection({
status: 400,
errorBody: '{"error":{"param":"reasoning.effort","type":"invalid_request_error","message":"Error from provider (Console Go): Upstream request failed: [invalid_request_error] reasoning_effort max requires an active Muse Code subscription for model muse-spark-1.3-contributor."}}',
})).toBe(false);
expect(isTransientConsoleGoUploadRejection({
status: 400,
errorBody: '{"type":"error","error":{"type":"MissingSessionID","message":"Request is missing x-opencode-session"}}',
})).toBe(false);
});
});

134 changes: 134 additions & 0 deletions tests/responses/responses-console-go-upload-retry.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
import { mkdtempSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { handleResponses } from "../../src/server/responses/core";
import type { RequestLogContext } from "../../src/server/request-log";
import type { OcxConfig } from "../../src/types";
import { removeTreeWithRetry } from "../helpers/remove-tree";

const originalFetch = globalThis.fetch;
const originalOpenCodexHome = process.env.OPENCODEX_HOME;

/** The exact Console Go rejection for a payload it accepts moments later. */
const UPLOAD_REFUSAL = JSON.stringify({
model: "muse-spark-1.3-contributor",
error: {
param: null,
type: "invalid_request_error",
message: "Error from provider (Console Go): Upstream request failed: [invalid_request_error] Invalid upload request.",
},
});

/** A deterministic 400 on the same wire: a verdict on the request, never a flap. */
const EFFORT_REFUSAL = JSON.stringify({
model: "muse-spark-1.3-contributor",
error: {
param: "reasoning.effort",
type: "invalid_request_error",
message: "Error from provider (Console Go): Upstream request failed: [invalid_request_error] reasoning_effort max requires an active Muse Code subscription for model muse-spark-1.3-contributor.",
},
});

let testDir = "";

beforeEach(() => {
testDir = mkdtempSync(join(tmpdir(), "ocx-console-go-upload-retry-"));
process.env.OPENCODEX_HOME = testDir;
});

afterEach(() => {
globalThis.fetch = originalFetch;
if (originalOpenCodexHome === undefined) delete process.env.OPENCODEX_HOME;
else process.env.OPENCODEX_HOME = originalOpenCodexHome;
removeTreeWithRetry(testDir);
});

function config(): OcxConfig {
return {
defaultProvider: "go",
providers: {
go: {
adapter: "openai-responses",
baseUrl: "https://opencode.ai/zen/go/v1",
authMode: "key",
apiKey: "go-test-key",
},
},
} as OcxConfig;
}

function request(stream = false): Request {
return new Request("http://localhost/v1/responses", {
method: "POST",
headers: {
"content-type": "application/json",
"session_id": "thread-console-go-upload-retry",
},
body: JSON.stringify({
model: "go/muse-spark-1.3-contributor",
stream,
store: false,
input: [
{ type: "message", role: "user", content: [{ type: "input_text", text: "hi" }] },
],
}),
});
}

function refusal(status = 400, body = UPLOAD_REFUSAL): Response {
return new Response(body, { status, headers: { "content-type": "application/json" } });
}

function success(id: string): Response {
return Response.json({ id, object: "response", status: "completed", model: "muse-spark-1.3-contributor", output: [] });
}

describe("Console Go transient upload refusal recovery", () => {
test("replays the refusal once and serves the retry with a byte-identical body", async () => {
const outbound: string[] = [];
globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => {
outbound.push(String(init?.body));
return outbound.length === 1 ? refusal() : success("resp-upload-retry-recovered");
}) as typeof fetch;
const logCtx: RequestLogContext = { model: "", provider: "" };

const response = await handleResponses(request(), config(), logCtx);

expect(response.status).toBe(200);
expect(outbound).toHaveLength(2);
// The replay must be the same request, not a reshaped one. Evidence: usage ledger
// ocx-ce696fa4a554e4f3e375cf120904b0da (400) vs ocx-ed78f47b96f7d1bb7c (200, +22s).
expect(outbound[1]).toBe(outbound[0]);
expect(logCtx.activeAttempt?.recoveryKinds).toEqual(["console-go-upload-retry"]);
});

test("does not replay a different 400 from the same wire", async () => {
let sends = 0;
globalThis.fetch = (async () => {
sends += 1;
return refusal(400, EFFORT_REFUSAL);
}) as typeof fetch;

const response = await handleResponses(request(), config(), { model: "", provider: "" });

expect(response.status).toBe(400);
expect(sends).toBe(1);
});

test("keeps a repeated refusal visible after the single bounded replay", async () => {
let sends = 0;
globalThis.fetch = (async () => {
sends += 1;
return refusal();
}) as typeof fetch;
const logCtx: RequestLogContext = { model: "", provider: "" };

const response = await handleResponses(request(), config(), logCtx);

expect(response.status).toBe(400);
expect(sends).toBe(2);
expect(logCtx.activeAttempt?.recoveryKinds).toEqual(["console-go-upload-retry"]);
});
});

Loading