diff --git a/apps/gateway/src/chat/chat.ts b/apps/gateway/src/chat/chat.ts index bd08b70750..ec182f3481 100644 --- a/apps/gateway/src/chat/chat.ts +++ b/apps/gateway/src/chat/chat.ts @@ -10022,6 +10022,12 @@ chat.openapi(completions, async (c) => { model: usedInternalModel, bufferSnapshot: buffer ? buffer.substring(0, 5000) : undefined, phase: "upstream_read", + // Stealth providers must not leak their identity through a + // mid-stream read fault: the raw message, undici cause chain + // (host / ENOTFOUND / TLS) and buffered body are scrubbed from + // the client SSE, matching the sibling redaction at + // chat.ts:8335 / chat.ts:9254. The internal log stays raw. + redact: shouldRedactProviderError(usedProvider), }); const upstreamReadErrorMeta = { diff --git a/apps/gateway/src/chat/tools/normalize-streaming-error.spec.ts b/apps/gateway/src/chat/tools/normalize-streaming-error.spec.ts index e045023c25..fe89ad645e 100644 --- a/apps/gateway/src/chat/tools/normalize-streaming-error.spec.ts +++ b/apps/gateway/src/chat/tools/normalize-streaming-error.spec.ts @@ -1,5 +1,7 @@ import { describe, expect, it } from "vitest"; +import { redactedProviderErrorText } from "@/lib/stealth-provider-errors.js"; + import { isUpstreamTermination, normalizeStreamingError, @@ -109,6 +111,141 @@ describe("normalizeStreamingError", () => { expect(normalized.log.details.responseText).not.toBe("[object Object]"); expect(normalized.client.message).not.toContain("[object Object]"); }); + + it("scrubs the client payload when redact is set but keeps the log raw", () => { + // A mid-stream read fault whose undici cause chain and buffered body both + // embed the secret stealth host / vendor identity. + const dnsError = new Error( + "getaddrinfo ENOTFOUND api.secretvendor.com", + ) as Error & { code?: string }; + dnsError.code = "ENOTFOUND"; + const error = new TypeError("fetch failed", { cause: dnsError }); + + const bufferSnapshot = + 'data: {"error":{"message":"SecretVendor quota exceeded at api.secretvendor.com"}}'; + + const normalized = normalizeStreamingError({ + error, + provider: "tundra", + model: "kimi-k2.6", + bufferSnapshot, + phase: "upstream_read", + redact: true, + }); + + // Client payload: no host, vendor identity, raw message or buffered body. + const clientSerialized = JSON.stringify(normalized.client); + expect(clientSerialized).not.toContain("SecretVendor"); + expect(clientSerialized).not.toContain("secretvendor.com"); + expect(clientSerialized).not.toContain("fetch failed"); + expect(clientSerialized).not.toContain("getaddrinfo"); + expect(clientSerialized).not.toContain(bufferSnapshot); + + // The cause chain (which embeds the secret host) is dropped entirely. + expect(normalized.client.details.cause).toBeUndefined(); + // Node-level failure identifiers are dropped too, matching the sibling + // redactErrorDetails: only statusCode/statusText survive. + expect(normalized.client.details.errorName).toBeUndefined(); + expect(normalized.client.details.errorCode).toBeUndefined(); + expect(clientSerialized).not.toContain("ENOTFOUND"); + // Only the gateway-derived status survives, as the generic status text. + expect(normalized.client.details.statusCode).toBe(500); + expect(normalized.client.responseText).toBe(redactedProviderErrorText(500)); + expect(normalized.client.message).toBe(redactedProviderErrorText(500)); + expect(normalized.client.code).toBe("streaming_error"); + + // The internal log is never redacted: it feeds the internal-only columns. + expect(normalized.log.details.responseText).toContain( + "api.secretvendor.com", + ); + expect(normalized.log.details.cause).toContain("ENOTFOUND"); + expect(normalized.log.details.bufferSnapshot).toBe(bufferSnapshot); + }); + + it("scrubs the buffered body on a redacted upstream termination", () => { + // The realistic mid-stream leak: the provider kills the socket while a + // half-written, vendor-branded event is still sitting in the SSE buffer, + // so the raw bytes reach the client through responseText rather than + // through the error message. + const socketCloseError = new Error("other side closed") as Error & { + code?: string; + }; + socketCloseError.name = "SocketError"; + socketCloseError.code = "UND_ERR_SOCKET"; + const error = new TypeError("terminated", { cause: socketCloseError }); + + const bufferSnapshot = + 'data: {"choices":[{"delta":{"content":"SecretVendor internal note: api.secretvendor.com'; + + const normalized = normalizeStreamingError({ + error, + provider: "tundra", + model: "kimi-k2.6", + bufferSnapshot, + phase: "upstream_read", + redact: true, + }); + + // Redaction must not change how the failure is classified: it is still an + // upstream termination, so it keeps the 502/warn treatment. + expect(normalized.terminated).toBe(true); + expect(normalized.client.details.statusCode).toBe(502); + expect(normalized.client.details.statusText).toBe( + "Upstream Stream Terminated", + ); + expect(normalized.client.type).toBe("gateway_error"); + + const clientSerialized = JSON.stringify(normalized.client); + expect(clientSerialized).not.toContain("SecretVendor"); + expect(clientSerialized).not.toContain("secretvendor.com"); + expect(clientSerialized).not.toContain("UND_ERR_SOCKET"); + expect(normalized.client.responseText).toBe(redactedProviderErrorText(502)); + expect(normalized.client.message).toBe(redactedProviderErrorText(502)); + + expect(normalized.log.details.bufferSnapshot).toBe(bufferSnapshot); + expect(normalized.log.details.errorCode).toBe("UND_ERR_SOCKET"); + }); + + it("leaves the client payload untouched when redact is false", () => { + const socketCloseError = new Error("other side closed") as Error & { + code?: string; + }; + socketCloseError.name = "SocketError"; + socketCloseError.code = "UND_ERR_SOCKET"; + const error = new TypeError("terminated", { cause: socketCloseError }); + + const normalized = normalizeStreamingError({ + error, + provider: "openai", + model: "gpt-4.1-mini", + bufferSnapshot: 'data: {"choices":[{"delta":{"content":"partial', + phase: "upstream_read", + redact: false, + }); + + expect(normalized.client.responseText).toBe( + 'data: {"choices":[{"delta":{"content":"partial', + ); + expect(normalized.client.details.errorCode).toBe("UND_ERR_SOCKET"); + expect(normalized.client.details.cause).toContain("other side closed"); + }); + + it("leaves the client payload untouched when redact is not set", () => { + const error = new SyntaxError("Unexpected end of JSON input"); + + const normalized = normalizeStreamingError({ + error, + provider: "openai", + model: "gpt-4.1-mini", + bufferSnapshot: "data: {", + phase: "upstream_read", + }); + + expect(normalized.client.message).toBe( + "Streaming error: Unexpected end of JSON input", + ); + expect(normalized.client.responseText).toBe("data: {"); + }); }); describe("isUpstreamTermination", () => { diff --git a/apps/gateway/src/chat/tools/normalize-streaming-error.ts b/apps/gateway/src/chat/tools/normalize-streaming-error.ts index 5adf8ffadc..84cc0f4b0e 100644 --- a/apps/gateway/src/chat/tools/normalize-streaming-error.ts +++ b/apps/gateway/src/chat/tools/normalize-streaming-error.ts @@ -1,3 +1,5 @@ +import { redactedProviderErrorText } from "@/lib/stealth-provider-errors.js"; + import { extractErrorCause } from "./extract-error-cause.js"; interface ErrorWithCode extends Error { @@ -11,6 +13,17 @@ export interface NormalizeStreamingErrorOptions { model: string; bufferSnapshot?: string; phase: "upstream_connect" | "upstream_read"; + /** + * When true, the client-facing payload is scrubbed of every raw upstream + * detail so a stealth provider's identity cannot leak through a mid-stream + * read fault. The raw error message, the undici `cause` chain (which can + * embed the secret host via ENOTFOUND/ECONNREFUSED/TLS errors) and the + * buffered upstream body are all dropped; only the gateway-derived status + * survives. The internal `log` payload is never redacted — it feeds the + * internal-only log columns, consistent with the sibling error branches + * (chat.ts:8335 / chat.ts:9254) and redactErrorDetails. + */ + redact?: boolean; } export interface NormalizedStreamingError { @@ -30,7 +43,7 @@ export interface NormalizedStreamingError { details: { statusCode: number; statusText: string; - errorName: string; + errorName?: string; errorCode?: string; cause?: string; }; @@ -153,7 +166,7 @@ export function isUpstreamTermination(error: unknown): boolean { export function normalizeStreamingError( options: NormalizeStreamingErrorOptions, ): NormalizedStreamingError { - const { error, provider, model, bufferSnapshot, phase } = options; + const { error, provider, model, bufferSnapshot, phase, redact } = options; const errorName = error instanceof Error @@ -176,22 +189,42 @@ export function normalizeStreamingError( : `Streaming error: ${rawMessage}`; const responseText = cause ? `${rawMessage} | cause: ${cause}` : rawMessage; + const client: NormalizedStreamingError["client"] = redact + ? { + // Generic, status-only payload: no raw message, no cause chain, + // no buffered upstream body — none of which may reach a client + // for a stealth provider. + message: redactedProviderErrorText(statusCode), + type: "gateway_error", + param: null, + code: "streaming_error", + responseText: redactedProviderErrorText(statusCode), + // Status only: errorName / errorCode / cause are all dropped so a stealth + // provider's failure mode cannot be inferred client-side, matching + // redactErrorDetails on the sibling branches (chat.ts 8335 / 9254). + details: { + statusCode, + statusText, + }, + } + : { + message, + type: "gateway_error", + param: null, + code: "streaming_error", + responseText: bufferSnapshot, + details: { + statusCode, + statusText, + errorName, + ...(errorCode ? { errorCode } : {}), + ...(cause ? { cause } : {}), + }, + }; + return { terminated, - client: { - message, - type: "gateway_error", - param: null, - code: "streaming_error", - responseText: bufferSnapshot, - details: { - statusCode, - statusText, - errorName, - ...(errorCode ? { errorCode } : {}), - ...(cause ? { cause } : {}), - }, - }, + client, log: { message: rawMessage, type: "streaming_error", diff --git a/apps/gateway/src/stealth-error-redaction.spec.ts b/apps/gateway/src/stealth-error-redaction.spec.ts index 29a059d9d3..9312aa95e7 100644 --- a/apps/gateway/src/stealth-error-redaction.spec.ts +++ b/apps/gateway/src/stealth-error-redaction.spec.ts @@ -49,6 +49,10 @@ const LEAKY_ERROR_BODY = JSON.stringify({ }, }); +// A half-written, unparseable SSE chunk that still carries the vendor identity. +const TRUNCATED_LEAKY_MARKER = `${SECRET_VENDOR} internal note: api.secretvendor.com`; +const TRUNCATED_LEAKY_CHUNK = `data: {"choices":[{"delta":{"content":"${TRUNCATED_LEAKY_MARKER}`; + describe("stealth provider error redaction (routes)", () => { const harness = createGatewayApiTestHarness(); @@ -63,10 +67,13 @@ describe("stealth provider error redaction (routes)", () => { req.on("end", () => { const isStream = body.includes('"stream":true'); // "http500" prompts force a plain HTTP error even for streams; - // "midstream" prompts get a valid delta chunk before the error. + // "midstream" prompts get a valid delta chunk before the error; + // "readfault" prompts kill the socket after a half-written event so + // the gateway fails inside reader.read() with the leaky bytes still + // sitting in its SSE buffer. if (isStream && !body.includes("http500")) { res.writeHead(200, { "content-type": "text/event-stream" }); - if (body.includes("midstream")) { + if (body.includes("midstream") || body.includes("readfault")) { res.write( `data: ${JSON.stringify({ id: "cmpl-1", @@ -83,6 +90,16 @@ describe("stealth provider error redaction (routes)", () => { })}\n\n`, ); } + if (body.includes("readfault")) { + // Truncated mid-JSON and never terminated, so the gateway can + // neither parse it as an event nor as an in-band error: the raw + // vendor-branded bytes are still sitting in its SSE buffer when + // the socket dies, which is what lands in bufferSnapshot. + res.write(TRUNCATED_LEAKY_CHUNK, () => { + res.socket?.destroy(); + }); + return; + } res.write(`data: ${LEAKY_ERROR_BODY}\n\n`); res.end(); return; @@ -105,6 +122,8 @@ describe("stealth provider error redaction (routes)", () => { "LLM_TUNDRA_BASE_URL", "LLM_GLACIER_API_KEY", "LLM_GLACIER_BASE_URL", + "LLM_OPENAI_API_KEY", + "LLM_OPENAI_BASE_URL", ]) { savedEnv[key] = process.env[key]; } @@ -112,6 +131,9 @@ describe("stealth provider error redaction (routes)", () => { process.env.LLM_TUNDRA_BASE_URL = leakyServerUrl; process.env.LLM_GLACIER_API_KEY = "glacier-env-key"; process.env.LLM_GLACIER_BASE_URL = leakyServerUrl; + // openai is the non-stealth control: same leaky mock, no redaction. + process.env.LLM_OPENAI_API_KEY = "openai-env-key"; + process.env.LLM_OPENAI_BASE_URL = leakyServerUrl; }); afterAll(async () => { @@ -281,6 +303,64 @@ describe("stealth provider error redaction (routes)", () => { await expectRedactedLog(requestId); }); + test("/v1/chat/completions mid-stream read fault hides the buffered upstream body", async () => { + await setupCreditsApiKey("stealth-token-stream-readfault"); + + const requestId = "stealth-stream-readfault-request"; + const res = await app.request("/v1/chat/completions", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: "Bearer stealth-token-stream-readfault", + "x-no-fallback": "true", + "x-request-id": requestId, + }, + body: JSON.stringify({ + model: STEALTH_MODEL, + stream: true, + messages: [{ role: "user", content: "readfault leak check" }], + }), + }); + + const text = await readSseText(res.body); + // The buffered upstream body is the leak vector here: it is emitted as the + // error event's responseText, so the vendor-branded partial event must not + // survive into the client SSE. + expectNoLeak(text); + expect(text).toContain("event: error"); + expect(text).toContain("Upstream provider error ("); + + const log = await waitForLogByRequestId(requestId); + expect(log.hasError).toBe(true); + expectNoLeak(JSON.stringify(log.errorDetails)); + }); + + test("a non-stealth provider still receives the mid-stream read fault detail", async () => { + await setupCreditsApiKey("nonstealth-token-stream-readfault"); + + const res = await app.request("/v1/chat/completions", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: "Bearer nonstealth-token-stream-readfault", + "x-no-fallback": "true", + "x-request-id": "nonstealth-stream-readfault-request", + }, + body: JSON.stringify({ + model: "openai/gpt-4o-mini", + stream: true, + messages: [{ role: "user", content: "readfault leak check" }], + }), + }); + + // The redaction must stay scoped to stealth providers: everyone else keeps + // the buffered upstream body and the underlying failure detail, which is + // what makes these errors debuggable. + const text = await readSseText(res.body); + expect(text).toContain(TRUNCATED_LEAKY_MARKER); + expect(text).toContain("UND_ERR_SOCKET"); + }); + test("/v1/responses hides the raw upstream error", async () => { await setupCreditsApiKey("stealth-token-responses");