From beaaca55d08db5df5b943af4f8783dbec1f8cb6d Mon Sep 17 00:00:00 2001 From: pacocartones Date: Mon, 3 Aug 2026 07:18:17 +0200 Subject: [PATCH 1/9] fix(gateway): redact stealth provider upstream error bodies in video routes --- .../gateway/src/videos/upstream-error.spec.ts | 44 +++ apps/gateway/src/videos/upstream-error.ts | 21 ++ apps/gateway/src/videos/videos.ts | 263 +++++++++++------- 3 files changed, 225 insertions(+), 103 deletions(-) create mode 100644 apps/gateway/src/videos/upstream-error.spec.ts create mode 100644 apps/gateway/src/videos/upstream-error.ts diff --git a/apps/gateway/src/videos/upstream-error.spec.ts b/apps/gateway/src/videos/upstream-error.spec.ts new file mode 100644 index 0000000000..2f307759f7 --- /dev/null +++ b/apps/gateway/src/videos/upstream-error.spec.ts @@ -0,0 +1,44 @@ +import { afterEach, describe, expect, it } from "vitest"; + +import { redactedProviderErrorText } from "@/lib/stealth-provider-errors.js"; + +import { clientFacingUpstreamMessage } from "./upstream-error.js"; + +const savedStealthBaseUrl = process.env.LLM_AVALANCHE_BASE_URL; + +afterEach(() => { + if (savedStealthBaseUrl === undefined) { + delete process.env.LLM_AVALANCHE_BASE_URL; + } else { + process.env.LLM_AVALANCHE_BASE_URL = savedStealthBaseUrl; + } +}); + +describe("clientFacingUpstreamMessage", () => { + it("redacts the raw upstream error for stealth providers", () => { + process.env.LLM_AVALANCHE_BASE_URL = + "https://plataforma-secreta.example.com"; + + const message = clientFacingUpstreamMessage( + "avalanche", + 500, + "quota exceeded at https://plataforma-secreta.example.com", + ); + + expect(message).toBe(redactedProviderErrorText(500)); + expect(message).not.toContain("plataforma-secreta"); + expect(message).not.toContain("quota"); + }); + + it("passes the raw upstream error through for non-stealth providers", () => { + const raw = "quota exceeded at https://api.openai.com/v1"; + + expect(clientFacingUpstreamMessage("openai", 500, raw)).toBe(raw); + }); + + it("passes the raw upstream error through when the provider is unknown", () => { + const raw = "quota exceeded at https://plataforma-secreta.example.com"; + + expect(clientFacingUpstreamMessage(undefined, 500, raw)).toBe(raw); + }); +}); diff --git a/apps/gateway/src/videos/upstream-error.ts b/apps/gateway/src/videos/upstream-error.ts new file mode 100644 index 0000000000..0d4d57552d --- /dev/null +++ b/apps/gateway/src/videos/upstream-error.ts @@ -0,0 +1,21 @@ +import { + redactedProviderErrorText, + shouldRedactProviderError, +} from "@/lib/stealth-provider-errors.js"; + +/** + * Client-facing message for an upstream video provider error. + * Stealth providers: never pass the raw upstream error body through + * to the client — only the upstream status code may be surfaced. + * Non-stealth providers keep the raw upstream message unchanged. + */ +export function clientFacingUpstreamMessage( + providerId: string | undefined, + statusCode: number, + rawMessage: string, +): string { + if (shouldRedactProviderError(providerId)) { + return redactedProviderErrorText(statusCode); + } + return rawMessage; +} diff --git a/apps/gateway/src/videos/videos.ts b/apps/gateway/src/videos/videos.ts index cf001d98b0..7a40a40efb 100644 --- a/apps/gateway/src/videos/videos.ts +++ b/apps/gateway/src/videos/videos.ts @@ -92,6 +92,8 @@ import { verifyVideoContentAccessToken, } from "@llmgateway/shared/video-access"; +import { clientFacingUpstreamMessage } from "./upstream-error.js"; + import type { ServerTypes } from "@/vars.js"; import type { ResolvedRoutingConfig } from "@llmgateway/shared/routing-config"; import type { Context } from "hono"; @@ -2786,6 +2788,7 @@ function isDebugMode(c: Context): boolean { async function fetchUpstreamJson( url: string, init: RequestInit, + providerId?: string, ): Promise> { // SSRF: never follow redirects on a tenant-baseUrl provider request. const response = await fetch(url, { ...init, redirect: "error" }); @@ -2853,7 +2856,11 @@ async function fetchUpstreamJson( body.error && "message" in body.error && typeof body.error.message === "string" - ? body.error.message + ? clientFacingUpstreamMessage( + providerId, + response.status, + body.error.message, + ) : `Upstream provider error (${response.status})`, }, ); @@ -2866,7 +2873,11 @@ async function fetchUpstreamJson( body, }); throw new HTTPException(upstreamApplicationError.status, { - message: upstreamApplicationError.message, + message: clientFacingUpstreamMessage( + providerId, + upstreamApplicationError.status, + upstreamApplicationError.message, + ), }); } @@ -2976,18 +2987,22 @@ async function createOpenAIVideoJob( referenceImages, ) : JSON.stringify(upstreamRequest); - const rawResponse = await fetchUpstreamJson(upstreamUrl, { - method: "POST", - headers: { - ...getProviderHeaders("openai", providerContext.token, { - requestId: providerContext.requestId, - }), - ...(referenceImages.length === 0 - ? { "Content-Type": "application/json" } - : {}), + const rawResponse = await fetchUpstreamJson( + upstreamUrl, + { + method: "POST", + headers: { + ...getProviderHeaders("openai", providerContext.token, { + requestId: providerContext.requestId, + }), + ...(referenceImages.length === 0 + ? { "Content-Type": "application/json" } + : {}), + }, + body: upstreamBody, }, - body: upstreamBody, - }); + providerContext.providerId, + ); const upstreamResponse = addRequestedVideoMetadata( { ...rawResponse, @@ -3069,16 +3084,20 @@ async function createAvalancheVeoVideoJob( enableFallback: false, ...(imageUrls.length > 0 ? { imageUrls } : {}), }; - const rawResponse = await fetchUpstreamJson(upstreamUrl, { - method: "POST", - headers: { - "Content-Type": "application/json", - ...getProviderHeaders("avalanche", providerContext.token, { - requestId: providerContext.requestId, - }), + const rawResponse = await fetchUpstreamJson( + upstreamUrl, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...getProviderHeaders("avalanche", providerContext.token, { + requestId: providerContext.requestId, + }), + }, + body: JSON.stringify(upstreamRequest), }, - body: JSON.stringify(upstreamRequest), - }); + providerContext.providerId, + ); const upstreamResponse = addRequestedVideoMetadata( { ...rawResponse, @@ -3145,16 +3164,20 @@ async function createAvalancheSoraVideoJob( model: upstreamModelName, input, }; - const rawResponse = await fetchUpstreamJson(upstreamUrl, { - method: "POST", - headers: { - "Content-Type": "application/json", - ...getProviderHeaders("avalanche", providerContext.token, { - requestId: providerContext.requestId, - }), + const rawResponse = await fetchUpstreamJson( + upstreamUrl, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...getProviderHeaders("avalanche", providerContext.token, { + requestId: providerContext.requestId, + }), + }, + body: JSON.stringify(upstreamRequest), }, - body: JSON.stringify(upstreamRequest), - }); + providerContext.providerId, + ); const upstreamResponse = addRequestedVideoMetadata( { ...rawResponse, @@ -3270,15 +3293,21 @@ async function createGoogleVertexVideoJob( ...(outputStorageUri ? { storageUri: outputStorageUri } : {}), }, }; - const rawResponse = await fetchUpstreamJson(authenticatedUpstreamUrl, { - method: "POST", - headers: { - "Content-Type": "application/json", - "x-request-id": providerContext.requestId, - ...(useOAuth ? { Authorization: `Bearer ${providerContext.token}` } : {}), + const rawResponse = await fetchUpstreamJson( + authenticatedUpstreamUrl, + { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-request-id": providerContext.requestId, + ...(useOAuth + ? { Authorization: `Bearer ${providerContext.token}` } + : {}), + }, + body: JSON.stringify(upstreamRequest), }, - body: JSON.stringify(upstreamRequest), - }); + providerContext.providerId, + ); const upstreamId = typeof rawResponse.name === "string" && rawResponse.name.length > 0 ? rawResponse.name @@ -3345,13 +3374,17 @@ async function uploadAtlasCloudMedia( }), `input.${fileExtension}`, ); - const response = await fetchUpstreamJson(uploadUrl, { - method: "POST", - headers: getProviderHeaders("atlascloud", providerContext.token, { - requestId: providerContext.requestId, - }), - body: formData, - }); + const response = await fetchUpstreamJson( + uploadUrl, + { + method: "POST", + headers: getProviderHeaders("atlascloud", providerContext.token, { + requestId: providerContext.requestId, + }), + body: formData, + }, + providerContext.providerId, + ); const uploadedUrl = extractAtlasCloudUploadedMediaUrl(response); if (!uploadedUrl) { @@ -3500,16 +3533,20 @@ async function createAtlasCloudVideoJob( providerContext.baseUrl, "/api/v1/model/generateVideo", ); - const rawResponse = await fetchUpstreamJson(upstreamUrl, { - method: "POST", - headers: { - "Content-Type": "application/json", - ...getProviderHeaders("atlascloud", providerContext.token, { - requestId: providerContext.requestId, - }), + const rawResponse = await fetchUpstreamJson( + upstreamUrl, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...getProviderHeaders("atlascloud", providerContext.token, { + requestId: providerContext.requestId, + }), + }, + body: JSON.stringify(upstreamRequest), }, - body: JSON.stringify(upstreamRequest), - }); + providerContext.providerId, + ); const upstreamResponse = addRequestedVideoMetadata( { @@ -3635,16 +3672,20 @@ async function createBytedanceVideoJob( providerContext.baseUrl, "/contents/generations/tasks", ); - const rawResponse = await fetchUpstreamJson(upstreamUrl, { - method: "POST", - headers: { - "Content-Type": "application/json", - ...getProviderHeaders("bytedance", providerContext.token, { - requestId: providerContext.requestId, - }), + const rawResponse = await fetchUpstreamJson( + upstreamUrl, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...getProviderHeaders("bytedance", providerContext.token, { + requestId: providerContext.requestId, + }), + }, + body: JSON.stringify(upstreamRequest), }, - body: JSON.stringify(upstreamRequest), - }); + providerContext.providerId, + ); const upstreamResponse = addRequestedVideoMetadata( { @@ -3704,16 +3745,20 @@ async function createMinimaxVideoJob( } const upstreamUrl = joinUrl(providerContext.baseUrl, "/v1/video_generation"); - const rawResponse = await fetchUpstreamJson(upstreamUrl, { - method: "POST", - headers: { - "Content-Type": "application/json", - ...getProviderHeaders("minimax", providerContext.token, { - requestId: providerContext.requestId, - }), + const rawResponse = await fetchUpstreamJson( + upstreamUrl, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...getProviderHeaders("minimax", providerContext.token, { + requestId: providerContext.requestId, + }), + }, + body: JSON.stringify(upstreamRequest), }, - body: JSON.stringify(upstreamRequest), - }); + providerContext.providerId, + ); const baseResp = rawResponse.base_resp as { status_code?: number; status_msg?: string } | undefined; @@ -3773,16 +3818,20 @@ async function createXaiVideoJob( providerContext.baseUrl, "/v1/videos/generations", ); - const rawResponse = await fetchUpstreamJson(upstreamUrl, { - method: "POST", - headers: { - "Content-Type": "application/json", - ...getProviderHeaders("xai", providerContext.token, { - requestId: providerContext.requestId, - }), + const rawResponse = await fetchUpstreamJson( + upstreamUrl, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...getProviderHeaders("xai", providerContext.token, { + requestId: providerContext.requestId, + }), + }, + body: JSON.stringify(upstreamRequest), }, - body: JSON.stringify(upstreamRequest), - }); + providerContext.providerId, + ); const upstreamResponse = addRequestedVideoMetadata( { @@ -3831,17 +3880,21 @@ async function createAlibabaVideoJob( providerContext.baseUrl, "/api/v1/services/aigc/video-generation/video-synthesis", ); - const rawResponse = await fetchUpstreamJson(upstreamUrl, { - method: "POST", - headers: { - "Content-Type": "application/json", - "X-DashScope-Async": "enable", - ...getProviderHeaders("alibaba", providerContext.token, { - requestId: providerContext.requestId, - }), + const rawResponse = await fetchUpstreamJson( + upstreamUrl, + { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-DashScope-Async": "enable", + ...getProviderHeaders("alibaba", providerContext.token, { + requestId: providerContext.requestId, + }), + }, + body: JSON.stringify(upstreamRequest), }, - body: JSON.stringify(upstreamRequest), - }); + providerContext.providerId, + ); const output = rawResponse.output && typeof rawResponse.output === "object" @@ -4315,19 +4368,23 @@ async function uploadAvalancheBase64Image( ), "/api/file-base64-upload", ); - const response = await fetchUpstreamJson(uploadUrl, { - method: "POST", - headers: { - "Content-Type": "application/json", - ...getProviderHeaders("avalanche", providerContext.token, { - requestId: providerContext.requestId, + const response = await fetchUpstreamJson( + uploadUrl, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...getProviderHeaders("avalanche", providerContext.token, { + requestId: providerContext.requestId, + }), + }, + body: JSON.stringify({ + base64Data: `data:${image.mimeType};base64,${image.bytesBase64Encoded}`, + uploadPath: "videos/input-images", }), }, - body: JSON.stringify({ - base64Data: `data:${image.mimeType};base64,${image.bytesBase64Encoded}`, - uploadPath: "videos/input-images", - }), - }); + providerContext.providerId, + ); const data = response.data && typeof response.data === "object" ? (response.data as Record) From e76a8d538c0e709a8cae95d4f3654b8c9194fa22 Mon Sep 17 00:00:00 2001 From: pacocartones Date: Mon, 3 Aug 2026 13:31:28 +0200 Subject: [PATCH 2/9] fix(gateway): close video error redaction --- .../src/lib/stealth-provider-errors.spec.ts | 23 ++++++++ .../src/lib/stealth-provider-errors.ts | 14 +++++ .../src/stealth-error-redaction.spec.ts | 33 ++++++++++++ .../src/test-utils/mock-openai-server.ts | 5 +- .../gateway/src/videos/upstream-error.spec.ts | 44 --------------- apps/gateway/src/videos/upstream-error.ts | 21 -------- apps/gateway/src/videos/videos.spec.ts | 54 +++++++++++++++++++ apps/gateway/src/videos/videos.ts | 30 +++++------ apps/worker/src/services/video-jobs.ts | 25 ++++++++- 9 files changed, 166 insertions(+), 83 deletions(-) delete mode 100644 apps/gateway/src/videos/upstream-error.spec.ts delete mode 100644 apps/gateway/src/videos/upstream-error.ts diff --git a/apps/gateway/src/lib/stealth-provider-errors.spec.ts b/apps/gateway/src/lib/stealth-provider-errors.spec.ts index 50ef4e273d..4b676dc7bb 100644 --- a/apps/gateway/src/lib/stealth-provider-errors.spec.ts +++ b/apps/gateway/src/lib/stealth-provider-errors.spec.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest"; import { buildUpstreamErrorClientPayload, canonicalStatusText, + clientFacingUpstreamErrorMessage, clientFacingUpstreamFailureMessage, redactErrorDetails, redactedProviderErrorText, @@ -118,6 +119,28 @@ describe("clientFacingUpstreamFailureMessage", () => { }); }); +describe("clientFacingUpstreamErrorMessage", () => { + it("redacts an HTTP error body for stealth providers", () => { + expect( + clientFacingUpstreamErrorMessage( + "avalanche", + 500, + "quota exceeded at https://plataforma-secreta.example.com", + ), + ).toBe(redactedProviderErrorText(500)); + }); + + it("keeps an HTTP error body for non-stealth providers", () => { + expect( + clientFacingUpstreamErrorMessage( + "openai", + 500, + "quota exceeded at https://api.openai.com/v1", + ), + ).toBe("quota exceeded at https://api.openai.com/v1"); + }); +}); + describe("canonicalStatusText", () => { it("maps known status codes and falls back to empty string", () => { expect(canonicalStatusText(404)).toBe("Not Found"); diff --git a/apps/gateway/src/lib/stealth-provider-errors.ts b/apps/gateway/src/lib/stealth-provider-errors.ts index 3d440ab4ac..75d7227739 100644 --- a/apps/gateway/src/lib/stealth-provider-errors.ts +++ b/apps/gateway/src/lib/stealth-provider-errors.ts @@ -67,6 +67,20 @@ export function clientFacingUpstreamFailureMessage( : `${prefix}: ${errorMessage}`; } +/** + * Client-facing message for an upstream HTTP error. Non-stealth providers + * retain the upstream message; stealth providers expose only the status code. + */ +export function clientFacingUpstreamErrorMessage( + provider: string, + statusCode: number, + rawMessage: string, +): string { + return shouldRedactProviderError(provider) + ? redactedProviderErrorText(statusCode) + : rawMessage; +} + /** * Client-facing `message`/`responseText` pair for an upstream HTTP error. * Non-stealth providers pass the raw upstream body through unchanged. diff --git a/apps/gateway/src/stealth-error-redaction.spec.ts b/apps/gateway/src/stealth-error-redaction.spec.ts index 9312aa95e7..be36a9fe3e 100644 --- a/apps/gateway/src/stealth-error-redaction.spec.ts +++ b/apps/gateway/src/stealth-error-redaction.spec.ts @@ -190,6 +190,39 @@ describe("stealth provider error redaction (routes)", () => { return log; } + test("/v1/videos hides the raw upstream error for a stealth provider", async () => { + await setupCreditsApiKey("stealth-video-token"); + await db.insert(tables.providerKey).values({ + id: "stealth-video-avalanche-key", + token: "stealth-video-provider-key", + provider: "avalanche", + organizationId: "org-id", + baseUrl: leakyServerUrl, + }); + + const res = await app.request("/v1/videos", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: "Bearer stealth-video-token", + "x-no-fallback": "true", + }, + body: JSON.stringify({ + model: "avalanche/veo-3.1-generate-preview", + prompt: "A mountain range at sunrise", + size: "1920x1080", + seconds: 8, + }), + }); + + expect(res.status).toBe(500); + const text = await res.text(); + expectNoLeak(text); + expect(JSON.parse(text).error.message).toBe( + "Upstream provider error (500 Internal Server Error)", + ); + }); + test("/v1/chat/completions non-streaming hides the raw upstream error", async () => { await setupCreditsApiKey("stealth-token-nonstream"); diff --git a/apps/gateway/src/test-utils/mock-openai-server.ts b/apps/gateway/src/test-utils/mock-openai-server.ts index e4caa054a4..ebb2c6cbf6 100644 --- a/apps/gateway/src/test-utils/mock-openai-server.ts +++ b/apps/gateway/src/test-utils/mock-openai-server.ts @@ -2816,7 +2816,10 @@ mockOpenAIServer.get("/api/v1/veo/record-info", async (c) => { return c.json({ code: 200, - msg: "success", + msg: + job.status === "failed" + ? (job.error?.message ?? "Mock video generation failed") + : "success", data: { taskId, successFlag, diff --git a/apps/gateway/src/videos/upstream-error.spec.ts b/apps/gateway/src/videos/upstream-error.spec.ts deleted file mode 100644 index 2f307759f7..0000000000 --- a/apps/gateway/src/videos/upstream-error.spec.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { afterEach, describe, expect, it } from "vitest"; - -import { redactedProviderErrorText } from "@/lib/stealth-provider-errors.js"; - -import { clientFacingUpstreamMessage } from "./upstream-error.js"; - -const savedStealthBaseUrl = process.env.LLM_AVALANCHE_BASE_URL; - -afterEach(() => { - if (savedStealthBaseUrl === undefined) { - delete process.env.LLM_AVALANCHE_BASE_URL; - } else { - process.env.LLM_AVALANCHE_BASE_URL = savedStealthBaseUrl; - } -}); - -describe("clientFacingUpstreamMessage", () => { - it("redacts the raw upstream error for stealth providers", () => { - process.env.LLM_AVALANCHE_BASE_URL = - "https://plataforma-secreta.example.com"; - - const message = clientFacingUpstreamMessage( - "avalanche", - 500, - "quota exceeded at https://plataforma-secreta.example.com", - ); - - expect(message).toBe(redactedProviderErrorText(500)); - expect(message).not.toContain("plataforma-secreta"); - expect(message).not.toContain("quota"); - }); - - it("passes the raw upstream error through for non-stealth providers", () => { - const raw = "quota exceeded at https://api.openai.com/v1"; - - expect(clientFacingUpstreamMessage("openai", 500, raw)).toBe(raw); - }); - - it("passes the raw upstream error through when the provider is unknown", () => { - const raw = "quota exceeded at https://plataforma-secreta.example.com"; - - expect(clientFacingUpstreamMessage(undefined, 500, raw)).toBe(raw); - }); -}); diff --git a/apps/gateway/src/videos/upstream-error.ts b/apps/gateway/src/videos/upstream-error.ts deleted file mode 100644 index 0d4d57552d..0000000000 --- a/apps/gateway/src/videos/upstream-error.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { - redactedProviderErrorText, - shouldRedactProviderError, -} from "@/lib/stealth-provider-errors.js"; - -/** - * Client-facing message for an upstream video provider error. - * Stealth providers: never pass the raw upstream error body through - * to the client — only the upstream status code may be surfaced. - * Non-stealth providers keep the raw upstream message unchanged. - */ -export function clientFacingUpstreamMessage( - providerId: string | undefined, - statusCode: number, - rawMessage: string, -): string { - if (shouldRedactProviderError(providerId)) { - return redactedProviderErrorText(statusCode); - } - return rawMessage; -} diff --git a/apps/gateway/src/videos/videos.spec.ts b/apps/gateway/src/videos/videos.spec.ts index 0d54174ab4..ee24751a39 100644 --- a/apps/gateway/src/videos/videos.spec.ts +++ b/apps/gateway/src/videos/videos.spec.ts @@ -85,6 +85,60 @@ describe("videos", () => { expect(JSON.stringify(json)).toContain("fixed 8s clips"); }); + test("/v1/videos redacts stealth-provider errors persisted by the worker", async () => { + const secret = "SecretVendor error at https://api.secretvendor.com"; + await db.insert(tables.apiKey).values({ + id: "token-id", + token: "real-token", + projectId: "project-id", + description: "Test API Key", + createdBy: "user-id", + }); + await db.insert(tables.providerKey).values({ + id: "provider-key-id", + token: "sk-avalanche-key", + provider: "avalanche", + organizationId: "org-id", + baseUrl: mockServerUrl, + }); + + const createRes = await app.request("/v1/videos", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: "Bearer real-token", + }, + body: JSON.stringify({ + model: "avalanche/veo-3.1-generate-preview", + prompt: "A mountain range at sunrise", + size: "1920x1080", + seconds: 8, + }), + }); + expect(createRes.status).toBe(200); + const created = await createRes.json(); + const job = await db.query.videoJob.findFirst({ + where: { id: { eq: created.id } }, + }); + expect(job).toBeTruthy(); + setMockVideoStatus(job!.upstreamId, "failed", { + error: { message: secret }, + }); + + await processPendingVideoJobs(); + + const statusRes = await app.request(`/v1/videos/${created.id}`, { + headers: { Authorization: "Bearer real-token" }, + }); + expect(statusRes.status).toBe(200); + const responseText = await statusRes.text(); + expect(responseText).not.toContain("SecretVendor"); + expect(responseText).not.toContain("secretvendor.com"); + expect(JSON.parse(responseText).error).toEqual({ + message: "Upstream provider error", + }); + }); + test("/v1/videos rejects dev-plan personal orgs with 403", async () => { await db.insert(tables.apiKey).values({ id: "token-id", diff --git a/apps/gateway/src/videos/videos.ts b/apps/gateway/src/videos/videos.ts index 7a40a40efb..ebf43cc1a8 100644 --- a/apps/gateway/src/videos/videos.ts +++ b/apps/gateway/src/videos/videos.ts @@ -38,6 +38,7 @@ import { validateRequestModelAccess } from "@/lib/iam.js"; import { getProviderMetricsForRouting } from "@/lib/provider-metrics-for-routing.js"; import { getResolvedRoutingConfig } from "@/lib/routing-config-loader.js"; import { getNoFallbackRoutingMetadata } from "@/lib/routing-metadata.js"; +import { clientFacingUpstreamErrorMessage } from "@/lib/stealth-provider-errors.js"; import { getCheapestFromAvailableProviders, @@ -92,8 +93,6 @@ import { verifyVideoContentAccessToken, } from "@llmgateway/shared/video-access"; -import { clientFacingUpstreamMessage } from "./upstream-error.js"; - import type { ServerTypes } from "@/vars.js"; import type { ResolvedRoutingConfig } from "@llmgateway/shared/routing-config"; import type { Context } from "hono"; @@ -2788,7 +2787,7 @@ function isDebugMode(c: Context): boolean { async function fetchUpstreamJson( url: string, init: RequestInit, - providerId?: string, + providerId: string, ): Promise> { // SSRF: never follow redirects on a tenant-baseUrl provider request. const response = await fetch(url, { ...init, redirect: "error" }); @@ -2842,6 +2841,13 @@ async function fetchUpstreamJson( : null; if (!response.ok) { + const rawMessage = + typeof body.error === "object" && + body.error && + "message" in body.error && + typeof body.error.message === "string" + ? body.error.message + : `Upstream provider error (${response.status})`; logger.warn("Upstream video request failed", { url, status: response.status, @@ -2851,17 +2857,11 @@ async function fetchUpstreamJson( response.status as 400 | 401 | 403 | 404 | 409 | 422 | 429 | 500 | 502 | 503 | 504, { - message: - typeof body.error === "object" && - body.error && - "message" in body.error && - typeof body.error.message === "string" - ? clientFacingUpstreamMessage( - providerId, - response.status, - body.error.message, - ) - : `Upstream provider error (${response.status})`, + message: clientFacingUpstreamErrorMessage( + providerId, + response.status, + rawMessage, + ), }, ); } @@ -2873,7 +2873,7 @@ async function fetchUpstreamJson( body, }); throw new HTTPException(upstreamApplicationError.status, { - message: clientFacingUpstreamMessage( + message: clientFacingUpstreamErrorMessage( providerId, upstreamApplicationError.status, upstreamApplicationError.message, diff --git a/apps/worker/src/services/video-jobs.ts b/apps/worker/src/services/video-jobs.ts index 2688974c33..553c373b46 100644 --- a/apps/worker/src/services/video-jobs.ts +++ b/apps/worker/src/services/video-jobs.ts @@ -27,8 +27,10 @@ import { getProviderEnvValue, getProviderEnvVar, getVariantEnvVarName, + isStealthProvider, models, type Provider, + type ProviderId, type ProviderModelMapping, resolveVertexTokenType, type VertexTokenType, @@ -577,6 +579,19 @@ function extractError(body: Record): VideoJobRecord["error"] { }; } +function clientFacingVideoJobError( + providerId: string, + error: VideoJobRecord["error"], +): VideoJobRecord["error"] { + if (!error || !isStealthProvider(providerId as ProviderId)) { + return error; + } + + return { + message: "Upstream provider error", + }; +} + function toUnixTimestamp(value: Date | null): number | null { return value ? Math.floor(value.getTime() / 1000) : null; } @@ -2463,7 +2478,10 @@ export async function processPendingVideoJobs(): Promise { .set({ status, progress, - error: extractError(enrichedUpstreamStatus), + error: clientFacingVideoJobError( + job.usedProvider, + extractError(enrichedUpstreamStatus), + ), contentUrl: extractContentUrl(enrichedUpstreamStatus) ?? job.contentUrl, storageProvider: @@ -2567,7 +2585,10 @@ export async function processPendingVideoJobs(): Promise { .set({ status: "failed", progress: 100, - error: extractError(failedResponse), + error: clientFacingVideoJobError( + job.usedProvider, + extractError(failedResponse), + ), completedAt: now, lastPolledAt: now, nextPollAt: now, From bf3409ab05bf9796b38394a4878187a90bcd02bb Mon Sep 17 00:00:00 2001 From: Paco Cartones <253313177+pacocartones@users.noreply.github.com> Date: Mon, 3 Aug 2026 15:50:17 +0200 Subject: [PATCH 3/9] fix(worker): preserve private video errors without public leaks --- apps/worker/src/services/video-jobs.ts | 46 ++++++++++++++++++++------ 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/apps/worker/src/services/video-jobs.ts b/apps/worker/src/services/video-jobs.ts index 553c373b46..d8e784492e 100644 --- a/apps/worker/src/services/video-jobs.ts +++ b/apps/worker/src/services/video-jobs.ts @@ -52,6 +52,8 @@ import { buildSignedGatewayVideoLogContentUrl } from "@llmgateway/shared/video-a const UPSTREAM_FETCH_TIMEOUT_MS = 30_000; const WEBHOOK_DELIVERY_TIMEOUT_MS = 30_000; +const VIDEO_JOB_PUBLIC_ERROR_STATUS_CODE = 502; +const VIDEO_JOB_PUBLIC_ERROR_STATUS_TEXT = "Bad Gateway"; function fetchWithSignals( url: string, @@ -588,7 +590,7 @@ function clientFacingVideoJobError( } return { - message: "Upstream provider error", + message: `Upstream provider error (${VIDEO_JOB_PUBLIC_ERROR_STATUS_CODE} ${VIDEO_JOB_PUBLIC_ERROR_STATUS_TEXT})`, }; } @@ -1840,10 +1842,35 @@ async function finalizeVideoJob(job: VideoJobRecord): Promise { const responsePayload = await serializeVideoJob(jobToLog, logId); const responseSize = JSON.stringify(responsePayload).length; + const rawVideoError = + jobToLog.upstreamStatusResponse && + typeof jobToLog.upstreamStatusResponse === "object" && + !Array.isArray(jobToLog.upstreamStatusResponse) + ? extractError(jobToLog.upstreamStatusResponse as Record) + : jobToLog.error; + const redactStealthProviderError = isStealthProvider( + jobToLog.usedProvider as ProviderId, + ); + const rawErrorDetails = rawVideoError + ? { + statusCode: VIDEO_JOB_PUBLIC_ERROR_STATUS_CODE, + statusText: jobToLog.status, + responseText: rawVideoError.message, + } + : null; + const publicErrorDetails = + redactStealthProviderError && rawErrorDetails + ? { + statusCode: VIDEO_JOB_PUBLIC_ERROR_STATUS_CODE, + statusText: VIDEO_JOB_PUBLIC_ERROR_STATUS_TEXT, + responseText: `Upstream provider error (${VIDEO_JOB_PUBLIC_ERROR_STATUS_CODE} ${VIDEO_JOB_PUBLIC_ERROR_STATUS_TEXT})`, + } + : rawErrorDetails; + const isContentFilterFailure = jobToLog.status === "failed" && isContentFilterErrorText( - [jobToLog.error?.code, jobToLog.error?.message] + [rawVideoError?.code, rawVideoError?.message] .filter(Boolean) .join(" "), ); @@ -1881,13 +1908,10 @@ async function finalizeVideoJob(job: VideoJobRecord): Promise { ? UnifiedFinishReason.COMPLETED : failureUnifiedFinishReason, hasError: jobToLog.status !== "completed", - errorDetails: jobToLog.error - ? { - statusCode: 502, - statusText: jobToLog.status, - responseText: jobToLog.error.message, - } - : null, + errorDetails: publicErrorDetails, + internalErrorDetails: redactStealthProviderError + ? rawErrorDetails + : undefined, cost: totalCost, requestCost: 0, imageInputCost, @@ -1911,7 +1935,9 @@ async function finalizeVideoJob(job: VideoJobRecord): Promise { jobToLog, "llmgateway_upstream_request", ), - upstreamResponse: jobToLog.upstreamStatusResponse, + upstreamResponse: redactStealthProviderError + ? null + : jobToLog.upstreamStatusResponse, processedAt: null, dataStorageCost: "0", }; From 3f0b6b50d2c16d85b46c78350687746bfd81436b Mon Sep 17 00:00:00 2001 From: Paco Cartones <253313177+pacocartones@users.noreply.github.com> Date: Mon, 3 Aug 2026 15:51:02 +0200 Subject: [PATCH 4/9] test(gateway): cover video stealth log redaction --- apps/gateway/src/videos/videos.spec.ts | 40 ++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/apps/gateway/src/videos/videos.spec.ts b/apps/gateway/src/videos/videos.spec.ts index ee24751a39..16219680b1 100644 --- a/apps/gateway/src/videos/videos.spec.ts +++ b/apps/gateway/src/videos/videos.spec.ts @@ -14,11 +14,17 @@ describe("videos", () => { const harness = createGatewayApiTestHarness(); let mockServerUrl: string; let originalGoogleVertexBaseUrl: string | undefined; + let originalAvalancheApiKey: string | undefined; + let originalAvalancheBaseUrl: string | undefined; beforeAll(() => { mockServerUrl = harness.mockServerUrl; originalGoogleVertexBaseUrl = process.env.LLM_GOOGLE_VERTEX_BASE_URL; + originalAvalancheApiKey = process.env.LLM_AVALANCHE_API_KEY; + originalAvalancheBaseUrl = process.env.LLM_AVALANCHE_BASE_URL; process.env.LLM_GOOGLE_VERTEX_BASE_URL = mockServerUrl; + process.env.LLM_AVALANCHE_API_KEY = "avalanche-env-key"; + process.env.LLM_AVALANCHE_BASE_URL = mockServerUrl; }); afterAll(() => { @@ -27,6 +33,16 @@ describe("videos", () => { } else { delete process.env.LLM_GOOGLE_VERTEX_BASE_URL; } + if (originalAvalancheApiKey !== undefined) { + process.env.LLM_AVALANCHE_API_KEY = originalAvalancheApiKey; + } else { + delete process.env.LLM_AVALANCHE_API_KEY; + } + if (originalAvalancheBaseUrl !== undefined) { + process.env.LLM_AVALANCHE_BASE_URL = originalAvalancheBaseUrl; + } else { + delete process.env.LLM_AVALANCHE_BASE_URL; + } }); async function setRoutingMetrics( @@ -86,7 +102,7 @@ describe("videos", () => { }); test("/v1/videos redacts stealth-provider errors persisted by the worker", async () => { - const secret = "SecretVendor error at https://api.secretvendor.com"; + const secret = "SecretVendor content filter at https://api.secretvendor.com"; await db.insert(tables.apiKey).values({ id: "token-id", token: "real-token", @@ -127,6 +143,26 @@ describe("videos", () => { await processPendingVideoJobs(); + const persistedJob = await db.query.videoJob.findFirst({ + where: { id: { eq: created.id } }, + }); + const log = await db.query.log.findFirst({ + where: { requestId: { eq: job!.requestId } }, + }); + expect(persistedJob).toBeTruthy(); + expect(log).toBeTruthy(); + expect(JSON.stringify(persistedJob!.upstreamStatusResponse)).toContain(secret); + expect(log!.finishReason).toBe("content_filter"); + expect(log!.upstreamResponse).toBeNull(); + expect(log!.internalErrorDetails).toMatchObject({ responseText: secret }); + expect(log!.errorDetails).toEqual({ + statusCode: 502, + statusText: "Bad Gateway", + responseText: "Upstream provider error (502 Bad Gateway)", + }); + expect(JSON.stringify(log)).not.toContain("SecretVendor"); + expect(JSON.stringify(log)).not.toContain("secretvendor.com"); + const statusRes = await app.request(`/v1/videos/${created.id}`, { headers: { Authorization: "Bearer real-token" }, }); @@ -135,7 +171,7 @@ describe("videos", () => { expect(responseText).not.toContain("SecretVendor"); expect(responseText).not.toContain("secretvendor.com"); expect(JSON.parse(responseText).error).toEqual({ - message: "Upstream provider error", + message: "Upstream provider error (502 Bad Gateway)", }); }); From 587e04265434f70e3f97faebdc6f6314598c8df8 Mon Sep 17 00:00:00 2001 From: Paco Cartones <253313177+pacocartones@users.noreply.github.com> Date: Mon, 3 Aug 2026 15:53:19 +0200 Subject: [PATCH 5/9] test(gateway): assert the public video log boundary --- apps/gateway/src/videos/videos.spec.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/gateway/src/videos/videos.spec.ts b/apps/gateway/src/videos/videos.spec.ts index 16219680b1..5a37f1ca68 100644 --- a/apps/gateway/src/videos/videos.spec.ts +++ b/apps/gateway/src/videos/videos.spec.ts @@ -160,8 +160,9 @@ describe("videos", () => { statusText: "Bad Gateway", responseText: "Upstream provider error (502 Bad Gateway)", }); - expect(JSON.stringify(log)).not.toContain("SecretVendor"); - expect(JSON.stringify(log)).not.toContain("secretvendor.com"); + const { internalErrorDetails: _internalErrorDetails, ...publicLog } = log!; + expect(JSON.stringify(publicLog)).not.toContain("SecretVendor"); + expect(JSON.stringify(publicLog)).not.toContain("secretvendor.com"); const statusRes = await app.request(`/v1/videos/${created.id}`, { headers: { Authorization: "Bearer real-token" }, From e2b4d78ac024e9690f32e98687101a6718b5da1c Mon Sep 17 00:00:00 2001 From: Paco Cartones <253313177+pacocartones@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:14:02 +0200 Subject: [PATCH 6/9] fix: retain raw video error for classification --- apps/worker/src/services/video-jobs.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/apps/worker/src/services/video-jobs.ts b/apps/worker/src/services/video-jobs.ts index d8e784492e..24e17f9cc6 100644 --- a/apps/worker/src/services/video-jobs.ts +++ b/apps/worker/src/services/video-jobs.ts @@ -590,7 +590,9 @@ function clientFacingVideoJobError( } return { - message: `Upstream provider error (${VIDEO_JOB_PUBLIC_ERROR_STATUS_CODE} ${VIDEO_JOB_PUBLIC_ERROR_STATUS_TEXT})`, + message: `Upstream provider error (${ + VIDEO_JOB_PUBLIC_ERROR_STATUS_CODE + } ${VIDEO_JOB_PUBLIC_ERROR_STATUS_TEXT})`, }; } @@ -1846,7 +1848,9 @@ async function finalizeVideoJob(job: VideoJobRecord): Promise { jobToLog.upstreamStatusResponse && typeof jobToLog.upstreamStatusResponse === "object" && !Array.isArray(jobToLog.upstreamStatusResponse) - ? extractError(jobToLog.upstreamStatusResponse as Record) + ? extractError( + jobToLog.upstreamStatusResponse as Record, + ) : jobToLog.error; const redactStealthProviderError = isStealthProvider( jobToLog.usedProvider as ProviderId, @@ -1863,7 +1867,9 @@ async function finalizeVideoJob(job: VideoJobRecord): Promise { ? { statusCode: VIDEO_JOB_PUBLIC_ERROR_STATUS_CODE, statusText: VIDEO_JOB_PUBLIC_ERROR_STATUS_TEXT, - responseText: `Upstream provider error (${VIDEO_JOB_PUBLIC_ERROR_STATUS_CODE} ${VIDEO_JOB_PUBLIC_ERROR_STATUS_TEXT})`, + responseText: `Upstream provider error (${ + VIDEO_JOB_PUBLIC_ERROR_STATUS_CODE + } ${VIDEO_JOB_PUBLIC_ERROR_STATUS_TEXT})`, } : rawErrorDetails; From 4937cbe62026440a49d82927d4e0a397ba7fa6a1 Mon Sep 17 00:00:00 2001 From: Paco Cartones <253313177+pacocartones@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:14:04 +0200 Subject: [PATCH 7/9] test: cover stealth video content-filter classification --- apps/gateway/src/videos/videos.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/gateway/src/videos/videos.spec.ts b/apps/gateway/src/videos/videos.spec.ts index 5a37f1ca68..9eea205f00 100644 --- a/apps/gateway/src/videos/videos.spec.ts +++ b/apps/gateway/src/videos/videos.spec.ts @@ -102,7 +102,7 @@ describe("videos", () => { }); test("/v1/videos redacts stealth-provider errors persisted by the worker", async () => { - const secret = "SecretVendor content filter at https://api.secretvendor.com"; + const secret = "SecretVendor SensitiveContentDetected at https://api.secretvendor.com"; await db.insert(tables.apiKey).values({ id: "token-id", token: "real-token", From b37ed91193ff67c14bfaa5c87a551c0d6c98e188 Mon Sep 17 00:00:00 2001 From: Paco Cartones <253313177+pacocartones@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:14:05 +0200 Subject: [PATCH 8/9] test: route stealth video error through mock provider --- apps/gateway/src/stealth-error-redaction.spec.ts | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/apps/gateway/src/stealth-error-redaction.spec.ts b/apps/gateway/src/stealth-error-redaction.spec.ts index be36a9fe3e..6a06cc384b 100644 --- a/apps/gateway/src/stealth-error-redaction.spec.ts +++ b/apps/gateway/src/stealth-error-redaction.spec.ts @@ -124,6 +124,8 @@ describe("stealth provider error redaction (routes)", () => { "LLM_GLACIER_BASE_URL", "LLM_OPENAI_API_KEY", "LLM_OPENAI_BASE_URL", + "LLM_AVALANCHE_API_KEY", + "LLM_AVALANCHE_BASE_URL", ]) { savedEnv[key] = process.env[key]; } @@ -134,6 +136,8 @@ describe("stealth provider error redaction (routes)", () => { // openai is the non-stealth control: same leaky mock, no redaction. process.env.LLM_OPENAI_API_KEY = "openai-env-key"; process.env.LLM_OPENAI_BASE_URL = leakyServerUrl; + process.env.LLM_AVALANCHE_API_KEY = "avalanche-env-key"; + process.env.LLM_AVALANCHE_BASE_URL = leakyServerUrl; }); afterAll(async () => { @@ -192,14 +196,6 @@ describe("stealth provider error redaction (routes)", () => { test("/v1/videos hides the raw upstream error for a stealth provider", async () => { await setupCreditsApiKey("stealth-video-token"); - await db.insert(tables.providerKey).values({ - id: "stealth-video-avalanche-key", - token: "stealth-video-provider-key", - provider: "avalanche", - organizationId: "org-id", - baseUrl: leakyServerUrl, - }); - const res = await app.request("/v1/videos", { method: "POST", headers: { From 2a231b065dd4a84a6119fd976a361ba8c4cd0ff6 Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Mon, 3 Aug 2026 17:32:01 +0100 Subject: [PATCH 9/9] style(worker): tidy video redaction constants Run pnpm format over the changed files and hoist the redacted error text into a single constant, with a note on why the public status code is a hard-coded 502. Co-Authored-By: Claude Opus 5 (1M context) --- apps/gateway/src/videos/videos.spec.ts | 7 +++++-- apps/worker/src/services/video-jobs.ts | 17 +++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/apps/gateway/src/videos/videos.spec.ts b/apps/gateway/src/videos/videos.spec.ts index 9eea205f00..94bde11830 100644 --- a/apps/gateway/src/videos/videos.spec.ts +++ b/apps/gateway/src/videos/videos.spec.ts @@ -102,7 +102,8 @@ describe("videos", () => { }); test("/v1/videos redacts stealth-provider errors persisted by the worker", async () => { - const secret = "SecretVendor SensitiveContentDetected at https://api.secretvendor.com"; + const secret = + "SecretVendor SensitiveContentDetected at https://api.secretvendor.com"; await db.insert(tables.apiKey).values({ id: "token-id", token: "real-token", @@ -151,7 +152,9 @@ describe("videos", () => { }); expect(persistedJob).toBeTruthy(); expect(log).toBeTruthy(); - expect(JSON.stringify(persistedJob!.upstreamStatusResponse)).toContain(secret); + expect(JSON.stringify(persistedJob!.upstreamStatusResponse)).toContain( + secret, + ); expect(log!.finishReason).toBe("content_filter"); expect(log!.upstreamResponse).toBeNull(); expect(log!.internalErrorDetails).toMatchObject({ responseText: secret }); diff --git a/apps/worker/src/services/video-jobs.ts b/apps/worker/src/services/video-jobs.ts index 24e17f9cc6..136a5fcbfa 100644 --- a/apps/worker/src/services/video-jobs.ts +++ b/apps/worker/src/services/video-jobs.ts @@ -52,8 +52,13 @@ import { buildSignedGatewayVideoLogContentUrl } from "@llmgateway/shared/video-a const UPSTREAM_FETCH_TIMEOUT_MS = 30_000; const WEBHOOK_DELIVERY_TIMEOUT_MS = 30_000; +// A failed video job is reported by an upstream status poll that itself +// returned 200, so there is no real upstream status code to surface. 502 is a +// stand-in that matches the status this path has always stamped on the log +// row's errorDetails — do not "fix" it to the poll's actual status. const VIDEO_JOB_PUBLIC_ERROR_STATUS_CODE = 502; const VIDEO_JOB_PUBLIC_ERROR_STATUS_TEXT = "Bad Gateway"; +const VIDEO_JOB_PUBLIC_ERROR_TEXT = `Upstream provider error (${VIDEO_JOB_PUBLIC_ERROR_STATUS_CODE} ${VIDEO_JOB_PUBLIC_ERROR_STATUS_TEXT})`; function fetchWithSignals( url: string, @@ -590,9 +595,7 @@ function clientFacingVideoJobError( } return { - message: `Upstream provider error (${ - VIDEO_JOB_PUBLIC_ERROR_STATUS_CODE - } ${VIDEO_JOB_PUBLIC_ERROR_STATUS_TEXT})`, + message: VIDEO_JOB_PUBLIC_ERROR_TEXT, }; } @@ -1849,8 +1852,8 @@ async function finalizeVideoJob(job: VideoJobRecord): Promise { typeof jobToLog.upstreamStatusResponse === "object" && !Array.isArray(jobToLog.upstreamStatusResponse) ? extractError( - jobToLog.upstreamStatusResponse as Record, - ) + jobToLog.upstreamStatusResponse as Record, + ) : jobToLog.error; const redactStealthProviderError = isStealthProvider( jobToLog.usedProvider as ProviderId, @@ -1867,9 +1870,7 @@ async function finalizeVideoJob(job: VideoJobRecord): Promise { ? { statusCode: VIDEO_JOB_PUBLIC_ERROR_STATUS_CODE, statusText: VIDEO_JOB_PUBLIC_ERROR_STATUS_TEXT, - responseText: `Upstream provider error (${ - VIDEO_JOB_PUBLIC_ERROR_STATUS_CODE - } ${VIDEO_JOB_PUBLIC_ERROR_STATUS_TEXT})`, + responseText: VIDEO_JOB_PUBLIC_ERROR_TEXT, } : rawErrorDetails;