fix(gateway): don't bill upstream failures

apps/gateway/src/api.spec.ts

@@ -8443,6 +8443,191 @@ describe("api", () => {
 		});
 	});
 
+	describe("upstream failure billing", () => {
+		// A request the gateway records as an upstream/gateway failure hands the
+		// caller an error, so it is not billed for whatever the provider emitted
+		// before dying — even though the tokens are still recorded for analytics.
+		function spyUpstreamResponse(
+			matchUrlFragment: string,
+			body: string,
+			contentType: string,
+		): ReturnType<typeof vi.spyOn> {
+			const originalFetch = globalThis.fetch;
+			return vi
+				.spyOn(globalThis, "fetch")
+				.mockImplementation(async (input, init) => {
+					const url =
+						typeof input === "string"
+							? input
+							: input instanceof URL
+								? input.toString()
+								: input.url;
+
+					if (url.includes(matchUrlFragment)) {
+						const stream = new ReadableStream({
+							start(controller) {
+								controller.enqueue(new TextEncoder().encode(body));
+								controller.close();
+							},
+						});
+						return new Response(stream, {
+							status: 200,
+							headers: { "Content-Type": contentType },
+						});
+					}
+
+					return await originalFetch(input as RequestInfo | URL, init);
+				});
+		}
+
+		test("stream truncated after partial output is not billed", async () => {
+			await db.insert(tables.apiKey).values({
+				id: "token-id",
+				token: "real-token",
+				projectId: "project-id",
+				description: "Test API Key",
+				createdBy: "user-id",
+			});
+			await db.insert(tables.providerKey).values({
+				id: "provider-key-id",
+				token: "sk-test-key",
+				provider: "anthropic",
+				organizationId: "org-id",
+				baseUrl: mockServerUrl,
+			});
+
+			// Partial output, then the upstream drops the connection without ever
+			// sending a terminal event — the shape of a mid-stream provider failure.
+			const sse = [
+				`event: message_start\ndata: ${JSON.stringify({
+					type: "message_start",
+					message: {
+						id: "msg_truncated",
+						type: "message",
+						role: "assistant",
+						model: "claude-opus-4-8",
+						content: [],
+						usage: { input_tokens: 100, output_tokens: 0 },
+					},
+				})}\n\n`,
+				`event: content_block_start\ndata: ${JSON.stringify({
+					type: "content_block_start",
+					index: 0,
+					content_block: { type: "text", text: "" },
+				})}\n\n`,
+				`event: content_block_delta\ndata: ${JSON.stringify({
+					type: "content_block_delta",
+					index: 0,
+					delta: { type: "text_delta", text: "Here is the start of an answer" },
+				})}\n\n`,
+			].join("");
+
+			const fetchSpy = spyUpstreamResponse(
+				`${mockServerUrl}/v1/messages`,
+				sse,
+				"text/event-stream",
+			);
+
+			try {
+				const res = await app.request("/v1/chat/completions", {
+					method: "POST",
+					headers: {
+						"Content-Type": "application/json",
+						Authorization: "Bearer real-token",
+						"x-no-fallback": "true",
+					},
+					body: JSON.stringify({
+						model: "anthropic/claude-opus-4-8",
+						messages: [{ role: "user", content: "Truncate the stream" }],
+						stream: true,
+					}),
+				});
+
+				expect(res.status).toBe(200);
+				const streamResult = await readAll(res.body);
+				expect(streamResult.hasError).toBe(true);
+				expect(streamResult.errorEvents[0].error.type).toBe("upstream_error");
+			} finally {
+				fetchSpy.mockRestore();
+			}
+
+			const logs = await waitForLogs(1);
+			expect(logs.length).toBe(1);
+			expect(logs[0].finishReason).toBe("upstream_error");
+			expect(logs[0].hasError).toBe(true);
+			// The caller got an error, so nothing is charged.
+			expect(Number(logs[0].cost)).toBe(0);
+			expect(Number(logs[0].inputCost)).toBe(0);
+			expect(Number(logs[0].outputCost)).toBe(0);
+			// Tokens are still recorded for analytics.
+			expect(Number(logs[0].promptTokens)).toBe(100);
+		});
+
+		test("empty non-streaming response is not billed", async () => {
+			await db.insert(tables.apiKey).values({
+				id: "token-id",
+				token: "real-token",
+				projectId: "project-id",
+				description: "Test API Key",
+				createdBy: "user-id",
+			});
+			await db.insert(tables.providerKey).values({
+				id: "provider-key-id",
+				token: "sk-test-key",
+				provider: "anthropic",
+				organizationId: "org-id",
+				baseUrl: mockServerUrl,
+			});
+
+			const fetchSpy = spyUpstreamResponse(
+				`${mockServerUrl}/v1/messages`,
+				JSON.stringify({
+					id: "msg_empty",
+					type: "message",
+					role: "assistant",
+					model: "claude-opus-4-8",
+					content: [],
+					stop_reason: "end_turn",
+					stop_sequence: null,
+					usage: { input_tokens: 100, output_tokens: 0 },
+				}),
+				"application/json",
+			);
+
+			let json: { usage?: { cost?: number } };
+			try {
+				const res = await app.request("/v1/chat/completions", {
+					method: "POST",
+					headers: {
+						"Content-Type": "application/json",
+						Authorization: "Bearer real-token",
+						"x-no-fallback": "true",
+					},
+					body: JSON.stringify({
+						model: "anthropic/claude-opus-4-8",
+						messages: [{ role: "user", content: "Return nothing" }],
+					}),
+				});
+
+				expect(res.status).toBe(200);
+				json = await res.json();
+			} finally {
+				fetchSpy.mockRestore();
+			}
+
+			// The cost echoed to the client matches the zeroed charge.
+			expect(json.usage?.cost).toBe(0);
+
+			const logs = await waitForLogs(1);
+			expect(logs.length).toBe(1);
+			expect(logs[0].finishReason).toBe("upstream_error");
+			expect(logs[0].hasError).toBe(true);
+			expect(Number(logs[0].cost)).toBe(0);
+			expect(Number(logs[0].inputCost)).toBe(0);
+			expect(Number(logs[0].promptTokens)).toBe(100);
+		});
+	});
+
 	describe("native /v1/messages server-side tools", () => {
 		// Anthropic server-side tools (e.g. web_search_20250305) carry a versioned
 		// `type` and no `description`/`input_schema`. They must pass validation and

apps/gateway/src/chat/chat.ts

@@ -49,6 +49,7 @@ import {
 } from "@/lib/compliance.js";
 import {
 	calculateCosts,
+	isBilledFailureFinishReason,
 	isRefusalFinishReason,
 	shouldBillCancelledRequests,
 	zeroInferenceCosts,
@@ -11162,6 +11163,19 @@ chat.openapi(completions, async (c) => {
 									finishReason === "content_filter",
 								));
 
+					// A stream that ended in a gateway- or upstream-side failure returns
+					// an error to the caller, so it is not billed for the tokens the
+					// provider emitted before dying. Cancellations are excluded so
+					// aborting a stream stays billable and cannot be used as a billing
+					// bypass (GHSA-724j-f2pf-phf7).
+					if (
+						!canceled &&
+						streamingError !== null &&
+						!isBilledFailureFinishReason(finishReason)
+					) {
+						zeroInferenceCosts(costs);
+					}
+
 					// Use costs.promptTokens as canonical value (includes image input
 					// tokens for providers that exclude them from upstream usage)
 					if (costs.promptTokens !== null && costs.promptTokens !== undefined) {
@@ -13420,6 +13434,40 @@ chat.openapi(completions, async (c) => {
 		zeroInferenceCosts(costs);
 	}
 
+	// Check if the non-streaming response is empty (no content, tokens, or tool
+	// calls). Exclude content filter responses as they are intentionally empty.
+	const isContentFilterResponse = isContentFilterFinishReason(
+		finishReason,
+		usedProvider,
+	);
+	// A length-limit finish reason (e.g. a tiny `max_tokens`) can legitimately
+	// produce no content at all, so an empty response in that case is expected
+	// behavior rather than an upstream error.
+	const isLengthLimitResponse = isLengthLimitFinishReason(
+		finishReason,
+		usedProvider,
+	);
+	const hasEmptyNonStreamingResponse =
+		!!finishReason &&
+		finishReason !== "incomplete" &&
+		!isContentFilterResponse &&
+		!isLengthLimitResponse &&
+		!hasMeaningfulAssistantOutput({
+			completionTokens: calculatedCompletionTokens,
+			reasoningTokens: calculatedReasoningTokens,
+			content,
+			toolResults,
+			images: convertedImages,
+		});
+
+	// An empty response is recorded as an upstream error and hands the caller
+	// nothing usable, so it is not billed. Computed before
+	// transformResponseToOpenai so the cost echoed back to the client matches
+	// what the log is charged.
+	if (hasEmptyNonStreamingResponse) {
+		zeroInferenceCosts(costs);
+	}
+
 	costs.dataStorageCost = toDataStorageCostNumber(
 		costs.promptTokens ?? calculatedPromptTokens,
 		cachedTokens,
@@ -13596,32 +13644,6 @@ chat.openapi(completions, async (c) => {
 		Object.keys(pluginResults).length > 0 ? pluginResults : undefined,
 	);
 
-	// Check if the non-streaming response is empty (no content, tokens, or tool calls)
-	// Exclude content filter responses as they are intentionally empty.
-	const isContentFilterResponse = isContentFilterFinishReason(
-		finishReason,
-		usedProvider,
-	);
-	// A length-limit finish reason (e.g. a tiny `max_tokens`) can legitimately
-	// produce no content at all, so an empty response in that case is expected
-	// behavior rather than an upstream error.
-	const isLengthLimitResponse = isLengthLimitFinishReason(
-		finishReason,
-		usedProvider,
-	);
-	const hasEmptyNonStreamingResponse =
-		!!finishReason &&
-		finishReason !== "incomplete" &&
-		!isContentFilterResponse &&
-		!isLengthLimitResponse &&
-		!hasMeaningfulAssistantOutput({
-			completionTokens: calculatedCompletionTokens,
-			reasoningTokens: calculatedReasoningTokens,
-			content,
-			toolResults,
-			images: convertedImages,
-		});
-
 	if (hasEmptyNonStreamingResponse) {
 		logger.debug("Empty non-streaming response detected", {
 			finishReason,

apps/gateway/src/lib/costs.spec.ts

@@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 
 import {
 	calculateCosts,
+	isBilledFailureFinishReason,
 	isRefusalFinishReason,
 	shouldBillCancelledRequests,
 	zeroInferenceCosts,
@@ -1982,6 +1983,21 @@ describe("zeroInferenceCosts", () => {
 	});
 });
 
+describe("isBilledFailureFinishReason", () => {
+	it("keeps client errors and content filters billable", () => {
+		expect(isBilledFailureFinishReason("client_error")).toBe(true);
+		expect(isBilledFailureFinishReason("content_filter")).toBe(true);
+	});
+
+	it("is false for gateway- and upstream-side failures", () => {
+		expect(isBilledFailureFinishReason("upstream_error")).toBe(false);
+		expect(isBilledFailureFinishReason("gateway_error")).toBe(false);
+		// A timeout or buffer overflow aborts before any finish reason is known.
+		expect(isBilledFailureFinishReason(null)).toBe(false);
+		expect(isBilledFailureFinishReason(undefined)).toBe(false);
+	});
+});
+
 describe("shouldBillCancelledRequests", () => {
 	const original = process.env.BILL_CANCELLED_REQUESTS;
 

apps/gateway/src/lib/costs.ts

@@ -109,6 +109,30 @@ export function zeroInferenceCosts(costs: MutableInferenceCosts): void {
 	costs.totalCost = 0;
 }
 
+/**
+ * True when a request that ended in a failure is still billed for the inference
+ * it consumed.
+ *
+ * A request the gateway records as its own or the provider's fault
+ * (`upstream_error`, `gateway_error`, an upstream read fault, a timeout, or a
+ * stream that died before any terminal event) hands the caller an error, so
+ * charging for whatever tokens the provider happened to emit would bill the
+ * caller for a failure that is not theirs. Two failure classes stay billable:
+ *
+ * - `client_error` — the caller's own malformed request still consumed upstream
+ *   inference, so it must not become a free-inference path.
+ * - `content_filter` — a safety block is a served response and is billed on
+ *   purpose.
+ *
+ * Cancellations are a separate axis and keep billing by default — see
+ * {@link shouldBillCancelledRequests}.
+ */
+export function isBilledFailureFinishReason(
+	finishReason: string | null | undefined,
+): boolean {
+	return finishReason === "client_error" || finishReason === "content_filter";
+}
+
 /**
  * Check if billing for cancelled requests is enabled via environment variable.
  * Defaults to true if not set: a cancelled streaming request has already