feat(gateway): add OpenAI realtime API support

.env.example

@@ -37,6 +37,13 @@ ADMIN_PORT=3006
 # Health check timeout in milliseconds (defaults to 5000ms)
 TIMEOUT_MS=5000
 
+# Host the /v1/realtime WebSocket proxy inside the main gateway process instead
+# of the dedicated realtime deployment. Set to "true" for local dev / single
+# process setups so `pnpm dev` serves realtime sessions on the gateway port with
+# no extra command. Leave unset in production, where a separate realtime
+# deployment scales long-lived sessions independently. Defaults to disabled.
+# REALTIME_INLINE=true
+
 # HTTP server keep-alive timeout in seconds. Should be higher than load balancer's
 # keepalive timeout to prevent 502 errors. GCP LB uses 600s.
 # Defaults: gateway=620s, api=60s

.gitignore

@@ -4,6 +4,7 @@ dist/
 .env
 packages/**/*.js
 apps/**/*.js
+!apps/*/public/**/*.js
 *spec.yaml
 .turbo/
 .pnpm-store/

apps/api/src/routes/activity.ts

@@ -72,6 +72,7 @@ const dailyActivitySchema = z.object({
 	dataStorageCost: z.number(),
 	imageInputCost: z.number(),
 	audioInputCost: z.number(),
+	audioOutputCost: z.number(),
 	imageOutputCost: z.number(),
 	videoOutputCost: z.number(),
 	cachedInputCost: z.number(),
@@ -109,6 +110,7 @@ function buildEmptyActivityRow(date: string): ActivityRow {
 		dataStorageCost: 0,
 		imageInputCost: 0,
 		audioInputCost: 0,
+		audioOutputCost: 0,
 		imageOutputCost: 0,
 		videoOutputCost: 0,
 		cachedInputCost: 0,
@@ -331,6 +333,10 @@ activity.openapi(getActivity, async (c) => {
 					sql<number>`COALESCE(SUM(${apiKeyHourlyStats.audioInputCost}), 0)`.as(
 						"audioInputCost",
 					),
+				audioOutputCost:
+					sql<number>`COALESCE(SUM(${apiKeyHourlyStats.audioOutputCost}), 0)`.as(
+						"audioOutputCost",
+					),
 				imageOutputCost:
 					sql<number>`COALESCE(SUM(${apiKeyHourlyStats.imageOutputCost}), 0)`.as(
 						"imageOutputCost",
@@ -473,6 +479,7 @@ activity.openapi(getActivity, async (c) => {
 			const discountSavings = Number(day.discountSavings);
 			const imageInputCost = Number(day.imageInputCost);
 			const audioInputCost = Number(day.audioInputCost);
+			const audioOutputCost = Number(day.audioOutputCost);
 			const imageOutputCost = Number(day.imageOutputCost);
 			const videoOutputCost = Number(day.videoOutputCost);
 			const cachedInputCost = Number(day.cachedInputCost);
@@ -505,6 +512,7 @@ activity.openapi(getActivity, async (c) => {
 				dataStorageCost,
 				imageInputCost,
 				audioInputCost,
+				audioOutputCost,
 				imageOutputCost,
 				videoOutputCost,
 				cachedInputCost,
@@ -594,6 +602,10 @@ activity.openapi(getActivity, async (c) => {
 				sql<number>`COALESCE(SUM(${projectHourlyStats.audioInputCost}), 0)`.as(
 					"audioInputCost",
 				),
+			audioOutputCost:
+				sql<number>`COALESCE(SUM(${projectHourlyStats.audioOutputCost}), 0)`.as(
+					"audioOutputCost",
+				),
 			imageOutputCost:
 				sql<number>`COALESCE(SUM(${projectHourlyStats.imageOutputCost}), 0)`.as(
 					"imageOutputCost",
@@ -804,6 +816,7 @@ activity.openapi(getActivity, async (c) => {
 		const dataStorageCost = Number(day.dataStorageCost);
 		const imageInputCost = Number(day.imageInputCost);
 		const audioInputCost = Number(day.audioInputCost);
+		const audioOutputCost = Number(day.audioOutputCost);
 		const imageOutputCost = Number(day.imageOutputCost);
 		const videoOutputCost = Number(day.videoOutputCost);
 		const cachedInputCost = Number(day.cachedInputCost);
@@ -837,6 +850,7 @@ activity.openapi(getActivity, async (c) => {
 			dataStorageCost,
 			imageInputCost,
 			audioInputCost,
+			audioOutputCost,
 			imageOutputCost,
 			videoOutputCost,
 			cachedInputCost,

apps/api/src/routes/internal-models.ts

@@ -68,6 +68,8 @@ const modelProviderMappingSchema = z.object({
 	imageInputTokensByResolution: z.record(z.number()).nullable(),
 	imageOutputTokensByResolution: z.record(z.number()).nullable(),
 	inputCharacterPrice: z.string().nullable(),
+	inputAudioPrice: z.string().nullable(),
+	cachedInputAudioPrice: z.string().nullable(),
 	outputAudioPrice: z.string().nullable(),
 	requestPrice: z.string().nullable(),
 	contextSize: z.number().nullable(),
@@ -86,6 +88,8 @@ const modelProviderMappingSchema = z.object({
 	jsonOutputSchema: z.boolean().nullable(),
 	webSearch: z.boolean().nullable(),
 	webSearchPrice: z.string().nullable(),
+	realtime: z.boolean().nullable(),
+	supportedVoices: z.array(z.string()).nullable(),
 	discount: z.string().nullable(),
 	stability: z.enum(["stable", "beta", "unstable", "experimental"]).nullable(),
 	supportedParameters: z.array(z.string()).nullable(),
@@ -242,6 +246,8 @@ internalModels.openapi(getModelsRoute, async (c) => {
 				reasoningEfforts: sharedMapping?.reasoningEfforts ?? null,
 				audio: sharedMapping?.audio ?? null,
 				document: sharedMapping?.document ?? null,
+				realtime: sharedMapping?.realtime ?? null,
+				supportedVoices: sharedMapping?.supportedVoices ?? null,
 				imageOutputPrice:
 					sharedMapping?.imageOutputPrice !== undefined
 						? String(sharedMapping.imageOutputPrice)
@@ -254,6 +260,14 @@ internalModels.openapi(getModelsRoute, async (c) => {
 					sharedMapping?.inputCharacterPrice !== undefined
 						? String(sharedMapping.inputCharacterPrice)
 						: null,
+				inputAudioPrice:
+					sharedMapping?.inputAudioPrice !== undefined
+						? String(sharedMapping.inputAudioPrice)
+						: null,
+				cachedInputAudioPrice:
+					sharedMapping?.cachedInputAudioPrice !== undefined
+						? String(sharedMapping.cachedInputAudioPrice)
+						: null,
 				outputAudioPrice:
 					sharedMapping?.outputAudioPrice !== undefined
 						? String(sharedMapping.outputAudioPrice)

apps/api/src/testing.ts

@@ -177,6 +177,14 @@ function getCommonAggregationFields() {
 			sql<number>`coalesce(sum(${tables.log.imageOutputCost}), 0)`.as(
 				"imageOutputCost",
 			),
+		audioInputCost:
+			sql<number>`coalesce(sum(${tables.log.audioInputCost}), 0)`.as(
+				"audioInputCost",
+			),
+		audioOutputCost:
+			sql<number>`coalesce(sum(${tables.log.audioOutputCost}), 0)`.as(
+				"audioOutputCost",
+			),
 		videoOutputCost:
 			sql<number>`coalesce(sum(${tables.log.videoOutputCost}), 0)`.as(
 				"videoOutputCost",

apps/gateway/package.json

@@ -15,12 +15,13 @@
 	],
 	"scripts": {
 		"build": "tsc && resolve-tspaths && pnpm run generate",
-		"dev": "concurrently \"tsc-watch --onSuccess 'resolve-tspaths'\" \"nodemon --exec node --enable-source-maps --env-file=../../.env dist/serve.js\"",
+		"dev": "concurrently \"tsc-watch --onSuccess 'resolve-tspaths'\" \"REALTIME_INLINE=true nodemon --exec node --enable-source-maps --env-file=../../.env dist/serve.js\"",
 		"exec-dev": "resolve-tspaths && node dist/serve.js",
 		"format": "eslint --fix . && prettier --write .",
 		"generate": "tsx ./src/scripts/generate-openapi.ts",
 		"lint": "eslint . && prettier --check .",
-		"start": "node --enable-source-maps dist/serve.js"
+		"start": "node --enable-source-maps dist/serve.js",
+		"start:realtime": "node --enable-source-maps dist/realtime/serve.js"
 	},
 	"dependencies": {
 		"@hono/node-server": "^1.19.13",
@@ -46,7 +47,11 @@
 		"ipaddr.js": "2.2.0",
 		"redis": "5.9.0",
 		"worker": "workspace:*",
+		"ws": "8.18.3",
 		"zod": "3.25.75",
 		"zod-openapi": "^5.4.3"
+	},
+	"devDependencies": {
+		"@types/ws": "8.18.1"
 	}
 }

apps/gateway/src/app.ts

@@ -38,6 +38,7 @@ import { tracingMiddleware } from "./middleware/tracing.js";
 import { models } from "./models/route.js";
 import { moderationsRoute } from "./moderations/route.js";
 import { ocrRoute } from "./ocr/route.js";
+import { realtimeClientSecretsRoute } from "./realtime/client-secrets-route.js";
 import { responses } from "./responses/responses.js";
 import { speechRoute } from "./speech/route.js";
 import { videosRoute } from "./videos/route.js";
@@ -349,6 +350,7 @@ v1.route("/ocr", ocrRoute);
 v1.route("/messages", anthropic);
 v1.route("/responses", responses);
 v1.route("/audio/speech", speechRoute);
+v1.route("/realtime", realtimeClientSecretsRoute);
 v1.route("/videos", videosRoute);
 
 app.route("/v1", v1);

apps/gateway/src/chat/chat.ts

@@ -1740,6 +1740,18 @@ chat.openapi(completions, async (c) => {
 	// chat-completions image flow.
 	validateModelOutput(modelInfo, requestedModel, ["text", "image"]);
 
+	// Realtime models declare text output but are only served over the
+	// dedicated WebSocket endpoint, so the output-based gate above doesn't
+	// catch them. Reject them here with the correct endpoint pointer.
+	if (
+		modelInfo.providers.length > 0 &&
+		modelInfo.providers.every((p) => p.realtime === true)
+	) {
+		throw new HTTPException(400, {
+			message: `Model ${requestedModel} is a realtime model and cannot be used with /v1/chat/completions. Connect to the /v1/realtime WebSocket endpoint instead.`,
+		});
+	}
+
 	// Validate that models requiring image input have at least one image in the request
 	if (
 		modelInfo.imageInputRequired &&

apps/gateway/src/lib/client-ip.ts

@@ -5,8 +5,9 @@ import type { Context } from "hono";
 // Extract the originating client IP from request headers. Mirrors the
 // ordering used elsewhere in the codebase (Cloudflare first, then the first
 // hop of X-Forwarded-For as set by the GCP load balancer, then X-Real-IP).
-export function getClientIpFromRequest(c: Context): string | undefined {
-	const xff = c.req.header("x-forwarded-for");
+export function getClientIpFromForwardedFor(
+	xff: string | undefined,
+): string | undefined {
 	if (xff) {
 		const first = xff.split(",")[0]?.trim();
 		if (first) {
@@ -16,6 +17,10 @@ export function getClientIpFromRequest(c: Context): string | undefined {
 	return undefined;
 }
 
+export function getClientIpFromRequest(c: Context): string | undefined {
+	return getClientIpFromForwardedFor(c.req.header("x-forwarded-for"));
+}
+
 // Normalize IPv4-mapped IPv6 (::ffff:1.2.3.4) to plain IPv4 so an IPv4 CIDR
 // matches a request that arrived with an IPv4-mapped IPv6 source.
 function normalize(addr: ipaddr.IPv4 | ipaddr.IPv6): ipaddr.IPv4 | ipaddr.IPv6 {

apps/gateway/src/models/models.ts

@@ -21,7 +21,9 @@ const modelSchema = z.object({
 	description: z.string().optional(),
 	family: z.string(),
 	architecture: z.object({
-		input_modalities: z.array(z.enum(["text", "image", "video", "embedding"])),
+		input_modalities: z.array(
+			z.enum(["text", "image", "video", "embedding", "audio"]),
+		),
 		output_modalities: z.array(
 			z.enum(["text", "image", "video", "embedding", "audio", "ocr"]),
 		),
@@ -42,6 +44,9 @@ const modelSchema = z.object({
 					prompt: z.string(),
 					completion: z.string(),
 					image: z.string().optional(),
+					input_audio: z.string().optional(),
+					input_audio_cache_read: z.string().optional(),
+					output_audio: z.string().optional(),
 					per_second: z.record(z.string()).optional(),
 					request: z.string().optional(),
 					input_cache_read: z.string().optional(),
@@ -52,6 +57,10 @@ const modelSchema = z.object({
 				.optional(),
 			streaming: z.union([z.boolean(), z.literal("only")]),
 			vision: z.boolean(),
+			realtime: z.boolean().optional().openapi({
+				description:
+					"Whether this mapping is served via the /v1/realtime WebSocket endpoint instead of /v1/chat/completions.",
+			}),
 			cancellation: z.boolean(),
 			tools: z.boolean(),
 			parallelToolCalls: z.boolean(),
@@ -78,6 +87,9 @@ const modelSchema = z.object({
 		prompt: z.string(),
 		completion: z.string(),
 		image: z.string().optional(),
+		input_audio: z.string().optional(),
+		input_audio_cache_read: z.string().optional(),
+		output_audio: z.string().optional(),
 		per_second: z.record(z.string()).optional(),
 		request: z.string().optional(),
 		input_cache_read: z.string().optional(),
@@ -205,15 +217,24 @@ modelsApi.openapi(listModels, async (c) => {
 
 		const modelData = filteredModels.map((model: ModelDefinition) => {
 			// Determine input modalities (if model supports images)
-			const inputModalities: ("text" | "image" | "video" | "embedding")[] = [
-				"text",
-			];
+			const inputModalities: (
+				| "text"
+				| "image"
+				| "video"
+				| "embedding"
+				| "audio"
+			)[] = ["text"];
 
 			// Check if any provider has vision support
 			if (model.providers.some((p) => p.vision)) {
 				inputModalities.push("image");
 			}
 
+			// Models that accept input_audio content (including realtime models)
+			if (model.providers.some((p) => p.audio)) {
+				inputModalities.push("audio");
+			}
+
 			// Determine output modalities from the model definition or default to
 			// text only. These mirror the model catalog 1:1 (including "ocr") so
 			// third-party clients can reference the same modality taxonomy.
@@ -265,6 +286,7 @@ modelsApi.openapi(listModels, async (c) => {
 							: undefined,
 						streaming: provider.streaming,
 						vision: provider.vision ?? false,
+						realtime: provider.realtime === true ? true : undefined,
 						cancellation: providerDef?.cancellation ?? false,
 						tools: provider.tools ?? false,
 						parallelToolCalls: provider.parallelToolCalls ?? false,
@@ -353,6 +375,9 @@ function buildPricingFields(p: ProviderModelMapping | undefined) {
 		prompt: p?.inputPrice?.toString() ?? "0",
 		completion: p?.outputPrice?.toString() ?? "0",
 		image: p?.imageInputPrice?.toString() ?? "0",
+		input_audio: p?.inputAudioPrice?.toString(),
+		input_audio_cache_read: p?.cachedInputAudioPrice?.toString(),
+		output_audio: p?.outputAudioPrice?.toString(),
 		per_second: p?.perSecondPrice
 			? Object.fromEntries(
 					Object.entries(p.perSecondPrice).map(([resolution, price]) => [