perf(gateway): prewarm upstream origins

apps/gateway/src/lib/upstream-dispatcher.spec.ts

@@ -16,12 +16,17 @@ describe("upstream dispatcher", () => {
 	let server: Server;
 	let baseUrl: string;
 	const clientPorts: number[] = [];
+	let headHits = 0;
 
 	beforeAll(async () => {
 		originalDispatcher = getGlobalDispatcher();
 		server = createServer((req, res) => {
 			clientPorts.push(req.socket.remotePort!);
-			if (req.url === "/sse") {
+			if (req.method === "HEAD") {
+				headHits++;
+				res.writeHead(200);
+				res.end();
+			} else if (req.url === "/sse") {
 				res.writeHead(200, { "content-type": "text/event-stream" });
 				res.write("data: first\n\n");
 				// keep the stream open; a later write + end completes it
@@ -48,7 +53,10 @@ describe("upstream dispatcher", () => {
 		await closeUpstreamDispatcher();
 		setGlobalDispatcher(originalDispatcher);
 		delete process.env.UPSTREAM_KEEPALIVE_TIMEOUT_MS;
+		delete process.env.UPSTREAM_PREWARM_ORIGINS;
+		delete process.env.UPSTREAM_PREWARM_INTERVAL_MS;
 		clientPorts.length = 0;
+		headHits = 0;
 	});
 
 	afterAll(async () => {
@@ -87,4 +95,45 @@ describe("upstream dispatcher", () => {
 		process.env.UPSTREAM_KEEPALIVE_TIMEOUT_MS = "not-a-number";
 		expect(() => installUpstreamDispatcher()).not.toThrow();
 	});
+
+	it("prewarms configured origins immediately and on the interval", async () => {
+		process.env.UPSTREAM_PREWARM_ORIGINS = baseUrl;
+		process.env.UPSTREAM_PREWARM_INTERVAL_MS = "25";
+		installUpstreamDispatcher();
+
+		const deadline = Date.now() + 2_000;
+		while (headHits < 3 && Date.now() < deadline) {
+			await new Promise((resolve) => setTimeout(resolve, 10));
+		}
+		expect(headHits).toBeGreaterThanOrEqual(3);
+	});
+
+	it("stops prewarming after the dispatcher is closed", async () => {
+		process.env.UPSTREAM_PREWARM_ORIGINS = baseUrl;
+		process.env.UPSTREAM_PREWARM_INTERVAL_MS = "25";
+		installUpstreamDispatcher();
+
+		const deadline = Date.now() + 2_000;
+		while (headHits < 1 && Date.now() < deadline) {
+			await new Promise((resolve) => setTimeout(resolve, 10));
+		}
+		await closeUpstreamDispatcher();
+		// let any in-flight ping land before snapshotting
+		await new Promise((resolve) => setTimeout(resolve, 50));
+		const settled = headHits;
+		await new Promise((resolve) => setTimeout(resolve, 100));
+		expect(headHits).toBe(settled);
+	});
+
+	it("ignores invalid prewarm origins without failing install", async () => {
+		process.env.UPSTREAM_PREWARM_ORIGINS = `not-a-url, ,${baseUrl}`;
+		process.env.UPSTREAM_PREWARM_INTERVAL_MS = "25";
+		expect(() => installUpstreamDispatcher()).not.toThrow();
+
+		const deadline = Date.now() + 2_000;
+		while (headHits < 1 && Date.now() < deadline) {
+			await new Promise((resolve) => setTimeout(resolve, 10));
+		}
+		expect(headHits).toBeGreaterThanOrEqual(1);
+	});
 });

apps/gateway/src/lib/upstream-dispatcher.ts

@@ -10,6 +10,41 @@ function envInt(name: string, fallback: number): number {
 }
 
 let agent: Agent | null = null;
+let prewarmTimer: NodeJS.Timeout | null = null;
+
+function parsePrewarmOrigins(raw: string | undefined): string[] {
+	if (!raw) {
+		return [];
+	}
+	const origins: string[] = [];
+	for (const entry of raw.split(",")) {
+		const trimmed = entry.trim();
+		if (!trimmed) {
+			continue;
+		}
+		try {
+			origins.push(new URL(trimmed).origin);
+		} catch {
+			logger.warn("Ignoring invalid UPSTREAM_PREWARM_ORIGINS entry", {
+				entry: trimmed,
+			});
+		}
+	}
+	return origins;
+}
+
+async function prewarmOrigin(origin: string): Promise<void> {
+	try {
+		const res = await fetch(origin, {
+			method: "HEAD",
+			redirect: "manual",
+			signal: AbortSignal.timeout(5_000),
+		});
+		await res.body?.cancel();
+	} catch {
+		// Best-effort: an unreachable prewarm origin must never affect serving.
+	}
+}
 
 /**
  * Installs a tuned undici Agent as the global dispatcher used by `fetch` for
@@ -23,11 +58,22 @@ let agent: Agent | null = null;
  * goes through search-domain expansion (ndots:5), where a single dropped UDP
  * packet stalls the request for multiple seconds. A long idle keep-alive plus
  * an in-process DNS cache removes both from the time-to-first-token path.
+ *
+ * Both mitigations only help while traffic keeps them warm: after an idle
+ * gap the next request still pays the full setup cost. The prewarm pinger
+ * closes that hole for the origins that matter — every interval it sends a
+ * HEAD request to each origin in UPSTREAM_PREWARM_ORIGINS through this same
+ * dispatcher, which keeps at least one pooled connection open and re-resolves
+ * DNS on the pinger's time instead of a user request's.
  */
 export function installUpstreamDispatcher(): Dispatcher {
 	const keepAliveTimeoutMs = envInt("UPSTREAM_KEEPALIVE_TIMEOUT_MS", 60_000);
 	const connectTimeoutMs = envInt("UPSTREAM_CONNECT_TIMEOUT_MS", 10_000);
-	const dnsCacheTtlMs = envInt("UPSTREAM_DNS_CACHE_TTL_MS", 30_000);
+	// Provider API hostnames resolve to CDN/anycast addresses that are stable
+	// over minutes, and a connect failure on a stale address is retried by the
+	// provider-fallback logic — so a long TTL is safe, while a short one expires
+	// between requests on quiet pods and puts DNS back on the TTFT path.
+	const dnsCacheTtlMs = envInt("UPSTREAM_DNS_CACHE_TTL_MS", 300_000);
 
 	agent = new Agent({
 		keepAliveTimeout: keepAliveTimeoutMs,
@@ -42,15 +88,39 @@ export function installUpstreamDispatcher(): Dispatcher {
 			: agent;
 
 	setGlobalDispatcher(dispatcher);
+
+	const prewarmOrigins = parsePrewarmOrigins(
+		process.env.UPSTREAM_PREWARM_ORIGINS,
+	);
+	// Must stay below both the local keep-alive timeout and typical provider
+	// edge idle timeouts (~60s) so the pooled connection never idles out.
+	const prewarmIntervalMs = envInt("UPSTREAM_PREWARM_INTERVAL_MS", 25_000);
+	if (prewarmOrigins.length > 0 && prewarmIntervalMs > 0) {
+		const prewarmAll = () => {
+			for (const origin of prewarmOrigins) {
+				void prewarmOrigin(origin);
+			}
+		};
+		prewarmAll();
+		prewarmTimer = setInterval(prewarmAll, prewarmIntervalMs);
+		prewarmTimer.unref();
+	}
+
 	logger.info("Upstream dispatcher installed", {
 		keepAliveTimeoutMs,
 		connectTimeoutMs,
 		dnsCacheTtlMs,
+		prewarmOrigins,
+		prewarmIntervalMs,
 	});
 	return dispatcher;
 }
 
 export async function closeUpstreamDispatcher(): Promise<void> {
+	if (prewarmTimer) {
+		clearInterval(prewarmTimer);
+		prewarmTimer = null;
+	}
 	if (agent) {
 		await agent.close();
 		agent = null;

infra/helm/llmgateway/templates/configmap.yaml

@@ -102,6 +102,18 @@ data:
   {{- if .maxStreamingBufferMb }}
   MAX_STREAMING_BUFFER_MB: {{ .maxStreamingBufferMb | quote }}
   {{- end }}
+  {{- if .upstreamPrewarmOrigins }}
+  UPSTREAM_PREWARM_ORIGINS: {{ .upstreamPrewarmOrigins | quote }}
+  {{- end }}
+  {{- if .upstreamPrewarmIntervalMs }}
+  UPSTREAM_PREWARM_INTERVAL_MS: {{ .upstreamPrewarmIntervalMs | quote }}
+  {{- end }}
+  {{- if .upstreamDnsCacheTtlMs }}
+  UPSTREAM_DNS_CACHE_TTL_MS: {{ .upstreamDnsCacheTtlMs | quote }}
+  {{- end }}
+  {{- if .upstreamKeepaliveTimeoutMs }}
+  UPSTREAM_KEEPALIVE_TIMEOUT_MS: {{ .upstreamKeepaliveTimeoutMs | quote }}
+  {{- end }}
   {{- end }}
 
   # API config

infra/helm/llmgateway/values.yaml

@@ -141,6 +141,9 @@ gateway:
     healthCheckSkipDatabase: ""
     healthCheckTimeoutMs: 15000
     maxStreamingBufferMb: 50
+    # Origins the upstream dispatcher keeps permanently warm (DNS cache +
+    # pooled connection) so an idle pod's next request skips connection setup.
+    upstreamPrewarmOrigins: "https://api.anthropic.com,https://api.openai.com"
   # Hard deadline for pod shutdown; caps every in-process drain above. Defaults
   # to 120s, or realtime.config.maxSessionSeconds + 120 when realtime.enabled.
   # Set explicitly only to override that.