From 9cb75821799d03aefe283219430e6afea9bc00d1 Mon Sep 17 00:00:00 2001 From: smakosh Date: Thu, 13 Aug 2026 21:24:02 +0200 Subject: [PATCH] perf(gateway): prewarm upstream origins Keep a pooled connection and fresh DNS entry to key provider origins on every pod, so an idle pod's next request skips connection setup to the provider. Raises the upstream DNS cache TTL default to 300s and adds chart plumbing for the dispatcher env knobs. Extracted from #3374 (prewarm + DNS TTL only; no hedging, no resource/dnsConfig changes). Claude-Session: https://claude.ai/code/session_01SyZUQMBQaXbH6HsFbkmaH1 --- .../src/lib/upstream-dispatcher.spec.ts | 51 ++++++++++++- apps/gateway/src/lib/upstream-dispatcher.ts | 72 ++++++++++++++++++- .../helm/llmgateway/templates/configmap.yaml | 12 ++++ infra/helm/llmgateway/values.yaml | 3 + 4 files changed, 136 insertions(+), 2 deletions(-) diff --git a/apps/gateway/src/lib/upstream-dispatcher.spec.ts b/apps/gateway/src/lib/upstream-dispatcher.spec.ts index 8a622f1839..f8f458bc69 100644 --- a/apps/gateway/src/lib/upstream-dispatcher.spec.ts +++ b/apps/gateway/src/lib/upstream-dispatcher.spec.ts @@ -16,12 +16,17 @@ describe("upstream dispatcher", () => { let server: Server; let baseUrl: string; const clientPorts: number[] = []; + let headHits = 0; beforeAll(async () => { originalDispatcher = getGlobalDispatcher(); server = createServer((req, res) => { clientPorts.push(req.socket.remotePort!); - if (req.url === "/sse") { + if (req.method === "HEAD") { + headHits++; + res.writeHead(200); + res.end(); + } else if (req.url === "/sse") { res.writeHead(200, { "content-type": "text/event-stream" }); res.write("data: first\n\n"); // keep the stream open; a later write + end completes it @@ -48,7 +53,10 @@ describe("upstream dispatcher", () => { await closeUpstreamDispatcher(); setGlobalDispatcher(originalDispatcher); delete process.env.UPSTREAM_KEEPALIVE_TIMEOUT_MS; + delete process.env.UPSTREAM_PREWARM_ORIGINS; + delete process.env.UPSTREAM_PREWARM_INTERVAL_MS; clientPorts.length = 0; + headHits = 0; }); afterAll(async () => { @@ -87,4 +95,45 @@ describe("upstream dispatcher", () => { process.env.UPSTREAM_KEEPALIVE_TIMEOUT_MS = "not-a-number"; expect(() => installUpstreamDispatcher()).not.toThrow(); }); + + it("prewarms configured origins immediately and on the interval", async () => { + process.env.UPSTREAM_PREWARM_ORIGINS = baseUrl; + process.env.UPSTREAM_PREWARM_INTERVAL_MS = "25"; + installUpstreamDispatcher(); + + const deadline = Date.now() + 2_000; + while (headHits < 3 && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(headHits).toBeGreaterThanOrEqual(3); + }); + + it("stops prewarming after the dispatcher is closed", async () => { + process.env.UPSTREAM_PREWARM_ORIGINS = baseUrl; + process.env.UPSTREAM_PREWARM_INTERVAL_MS = "25"; + installUpstreamDispatcher(); + + const deadline = Date.now() + 2_000; + while (headHits < 1 && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + await closeUpstreamDispatcher(); + // let any in-flight ping land before snapshotting + await new Promise((resolve) => setTimeout(resolve, 50)); + const settled = headHits; + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(headHits).toBe(settled); + }); + + it("ignores invalid prewarm origins without failing install", async () => { + process.env.UPSTREAM_PREWARM_ORIGINS = `not-a-url, ,${baseUrl}`; + process.env.UPSTREAM_PREWARM_INTERVAL_MS = "25"; + expect(() => installUpstreamDispatcher()).not.toThrow(); + + const deadline = Date.now() + 2_000; + while (headHits < 1 && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(headHits).toBeGreaterThanOrEqual(1); + }); }); diff --git a/apps/gateway/src/lib/upstream-dispatcher.ts b/apps/gateway/src/lib/upstream-dispatcher.ts index c5847da3a3..204061d62d 100644 --- a/apps/gateway/src/lib/upstream-dispatcher.ts +++ b/apps/gateway/src/lib/upstream-dispatcher.ts @@ -10,6 +10,41 @@ function envInt(name: string, fallback: number): number { } let agent: Agent | null = null; +let prewarmTimer: NodeJS.Timeout | null = null; + +function parsePrewarmOrigins(raw: string | undefined): string[] { + if (!raw) { + return []; + } + const origins: string[] = []; + for (const entry of raw.split(",")) { + const trimmed = entry.trim(); + if (!trimmed) { + continue; + } + try { + origins.push(new URL(trimmed).origin); + } catch { + logger.warn("Ignoring invalid UPSTREAM_PREWARM_ORIGINS entry", { + entry: trimmed, + }); + } + } + return origins; +} + +async function prewarmOrigin(origin: string): Promise { + try { + const res = await fetch(origin, { + method: "HEAD", + redirect: "manual", + signal: AbortSignal.timeout(5_000), + }); + await res.body?.cancel(); + } catch { + // Best-effort: an unreachable prewarm origin must never affect serving. + } +} /** * Installs a tuned undici Agent as the global dispatcher used by `fetch` for @@ -23,11 +58,22 @@ let agent: Agent | null = null; * goes through search-domain expansion (ndots:5), where a single dropped UDP * packet stalls the request for multiple seconds. A long idle keep-alive plus * an in-process DNS cache removes both from the time-to-first-token path. + * + * Both mitigations only help while traffic keeps them warm: after an idle + * gap the next request still pays the full setup cost. The prewarm pinger + * closes that hole for the origins that matter — every interval it sends a + * HEAD request to each origin in UPSTREAM_PREWARM_ORIGINS through this same + * dispatcher, which keeps at least one pooled connection open and re-resolves + * DNS on the pinger's time instead of a user request's. */ export function installUpstreamDispatcher(): Dispatcher { const keepAliveTimeoutMs = envInt("UPSTREAM_KEEPALIVE_TIMEOUT_MS", 60_000); const connectTimeoutMs = envInt("UPSTREAM_CONNECT_TIMEOUT_MS", 10_000); - const dnsCacheTtlMs = envInt("UPSTREAM_DNS_CACHE_TTL_MS", 30_000); + // Provider API hostnames resolve to CDN/anycast addresses that are stable + // over minutes, and a connect failure on a stale address is retried by the + // provider-fallback logic — so a long TTL is safe, while a short one expires + // between requests on quiet pods and puts DNS back on the TTFT path. + const dnsCacheTtlMs = envInt("UPSTREAM_DNS_CACHE_TTL_MS", 300_000); agent = new Agent({ keepAliveTimeout: keepAliveTimeoutMs, @@ -42,15 +88,39 @@ export function installUpstreamDispatcher(): Dispatcher { : agent; setGlobalDispatcher(dispatcher); + + const prewarmOrigins = parsePrewarmOrigins( + process.env.UPSTREAM_PREWARM_ORIGINS, + ); + // Must stay below both the local keep-alive timeout and typical provider + // edge idle timeouts (~60s) so the pooled connection never idles out. + const prewarmIntervalMs = envInt("UPSTREAM_PREWARM_INTERVAL_MS", 25_000); + if (prewarmOrigins.length > 0 && prewarmIntervalMs > 0) { + const prewarmAll = () => { + for (const origin of prewarmOrigins) { + void prewarmOrigin(origin); + } + }; + prewarmAll(); + prewarmTimer = setInterval(prewarmAll, prewarmIntervalMs); + prewarmTimer.unref(); + } + logger.info("Upstream dispatcher installed", { keepAliveTimeoutMs, connectTimeoutMs, dnsCacheTtlMs, + prewarmOrigins, + prewarmIntervalMs, }); return dispatcher; } export async function closeUpstreamDispatcher(): Promise { + if (prewarmTimer) { + clearInterval(prewarmTimer); + prewarmTimer = null; + } if (agent) { await agent.close(); agent = null; diff --git a/infra/helm/llmgateway/templates/configmap.yaml b/infra/helm/llmgateway/templates/configmap.yaml index a61f50b03a..fb6b9349ff 100644 --- a/infra/helm/llmgateway/templates/configmap.yaml +++ b/infra/helm/llmgateway/templates/configmap.yaml @@ -102,6 +102,18 @@ data: {{- if .maxStreamingBufferMb }} MAX_STREAMING_BUFFER_MB: {{ .maxStreamingBufferMb | quote }} {{- end }} + {{- if .upstreamPrewarmOrigins }} + UPSTREAM_PREWARM_ORIGINS: {{ .upstreamPrewarmOrigins | quote }} + {{- end }} + {{- if .upstreamPrewarmIntervalMs }} + UPSTREAM_PREWARM_INTERVAL_MS: {{ .upstreamPrewarmIntervalMs | quote }} + {{- end }} + {{- if .upstreamDnsCacheTtlMs }} + UPSTREAM_DNS_CACHE_TTL_MS: {{ .upstreamDnsCacheTtlMs | quote }} + {{- end }} + {{- if .upstreamKeepaliveTimeoutMs }} + UPSTREAM_KEEPALIVE_TIMEOUT_MS: {{ .upstreamKeepaliveTimeoutMs | quote }} + {{- end }} {{- end }} # API config diff --git a/infra/helm/llmgateway/values.yaml b/infra/helm/llmgateway/values.yaml index 9cfb453ed8..2c5ee16c07 100644 --- a/infra/helm/llmgateway/values.yaml +++ b/infra/helm/llmgateway/values.yaml @@ -141,6 +141,9 @@ gateway: healthCheckSkipDatabase: "" healthCheckTimeoutMs: 15000 maxStreamingBufferMb: 50 + # Origins the upstream dispatcher keeps permanently warm (DNS cache + + # pooled connection) so an idle pod's next request skips connection setup. + upstreamPrewarmOrigins: "https://api.anthropic.com,https://api.openai.com" # Hard deadline for pod shutdown; caps every in-process drain above. Defaults # to 120s, or realtime.config.maxSessionSeconds + 120 when realtime.enabled. # Set explicitly only to override that.