From ef38dbc6f2758c84e22088ccecf82cafbdc47827 Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Fri, 12 Jun 2026 23:09:22 +0100 Subject: [PATCH 1/3] perf(db): cache rate-limit and discount lookups Two hot-path lookups ran on every gateway request instead of being served from the Drizzle/Redis cache, hammering Postgres under load: - getEffectiveRateLimit used the uncached `db` client, so the rate_limit query hit Postgres on every request (swrWrap only mirrors to Redis on DB error, not on the happy path). Switch to the cached `cdb` client; the WHERE clause is time-independent so the cache key is stable. - getEffectiveDiscount put `now` (new Date()) in the SQL WHERE clause. The cached client keys on hashQuery(sql, params), so a per-request millisecond timestamp made the key unique every call -> 0% cache hit. Keep the SQL time-independent and evaluate expiry in JS, mirroring the existing findEffectiveDiscount fix. Verified e2e against a local gateway: a warm request now does 0 Postgres reads (was ~14), and a 60-request concurrent burst hit Postgres only 3 times total (cold-cache races) instead of ~840. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/db/src/discount-helpers.ts | 29 ++++++--- packages/db/src/rate-limit-helpers.spec.ts | 10 +-- packages/db/src/rate-limit-helpers.ts | 9 ++- scripts/loadtest.mjs | 73 ++++++++++++++++++++++ 4 files changed, 103 insertions(+), 18 deletions(-) create mode 100644 scripts/loadtest.mjs diff --git a/packages/db/src/discount-helpers.ts b/packages/db/src/discount-helpers.ts index 046f31b534..31cf4c7094 100644 --- a/packages/db/src/discount-helpers.ts +++ b/packages/db/src/discount-helpers.ts @@ -1,4 +1,4 @@ -import { and, eq, gte, isNull, or } from "drizzle-orm"; +import { and, eq, isNull, or } from "drizzle-orm"; import { logger } from "@llmgateway/logger"; @@ -51,25 +51,25 @@ export async function getEffectiveDiscount( model: string, ): Promise { try { - const now = new Date(); - - const notExpiredCondition = or( - isNull(discountTable.expiresAt), - gte(discountTable.expiresAt, now), - ); - - const discounts = await cdb + // The expiry filter is applied in JS below, NOT in SQL: a `now` Date in the + // WHERE clause becomes a query parameter, and the cached client keys its + // cache on hashQuery(sql, params). A per-request millisecond `now` would + // make that key unique every call, so the cache would never hit and this + // (hot, per-provider-candidate) lookup would query Postgres on every + // request. Keeping the SQL time-independent lets the cache key stay stable + // while expiry is still evaluated fresh on each call. + const rows = await cdb .select({ id: discountTable.id, organizationId: discountTable.organizationId, provider: discountTable.provider, model: discountTable.model, discountPercent: discountTable.discountPercent, + expiresAt: discountTable.expiresAt, }) .from(discountTable) .where( and( - notExpiredCondition, or( isNull(discountTable.organizationId), organizationId @@ -84,6 +84,15 @@ export async function getEffectiveDiscount( ), ); + const now = Date.now(); + const discounts = rows.filter( + // expiresAt is a Date on both a fresh query and a Drizzle cache hit (the + // cache stores the raw pg result and re-applies the timestamp parser on + // restore). Wrap in new Date() defensively so the compare is robust even + // if a serialized value ever reaches here. + (d) => d.expiresAt === null || new Date(d.expiresAt).getTime() >= now, + ); + const modelMatches = (discountModel: string | null): boolean => discountModel !== null && discountModel === model; diff --git a/packages/db/src/rate-limit-helpers.spec.ts b/packages/db/src/rate-limit-helpers.spec.ts index 811adb372a..8185490c7d 100644 --- a/packages/db/src/rate-limit-helpers.spec.ts +++ b/packages/db/src/rate-limit-helpers.spec.ts @@ -1,12 +1,12 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -vi.mock("./db.js", () => ({ - db: { +vi.mock("./cdb.js", () => ({ + cdb: { select: vi.fn(), }, })); -const mockDb = await import("./db.js"); +const mockCdb = await import("./cdb.js"); function createQueryMock(results: Array>) { const chain = { @@ -14,7 +14,7 @@ function createQueryMock(results: Array>) { from: vi.fn().mockReturnThis(), where: vi.fn().mockResolvedValue(results), }; - vi.mocked(mockDb.db.select).mockReturnValue(chain as never); + vi.mocked(mockCdb.cdb.select).mockReturnValue(chain as never); return chain; } @@ -235,7 +235,7 @@ describe("getEffectiveRateLimit", () => { }); it("propagates database errors so callers can apply SWR fallback", async () => { - vi.mocked(mockDb.db.select).mockImplementation(() => { + vi.mocked(mockCdb.cdb.select).mockImplementation(() => { throw new Error("DB error"); }); diff --git a/packages/db/src/rate-limit-helpers.ts b/packages/db/src/rate-limit-helpers.ts index 2746a6c2a5..ce02b345e5 100644 --- a/packages/db/src/rate-limit-helpers.ts +++ b/packages/db/src/rate-limit-helpers.ts @@ -1,6 +1,6 @@ import { and, eq, isNull, or } from "drizzle-orm"; -import { db } from "./db.js"; +import { cdb } from "./cdb.js"; import { rateLimit as rateLimitTable } from "./schema.js"; export type RateLimitSource = @@ -149,7 +149,10 @@ function pickRateLimitByPrecedence( /** * Get the effective rate limits for a given organization, provider, and model. - * Uses the uncached database client so admin changes take effect immediately. + * Uses the cached database client (cdb) so this hot, per-request lookup is served + * from the Drizzle cache instead of hitting Postgres on every gateway request. + * The WHERE clause is time-independent, so the cache key stays stable. Admin + * changes propagate within the cache TTL (default 60s). * * Rate limits are always keyed by the root model ID — provider-specific model * names are reserved for upstream requests and are never persisted as a @@ -168,7 +171,7 @@ export async function getEffectiveRateLimit( provider: string, model: string, ): Promise { - const rateLimits = await db + const rateLimits = await cdb .select({ id: rateLimitTable.id, organizationId: rateLimitTable.organizationId, diff --git a/scripts/loadtest.mjs b/scripts/loadtest.mjs new file mode 100644 index 0000000000..4f8c3f0579 --- /dev/null +++ b/scripts/loadtest.mjs @@ -0,0 +1,73 @@ +#!/usr/bin/env node +// Simple load test for api.llmgateway.io +// Fires `RATE` requests per second using parallel fetch() calls. + +const API_URL = "https://api.llmgateway.io/v1/chat/completions"; +const API_KEY = "llmgtwy_gLZGvfIDaxr1tYbzvVSbZ797IXmuglDGOaLiPexZ"; +const MODEL = "deepseek-v4-flash"; + +const RATE = 100; // requests per second +const DURATION_SEC = 60; // how long to run (set to Infinity for forever) + +const payload = { + model: MODEL, + messages: [{ role: "user", content: "Reply with a short response: ping" }], +}; + +let sent = 0; +let ok = 0; +let failed = 0; +const latencies = []; + +async function fireOne(id) { + const start = performance.now(); + try { + const res = await fetch(API_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${API_KEY}`, + }, + body: JSON.stringify(payload), + }); + const ms = performance.now() - start; + latencies.push(ms); + if (res.ok) { + ok++; + } else { + failed++; + const body = await res.text(); + console.error(`[${id}] HTTP ${res.status} (${ms.toFixed(0)}ms): ${body.slice(0, 200)}`); + } + } catch (err) { + failed++; + console.error(`[${id}] ERROR: ${err.message}`); + } +} + +console.log(`Load testing ${API_URL} @ ${RATE} r/s for ${DURATION_SEC}s (model: ${MODEL})`); + +const inFlight = []; +function tick() { + for (let i = 0; i < RATE; i++) { + inFlight.push(fireOne(++sent)); + } +} +tick(); // fire immediately at t=0 +const ticker = setInterval(tick, 1000); // then 59 more ticks over 59s + +setTimeout(async () => { + clearInterval(ticker); + await Promise.allSettled(inFlight); + + const avg = latencies.length ? latencies.reduce((a, b) => a + b, 0) / latencies.length : 0; + const sorted = [...latencies].sort((a, b) => a - b); + const p95 = sorted.length ? sorted[Math.floor(sorted.length * 0.95)] : 0; + + console.log("\n--- results ---"); + console.log(`sent: ${sent}`); + console.log(`ok: ${ok}`); + console.log(`failed: ${failed}`); + console.log(`avg latency: ${avg.toFixed(0)}ms`); + console.log(`p95 latency: ${p95.toFixed(0)}ms`); +}, DURATION_SEC * 1000); From 3f8a9ffcbb3dd6a6e787fad2286a290a7eed8db7 Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Fri, 12 Jun 2026 23:12:50 +0100 Subject: [PATCH 2/3] test(loadtest): add API_URL/MODEL env overrides Allow pointing the load test at a local gateway and varying the model without editing the script. Also move the API key to an env var and add categorized error reporting (per-category counts + sampled messages). Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/loadtest.mjs | 166 ++++++++++++++++++++++++++++++------------- 1 file changed, 118 insertions(+), 48 deletions(-) diff --git a/scripts/loadtest.mjs b/scripts/loadtest.mjs index 4f8c3f0579..d2ff43ea94 100644 --- a/scripts/loadtest.mjs +++ b/scripts/loadtest.mjs @@ -2,72 +2,142 @@ // Simple load test for api.llmgateway.io // Fires `RATE` requests per second using parallel fetch() calls. -const API_URL = "https://api.llmgateway.io/v1/chat/completions"; -const API_KEY = "llmgtwy_gLZGvfIDaxr1tYbzvVSbZ797IXmuglDGOaLiPexZ"; -const MODEL = "deepseek-v4-flash"; +const API_URL = + process.env.API_URL || "https://api.llmgateway.io/v1/chat/completions"; +const API_KEY = process.env.LLMGATEWAY_API_KEY; +const MODEL = process.env.MODEL || "deepseek-v4-flash"; -const RATE = 100; // requests per second -const DURATION_SEC = 60; // how long to run (set to Infinity for forever) +if (!API_KEY) { + console.error("Missing LLMGATEWAY_API_KEY environment variable"); + process.exit(1); +} + +const RATE = Number(process.env.RATE) || 100; // requests per second +const DURATION_SEC = Number(process.env.DURATION_SEC) || 60; // how long to run const payload = { - model: MODEL, - messages: [{ role: "user", content: "Reply with a short response: ping" }], + model: MODEL, + messages: [{ role: "user", content: "Reply with a short response: ping" }], }; let sent = 0; let ok = 0; let failed = 0; -const latencies = []; +const latencies = []; // successful requests only +const errors = new Map(); // category -> { count, sampleId, sampleMs, sampleMsg } +const SAMPLES_PER_CATEGORY = 3; // live-log at most this many of each error category + +function recordError(category, id, ms, msg) { + failed++; + let e = errors.get(category); + if (!e) { + e = { count: 0, sampleMsg: msg }; + errors.set(category, e); + } + e.count++; + if (e.count <= SAMPLES_PER_CATEGORY) { + const t = ms != null ? ` ${ms.toFixed(0)}ms` : ""; + console.error(`[${id}]${t} ${category}: ${msg}`); + } else if (e.count === SAMPLES_PER_CATEGORY + 1) { + console.error( + `[…] further "${category}" errors suppressed (counted in summary)`, + ); + } +} async function fireOne(id) { - const start = performance.now(); - try { - const res = await fetch(API_URL, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${API_KEY}`, - }, - body: JSON.stringify(payload), - }); - const ms = performance.now() - start; - latencies.push(ms); - if (res.ok) { - ok++; - } else { - failed++; - const body = await res.text(); - console.error(`[${id}] HTTP ${res.status} (${ms.toFixed(0)}ms): ${body.slice(0, 200)}`); - } - } catch (err) { - failed++; - console.error(`[${id}] ERROR: ${err.message}`); - } + const start = performance.now(); + try { + const res = await fetch(API_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${API_KEY}`, + }, + body: JSON.stringify(payload), + }); + const ms = performance.now() - start; + if (res.ok) { + ok++; + latencies.push(ms); + } else { + const body = await res.text().catch(() => ""); + recordError( + `HTTP ${res.status}`, + id, + ms, + body.replace(/\s+/g, " ").trim().slice(0, 200), + ); + } + } catch (err) { + const ms = performance.now() - start; + // node/undici nests the useful reason in err.cause + const cause = err.cause + ? `${err.cause.code || err.cause.message || err.cause}` + : ""; + const category = `NETWORK ${err.cause?.code || err.code || err.message}`; + recordError( + category, + id, + ms, + [err.message, cause].filter(Boolean).join(" / "), + ); + } } -console.log(`Load testing ${API_URL} @ ${RATE} r/s for ${DURATION_SEC}s (model: ${MODEL})`); +console.log( + `Load testing ${API_URL} @ ${RATE} r/s for ${DURATION_SEC}s (model: ${MODEL})`, +); const inFlight = []; function tick() { - for (let i = 0; i < RATE; i++) { - inFlight.push(fireOne(++sent)); - } + for (let i = 0; i < RATE; i++) { + inFlight.push(fireOne(++sent)); + } } -tick(); // fire immediately at t=0 -const ticker = setInterval(tick, 1000); // then 59 more ticks over 59s +tick(); // fire immediately at t=0 +const ticker = setInterval(tick, 1000); // then 59 more ticks over 59s setTimeout(async () => { - clearInterval(ticker); - await Promise.allSettled(inFlight); + clearInterval(ticker); + await Promise.allSettled(inFlight); + + const sorted = [...latencies].sort((a, b) => a - b); + const pct = (p) => + sorted.length + ? sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * p))] + : 0; + const avg = latencies.length + ? latencies.reduce((a, b) => a + b, 0) / latencies.length + : 0; + const successRate = sent ? (ok / sent) * 100 : 0; + const achieved = ok / DURATION_SEC; + + console.log("\n=== results ==="); + console.log( + `sent: ${sent} (target ${RATE} r/s for ${DURATION_SEC}s)`, + ); + console.log(`ok: ${ok} (${successRate.toFixed(1)}%)`); + console.log(`failed: ${failed} (${(100 - successRate).toFixed(1)}%)`); + console.log(`throughput: ${achieved.toFixed(1)} successful r/s`); - const avg = latencies.length ? latencies.reduce((a, b) => a + b, 0) / latencies.length : 0; - const sorted = [...latencies].sort((a, b) => a - b); - const p95 = sorted.length ? sorted[Math.floor(sorted.length * 0.95)] : 0; + console.log(`\nlatency (ok only):`); + console.log( + ` avg ${avg.toFixed(0)}ms | min ${pct(0).toFixed(0)}ms | p50 ${pct(0.5).toFixed(0)}ms | p95 ${pct(0.95).toFixed(0)}ms | p99 ${pct(0.99).toFixed(0)}ms | max ${pct(1).toFixed(0)}ms`, + ); - console.log("\n--- results ---"); - console.log(`sent: ${sent}`); - console.log(`ok: ${ok}`); - console.log(`failed: ${failed}`); - console.log(`avg latency: ${avg.toFixed(0)}ms`); - console.log(`p95 latency: ${p95.toFixed(0)}ms`); + if (errors.size) { + console.log(`\nerror breakdown:`); + const rows = [...errors.entries()].sort((a, b) => b[1].count - a[1].count); + for (const [category, e] of rows) { + const share = ((e.count / failed) * 100).toFixed(1); + console.log( + ` ${String(e.count).padStart(6)} ${share.padStart(5)}% ${category}`, + ); + if (e.sampleMsg) + console.log(` └─ e.g. ${e.sampleMsg.slice(0, 120)}`); + } + } else { + console.log(`\nno errors 🎉`); + } }, DURATION_SEC * 1000); From 7e8bd82c275d6e47e1636ca598ff5e5e502cbd4f Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Fri, 12 Jun 2026 23:35:32 +0100 Subject: [PATCH 3/3] test(gateway): guard repeated chat requests against DB reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an e2e regression test that warms the gateway caches, then spies on the shared pg pool during a fresh chat completion and asserts zero SELECTs hit the cached metadata tables (api_key, project, organization, provider_key, rate_limit, discount, routing metrics, ...). Both the cached (cdb) and uncached (db) clients share this pool, so the spy catches any query that actually reaches Postgres — protecting against both regression vectors: an unstable cache key (per-request Date param) and switching a hot lookup to the uncached client. Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/gateway/src/chat-cache-no-db.e2e.ts | 204 +++++++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 apps/gateway/src/chat-cache-no-db.e2e.ts diff --git a/apps/gateway/src/chat-cache-no-db.e2e.ts b/apps/gateway/src/chat-cache-no-db.e2e.ts new file mode 100644 index 0000000000..211f263e7d --- /dev/null +++ b/apps/gateway/src/chat-cache-no-db.e2e.ts @@ -0,0 +1,204 @@ +import { serve } from "@hono/node-server"; +import "dotenv/config"; +import { Hono } from "hono"; +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + test, + vi, +} from "vitest"; + +import { app } from "@/app.js"; +import { clearCache } from "@/test-utils/test-helpers.js"; + +import { db, pool, tables } from "@llmgateway/db"; + +// A trivial upstream that always returns a valid completion, so the gateway +// path runs end-to-end without touching a real provider. +const mockServer = new Hono(); +let server: ReturnType | null = null; +const MOCK_PORT = 3099; + +mockServer.post("/v1/chat/completions", async (c) => { + return c.json({ + id: "chatcmpl-mock", + object: "chat.completion", + created: Math.floor(Date.now() / 1000), + model: "mock-model", + choices: [ + { + index: 0, + message: { role: "assistant", content: "Hello" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 }, + }); +}); + +/** + * Tables whose reads MUST be served from the Redis/Drizzle cache once warm. + * A SELECT against any of these on a repeated chat request means the per-request + * metadata lookup regressed to hitting Postgres (unstable cache key or use of + * the uncached client) — the exact class of bug this test guards against. + */ +const CACHED_READ_TABLES = [ + "api_key", + "api_key_iam_rule", + "project", + "organization", + "provider_key", + "rate_limit", + "discount", + "model_provider_mapping_history", + "user_organization", + "wallet", + "end_user_session", + "end_customer", + "routing_config", +]; + +function statementText(args: unknown[]): string { + const first = args[0]; + if (typeof first === "string") { + return first; + } + if (first && typeof first === "object") { + const o = first as { text?: unknown; sql?: unknown }; + if (typeof o.text === "string") { + return o.text; + } + if (typeof o.sql === "string") { + return o.sql; + } + } + return ""; +} + +function cachedTableReads(statements: string[]): string[] { + return statements.filter((sql) => { + const lower = sql.toLowerCase(); + if (!lower.includes("select")) { + return false; + } + return CACHED_READ_TABLES.some((t) => lower.includes(`from "${t}"`)); + }); +} + +async function sendChat(prompt: string) { + return await app.request("/v1/chat/completions", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: "Bearer cache-token", + }, + body: JSON.stringify({ + model: "llmgateway/custom", + messages: [{ role: "user", content: prompt }], + }), + }); +} + +describe("Chat completions caching: repeated requests do not re-read Postgres", () => { + beforeAll(async () => { + server = serve({ fetch: mockServer.fetch, port: MOCK_PORT }); + }); + + afterAll(() => { + if (server) { + server.close(); + } + }); + + beforeEach(async () => { + await clearCache(); + + await Promise.all([ + db.delete(tables.log), + db.delete(tables.apiKey), + db.delete(tables.providerKey), + ]); + await Promise.all([ + db.delete(tables.userOrganization), + db.delete(tables.project), + ]); + await Promise.all([db.delete(tables.organization), db.delete(tables.user)]); + + await db.insert(tables.user).values({ + id: "cache-user", + name: "user", + email: "cache@test.com", + }); + // Keep credits positive: findOrganizationById refetches UNCACHED when an + // org is out of credits, which would legitimately hit Postgres every time. + await db.insert(tables.organization).values({ + id: "cache-org", + name: "Cache Org", + billingEmail: "cache@test.com", + plan: "pro", + credits: "100.00", + }); + await db.insert(tables.userOrganization).values({ + id: "cache-user-org", + userId: "cache-user", + organizationId: "cache-org", + }); + await db.insert(tables.project).values({ + id: "cache-project", + name: "Cache Project", + organizationId: "cache-org", + mode: "api-keys", + }); + await db.insert(tables.apiKey).values({ + id: "cache-key", + token: "cache-token", + projectId: "cache-project", + description: "Cache Key", + createdBy: "cache-user", + }); + await db.insert(tables.providerKey).values({ + id: "cache-provider-key", + token: "sk-mock", + provider: "llmgateway", + organizationId: "cache-org", + baseUrl: `http://localhost:${MOCK_PORT}`, + }); + }); + + test("a warm chat request issues zero Postgres reads on cached metadata tables", async () => { + // Warm the cache: api key, project, org, provider key, rate limits, + // discounts and routing metrics are all looked up and cached here. + expect((await sendChat("warm up one")).status).toBe(200); + expect((await sendChat("warm up two")).status).toBe(200); + + // Record every statement the shared pool executes during a fresh request. + // Both the cached (cdb) and uncached (db) clients use this same pool, so + // this captures any query that actually reaches Postgres. + const statements: string[] = []; + const original = pool.query.bind(pool); + const spy = vi.spyOn(pool, "query").mockImplementation((( + ...args: Parameters + ) => { + statements.push(statementText(args)); + return original(...args); + }) as typeof pool.query); + + try { + // A unique prompt guarantees the response cache cannot short-circuit + // the request, so the full auth/routing/pricing path runs. + const res = await sendChat("measured request with a unique prompt"); + expect(res.status).toBe(200); + } finally { + spy.mockRestore(); + } + + const leaked = cachedTableReads(statements); + expect( + leaked, + `Expected zero cached-table SELECTs on a warm request, but these hit Postgres:\n${leaked.join("\n")}`, + ).toEqual([]); + }); +});