Repository navigation
fix(cache): hotpath db queries caching #2662
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 2 commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,6 @@ | ||
| import { and, eq, isNull, or } from "drizzle-orm"; | ||
|
|
||
| import { db } from "./db.js"; | ||
| import { cdb } from "./cdb.js"; | ||
| import { rateLimit as rateLimitTable } from "./schema.js"; | ||
|
|
||
| export type RateLimitSource = | ||
|
|
@@ -149,7 +149,10 @@ function pickRateLimitByPrecedence( | |
|
|
||
| /** | ||
| * Get the effective rate limits for a given organization, provider, and model. | ||
| * Uses the uncached database client so admin changes take effect immediately. | ||
| * Uses the cached database client (cdb) so this hot, per-request lookup is served | ||
| * from the Drizzle cache instead of hitting Postgres on every gateway request. | ||
| * The WHERE clause is time-independent, so the cache key stays stable. Admin | ||
| * changes propagate within the cache TTL (default 60s). | ||
| * | ||
| * Rate limits are always keyed by the root model ID — provider-specific model | ||
| * names are reserved for upstream requests and are never persisted as a | ||
|
|
@@ -168,7 +171,7 @@ export async function getEffectiveRateLimit( | |
| provider: string, | ||
| model: string, | ||
| ): Promise<EffectiveRateLimit> { | ||
| const rateLimits = await db | ||
| const rateLimits = await cdb | ||
| .select({ | ||
|
Comment on lines
+174
to
175
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When an admin creates or deletes a rate limit to stop traffic, this cached lookup can keep returning the previous result because the admin handlers mutate Useful? React with 👍 / 👎. |
||
| id: rateLimitTable.id, | ||
| organizationId: rateLimitTable.organizationId, | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,143 @@ | ||
| #!/usr/bin/env node | ||
| // Simple load test for api.llmgateway.io | ||
| // Fires `RATE` requests per second using parallel fetch() calls. | ||
|
|
||
| const API_URL = | ||
| process.env.API_URL || "https://api.llmgateway.io/v1/chat/completions"; | ||
| const API_KEY = process.env.LLMGATEWAY_API_KEY; | ||
| const MODEL = process.env.MODEL || "deepseek-v4-flash"; | ||
|
|
||
| if (!API_KEY) { | ||
| console.error("Missing LLMGATEWAY_API_KEY environment variable"); | ||
| process.exit(1); | ||
| } | ||
|
|
||
| const RATE = Number(process.env.RATE) || 100; // requests per second | ||
| const DURATION_SEC = Number(process.env.DURATION_SEC) || 60; // how long to run | ||
|
|
||
| const payload = { | ||
| model: MODEL, | ||
| messages: [{ role: "user", content: "Reply with a short response: ping" }], | ||
| }; | ||
|
|
||
| let sent = 0; | ||
| let ok = 0; | ||
| let failed = 0; | ||
| const latencies = []; // successful requests only | ||
| const errors = new Map(); // category -> { count, sampleId, sampleMs, sampleMsg } | ||
| const SAMPLES_PER_CATEGORY = 3; // live-log at most this many of each error category | ||
|
|
||
| function recordError(category, id, ms, msg) { | ||
| failed++; | ||
| let e = errors.get(category); | ||
| if (!e) { | ||
| e = { count: 0, sampleMsg: msg }; | ||
| errors.set(category, e); | ||
| } | ||
| e.count++; | ||
| if (e.count <= SAMPLES_PER_CATEGORY) { | ||
| const t = ms != null ? ` ${ms.toFixed(0)}ms` : ""; | ||
| console.error(`[${id}]${t} ${category}: ${msg}`); | ||
| } else if (e.count === SAMPLES_PER_CATEGORY + 1) { | ||
| console.error( | ||
| `[…] further "${category}" errors suppressed (counted in summary)`, | ||
| ); | ||
| } | ||
| } | ||
|
|
||
| async function fireOne(id) { | ||
| const start = performance.now(); | ||
| try { | ||
| const res = await fetch(API_URL, { | ||
| method: "POST", | ||
| headers: { | ||
| "Content-Type": "application/json", | ||
| Authorization: `Bearer ${API_KEY}`, | ||
| }, | ||
| body: JSON.stringify(payload), | ||
| }); | ||
| const ms = performance.now() - start; | ||
| if (res.ok) { | ||
| ok++; | ||
| latencies.push(ms); | ||
| } else { | ||
| const body = await res.text().catch(() => ""); | ||
| recordError( | ||
| `HTTP ${res.status}`, | ||
| id, | ||
| ms, | ||
| body.replace(/\s+/g, " ").trim().slice(0, 200), | ||
| ); | ||
| } | ||
| } catch (err) { | ||
| const ms = performance.now() - start; | ||
| // node/undici nests the useful reason in err.cause | ||
| const cause = err.cause | ||
| ? `${err.cause.code || err.cause.message || err.cause}` | ||
| : ""; | ||
| const category = `NETWORK ${err.cause?.code || err.code || err.message}`; | ||
| recordError( | ||
| category, | ||
| id, | ||
| ms, | ||
| [err.message, cause].filter(Boolean).join(" / "), | ||
| ); | ||
| } | ||
| } | ||
|
|
||
| console.log( | ||
| `Load testing ${API_URL} @ ${RATE} r/s for ${DURATION_SEC}s (model: ${MODEL})`, | ||
| ); | ||
|
|
||
| const inFlight = []; | ||
| function tick() { | ||
| for (let i = 0; i < RATE; i++) { | ||
| inFlight.push(fireOne(++sent)); | ||
| } | ||
| } | ||
| tick(); // fire immediately at t=0 | ||
| const ticker = setInterval(tick, 1000); // then 59 more ticks over 59s | ||
|
|
||
| setTimeout(async () => { | ||
| clearInterval(ticker); | ||
| await Promise.allSettled(inFlight); | ||
|
|
||
| const sorted = [...latencies].sort((a, b) => a - b); | ||
| const pct = (p) => | ||
| sorted.length | ||
| ? sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * p))] | ||
| : 0; | ||
| const avg = latencies.length | ||
| ? latencies.reduce((a, b) => a + b, 0) / latencies.length | ||
| : 0; | ||
| const successRate = sent ? (ok / sent) * 100 : 0; | ||
| const achieved = ok / DURATION_SEC; | ||
|
|
||
| console.log("\n=== results ==="); | ||
| console.log( | ||
| `sent: ${sent} (target ${RATE} r/s for ${DURATION_SEC}s)`, | ||
| ); | ||
| console.log(`ok: ${ok} (${successRate.toFixed(1)}%)`); | ||
| console.log(`failed: ${failed} (${(100 - successRate).toFixed(1)}%)`); | ||
| console.log(`throughput: ${achieved.toFixed(1)} successful r/s`); | ||
|
|
||
| console.log(`\nlatency (ok only):`); | ||
| console.log( | ||
| ` avg ${avg.toFixed(0)}ms | min ${pct(0).toFixed(0)}ms | p50 ${pct(0.5).toFixed(0)}ms | p95 ${pct(0.95).toFixed(0)}ms | p99 ${pct(0.99).toFixed(0)}ms | max ${pct(1).toFixed(0)}ms`, | ||
| ); | ||
|
|
||
| if (errors.size) { | ||
| console.log(`\nerror breakdown:`); | ||
| const rows = [...errors.entries()].sort((a, b) => b[1].count - a[1].count); | ||
| for (const [category, e] of rows) { | ||
| const share = ((e.count / failed) * 100).toFixed(1); | ||
| console.log( | ||
| ` ${String(e.count).padStart(6)} ${share.padStart(5)}% ${category}`, | ||
| ); | ||
| if (e.sampleMsg) | ||
| console.log(` └─ e.g. ${e.sampleMsg.slice(0, 120)}`); | ||
| } | ||
| } else { | ||
| console.log(`\nno errors 🎉`); | ||
| } | ||
| }, DURATION_SEC * 1000); |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
With this lookup now using a stable
cdbcache key, admin-created or deleted discounts can be ignored until the 60s Drizzle cache TTL expires, because the admin discount routes mutatetables.discountthrough the uncacheddbclient (apps/api/src/routes/admin.tsinserts/deletes at lines 3165/3183 and 3273/3291), soRedisCache.onMutateis not invoked. In production this means a newly added discount may not be applied, or a removed discount may keep affecting gateway pricing/cost accounting, for requests whose org/provider/model tuple was already warmed in cache; invalidate thediscountcache on these mutations or keep this enforcement/pricing lookup uncached.Useful? React with 👍 / 👎.