Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 19 additions & 10 deletions packages/db/src/discount-helpers.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { and, eq, gte, isNull, or } from "drizzle-orm";
import { and, eq, isNull, or } from "drizzle-orm";

import { logger } from "@llmgateway/logger";

Expand Down Expand Up @@ -51,25 +51,25 @@ export async function getEffectiveDiscount(
model: string,
): Promise<EffectiveDiscount> {
try {
const now = new Date();

const notExpiredCondition = or(
isNull(discountTable.expiresAt),
gte(discountTable.expiresAt, now),
);

const discounts = await cdb
// The expiry filter is applied in JS below, NOT in SQL: a `now` Date in the
// WHERE clause becomes a query parameter, and the cached client keys its
// cache on hashQuery(sql, params). A per-request millisecond `now` would
// make that key unique every call, so the cache would never hit and this
// (hot, per-provider-candidate) lookup would query Postgres on every
// request. Keeping the SQL time-independent lets the cache key stay stable
// while expiry is still evaluated fresh on each call.
const rows = await cdb

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Invalidate discount cache on admin changes

With this lookup now using a stable cdb cache key, admin-created or deleted discounts can be ignored until the 60s Drizzle cache TTL expires, because the admin discount routes mutate tables.discount through the uncached db client (apps/api/src/routes/admin.ts inserts/deletes at lines 3165/3183 and 3273/3291), so RedisCache.onMutate is not invoked. In production this means a newly added discount may not be applied, or a removed discount may keep affecting gateway pricing/cost accounting, for requests whose org/provider/model tuple was already warmed in cache; invalidate the discount cache on these mutations or keep this enforcement/pricing lookup uncached.

Useful? React with 👍 / 👎.

.select({
id: discountTable.id,
organizationId: discountTable.organizationId,
provider: discountTable.provider,
model: discountTable.model,
discountPercent: discountTable.discountPercent,
expiresAt: discountTable.expiresAt,
})
.from(discountTable)
.where(
and(
notExpiredCondition,
or(
isNull(discountTable.organizationId),
organizationId
Expand All @@ -84,6 +84,15 @@ export async function getEffectiveDiscount(
),
);

const now = Date.now();
const discounts = rows.filter(
// expiresAt is a Date on both a fresh query and a Drizzle cache hit (the
// cache stores the raw pg result and re-applies the timestamp parser on
// restore). Wrap in new Date() defensively so the compare is robust even
// if a serialized value ever reaches here.
(d) => d.expiresAt === null || new Date(d.expiresAt).getTime() >= now,
);

const modelMatches = (discountModel: string | null): boolean =>
discountModel !== null && discountModel === model;

Expand Down
10 changes: 5 additions & 5 deletions packages/db/src/rate-limit-helpers.spec.ts
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
import { beforeEach, describe, expect, it, vi } from "vitest";

vi.mock("./db.js", () => ({
db: {
vi.mock("./cdb.js", () => ({
cdb: {
select: vi.fn(),
},
}));

const mockDb = await import("./db.js");
const mockCdb = await import("./cdb.js");

function createQueryMock(results: Array<Record<string, unknown>>) {
const chain = {
select: vi.fn().mockReturnThis(),
from: vi.fn().mockReturnThis(),
where: vi.fn().mockResolvedValue(results),
};
vi.mocked(mockDb.db.select).mockReturnValue(chain as never);
vi.mocked(mockCdb.cdb.select).mockReturnValue(chain as never);
return chain;
}

Expand Down Expand Up @@ -235,7 +235,7 @@ describe("getEffectiveRateLimit", () => {
});

it("propagates database errors so callers can apply SWR fallback", async () => {
vi.mocked(mockDb.db.select).mockImplementation(() => {
vi.mocked(mockCdb.cdb.select).mockImplementation(() => {
throw new Error("DB error");
});

Expand Down
9 changes: 6 additions & 3 deletions packages/db/src/rate-limit-helpers.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { and, eq, isNull, or } from "drizzle-orm";

import { db } from "./db.js";
import { cdb } from "./cdb.js";
import { rateLimit as rateLimitTable } from "./schema.js";

export type RateLimitSource =
Expand Down Expand Up @@ -149,7 +149,10 @@ function pickRateLimitByPrecedence(

/**
* Get the effective rate limits for a given organization, provider, and model.
* Uses the uncached database client so admin changes take effect immediately.
* Uses the cached database client (cdb) so this hot, per-request lookup is served
* from the Drizzle cache instead of hitting Postgres on every gateway request.
* The WHERE clause is time-independent, so the cache key stays stable. Admin
* changes propagate within the cache TTL (default 60s).
*
* Rate limits are always keyed by the root model ID — provider-specific model
* names are reserved for upstream requests and are never persisted as a
Expand All @@ -168,7 +171,7 @@ export async function getEffectiveRateLimit(
provider: string,
model: string,
): Promise<EffectiveRateLimit> {
const rateLimits = await db
const rateLimits = await cdb
.select({
Comment on lines +174 to 175

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Invalidate rate-limit cache on admin mutations

When an admin creates or deletes a rate limit to stop traffic, this cached lookup can keep returning the previous result because the admin handlers mutate tables.rateLimit through the uncached db client (for example apps/api/src/routes/admin.ts uses db.insert/db.delete for these routes), so RedisCache.onMutate is not invoked for the Drizzle cache. A gateway that has already cached “no limit” for an org/provider/model will continue allowing requests until the cache entry expires, regressing the previous immediate enforcement behavior for emergency throttles; either invalidate this cache from the admin mutations or keep this lookup uncached for enforcement-sensitive changes.

Useful? React with 👍 / 👎.

id: rateLimitTable.id,
organizationId: rateLimitTable.organizationId,
Expand Down
143 changes: 143 additions & 0 deletions scripts/loadtest.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
#!/usr/bin/env node
// Simple load test for api.llmgateway.io
// Fires `RATE` requests per second using parallel fetch() calls.

const API_URL =
process.env.API_URL || "https://api.llmgateway.io/v1/chat/completions";
const API_KEY = process.env.LLMGATEWAY_API_KEY;
const MODEL = process.env.MODEL || "deepseek-v4-flash";

if (!API_KEY) {
console.error("Missing LLMGATEWAY_API_KEY environment variable");
process.exit(1);
}

const RATE = Number(process.env.RATE) || 100; // requests per second
const DURATION_SEC = Number(process.env.DURATION_SEC) || 60; // how long to run

const payload = {
model: MODEL,
messages: [{ role: "user", content: "Reply with a short response: ping" }],
};

let sent = 0;
let ok = 0;
let failed = 0;
const latencies = []; // successful requests only
const errors = new Map(); // category -> { count, sampleId, sampleMs, sampleMsg }
const SAMPLES_PER_CATEGORY = 3; // live-log at most this many of each error category

function recordError(category, id, ms, msg) {
failed++;
let e = errors.get(category);
if (!e) {
e = { count: 0, sampleMsg: msg };
errors.set(category, e);
}
e.count++;
if (e.count <= SAMPLES_PER_CATEGORY) {
const t = ms != null ? ` ${ms.toFixed(0)}ms` : "";
console.error(`[${id}]${t} ${category}: ${msg}`);
} else if (e.count === SAMPLES_PER_CATEGORY + 1) {
console.error(
`[…] further "${category}" errors suppressed (counted in summary)`,
);
}
}

async function fireOne(id) {
const start = performance.now();
try {
const res = await fetch(API_URL, {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${API_KEY}`,
},
body: JSON.stringify(payload),
});
const ms = performance.now() - start;
if (res.ok) {
ok++;
latencies.push(ms);
} else {
const body = await res.text().catch(() => "");
recordError(
`HTTP ${res.status}`,
id,
ms,
body.replace(/\s+/g, " ").trim().slice(0, 200),
);
}
} catch (err) {
const ms = performance.now() - start;
// node/undici nests the useful reason in err.cause
const cause = err.cause
? `${err.cause.code || err.cause.message || err.cause}`
: "";
const category = `NETWORK ${err.cause?.code || err.code || err.message}`;
recordError(
category,
id,
ms,
[err.message, cause].filter(Boolean).join(" / "),
);
}
}

console.log(
`Load testing ${API_URL} @ ${RATE} r/s for ${DURATION_SEC}s (model: ${MODEL})`,
);

const inFlight = [];
function tick() {
for (let i = 0; i < RATE; i++) {
inFlight.push(fireOne(++sent));
}
}
tick(); // fire immediately at t=0
const ticker = setInterval(tick, 1000); // then 59 more ticks over 59s

setTimeout(async () => {
clearInterval(ticker);
await Promise.allSettled(inFlight);

const sorted = [...latencies].sort((a, b) => a - b);
const pct = (p) =>
sorted.length
? sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * p))]
: 0;
const avg = latencies.length
? latencies.reduce((a, b) => a + b, 0) / latencies.length
: 0;
const successRate = sent ? (ok / sent) * 100 : 0;
const achieved = ok / DURATION_SEC;

console.log("\n=== results ===");
console.log(
`sent: ${sent} (target ${RATE} r/s for ${DURATION_SEC}s)`,
);
console.log(`ok: ${ok} (${successRate.toFixed(1)}%)`);
console.log(`failed: ${failed} (${(100 - successRate).toFixed(1)}%)`);
console.log(`throughput: ${achieved.toFixed(1)} successful r/s`);

console.log(`\nlatency (ok only):`);
console.log(
` avg ${avg.toFixed(0)}ms | min ${pct(0).toFixed(0)}ms | p50 ${pct(0.5).toFixed(0)}ms | p95 ${pct(0.95).toFixed(0)}ms | p99 ${pct(0.99).toFixed(0)}ms | max ${pct(1).toFixed(0)}ms`,
);

if (errors.size) {
console.log(`\nerror breakdown:`);
const rows = [...errors.entries()].sort((a, b) => b[1].count - a[1].count);
for (const [category, e] of rows) {
const share = ((e.count / failed) * 100).toFixed(1);
console.log(
` ${String(e.count).padStart(6)} ${share.padStart(5)}% ${category}`,
);
if (e.sampleMsg)
console.log(` └─ e.g. ${e.sampleMsg.slice(0, 120)}`);
}
} else {
console.log(`\nno errors 🎉`);
}
}, DURATION_SEC * 1000);
Loading