diff --git a/bin/cli/commands/quota.mjs b/bin/cli/commands/quota.mjs index a657845e51ee..dee142db53c4 100644 --- a/bin/cli/commands/quota.mjs +++ b/bin/cli/commands/quota.mjs @@ -2,7 +2,7 @@ import { apiFetch, isServerUp } from "../api.mjs"; import { t } from "../i18n.mjs"; export function registerQuota(program) { - program + const quota = program .command("quota") .description(t("quota.description")) .option("--provider ", "Filter by provider") @@ -12,6 +12,60 @@ export function registerQuota(program) { const exitCode = await runQuotaCommand({ ...opts, output: globalOpts.output }); if (exitCode !== 0) process.exit(exitCode); }); + + quota + .command("status") + .description("Show truthful OmniRoute gateway, quota, pool, and circuit state") + .action(async (opts, cmd) => runBoundedJson("/api/omniroute/status", cmd.optsWithGlobals())); + + quota + .command("preview") + .description("Preview allocation enforcement without an upstream request") + .requiredOption("--api-key-id ", "API key id") + .requiredOption("--pool-id ", "quota pool id") + .option("--tokens ", "estimated token usage") + .action(async (opts, cmd) => { + const params = new URLSearchParams({ apiKeyId: opts.apiKeyId, poolId: opts.poolId }); + if (opts.tokens != null) params.set("estimatedTokens", opts.tokens); + await runBoundedJson(`/api/quota/preview?${params}`, cmd.optsWithGlobals()); + }); + + quota + .command("ensure ") + .description("Idempotently create or update a quota pool from a JSON object") + .action(async (json, opts, cmd) => { + let body; + try { + body = JSON.parse(json); + } catch { + console.error("Invalid pool JSON"); + process.exit(2); + } + await runBoundedJson("/api/quota/pools?ensure=true", cmd.optsWithGlobals(), { + method: "POST", + body, + }); + }); +} + +async function runBoundedJson(path, opts, request = {}) { + const started = performance.now(); + const res = await apiFetch(path, { + ...request, + retry: false, + timeout: Math.min(opts.timeout ?? 5000, 5000), + acceptNotOk: true, + }); + const elapsed = Math.round(performance.now() - started); + if (process.env.OMNIROUTE_DEBUG === "1") { + console.error(`[omniroute] ${request.method ?? "GET"} ${path} completed in ${elapsed}ms`); + } + const payload = await res.json().catch(() => ({ error: `HTTP ${res.status}` })); + if (!res.ok) { + console.error(JSON.stringify(payload)); + process.exit(res.exitCode ?? 1); + } + console.log(JSON.stringify(payload, null, 2)); } export async function runQuotaCommand(opts = {}) { diff --git a/bin/cli/utils/cliToken.mjs b/bin/cli/utils/cliToken.mjs index da504019a3b7..f6e2063d2cfb 100644 --- a/bin/cli/utils/cliToken.mjs +++ b/bin/cli/utils/cliToken.mjs @@ -8,13 +8,11 @@ let _cached = null; export async function getCliToken() { if (_cached !== null) return _cached; try { - const { machineIdSync } = await import("node-machine-id"); - const mid = machineIdSync(); - _cached = crypto - .createHash("sha256") - .update(mid + SALT) - .digest("hex") - .substring(0, 32); + const module = await import("node-machine-id"); + const machineIdSync = module.machineIdSync ?? module.default?.machineIdSync; + if (typeof machineIdSync !== "function") throw new Error("machine-id API unavailable"); + const mid = machineIdSync(true); + _cached = crypto.createHmac("sha256", mid).update(SALT).digest("hex"); } catch { _cached = ""; } diff --git a/docs/OMNIROUTE_ALLOCATION_HANDOFF.md b/docs/OMNIROUTE_ALLOCATION_HANDOFF.md new file mode 100644 index 000000000000..e523086c7c78 --- /dev/null +++ b/docs/OMNIROUTE_ALLOCATION_HANDOFF.md @@ -0,0 +1,9 @@ +# OmniRoute Allocation Handoff + +Allocation is not provider quota. + +Quota pools define which API keys may consume a provider pool and how hard, soft, or burst policies apply. Provider quota is external capacity reported by a provider or an explicitly configured source. Ghostlight internal budgets are governance limits defined by the administrator. + +The `ensurePool` operation is idempotent: an identical pool is unchanged, a changed allocation is updated, and a missing pool is created. This is intended for automation and bounded API callers. + +The read-only status endpoint is `GET /api/omniroute/status`. The verification command is `npm run omniroute:verify`; it makes no live model request. diff --git a/docs/OMNIROUTE_PROVIDER_FAILOVER.md b/docs/OMNIROUTE_PROVIDER_FAILOVER.md new file mode 100644 index 000000000000..4c0514b44881 --- /dev/null +++ b/docs/OMNIROUTE_PROVIDER_FAILOVER.md @@ -0,0 +1,9 @@ +# OmniRoute Provider Failover + +Failures are classified before retry decisions are made. + +Transient failures such as timeouts, network errors, rate limits, and provider 5xx responses may fail over. Authentication errors, permission errors, invalid requests, unavailable models, and unknown failures are not retried blindly. + +The default cross-provider policy allows up to three provider attempts, retries rate limits and timeouts, and keeps administrative disablement separate from temporary circuit state. + +Circuit states are `closed`, `open`, and `half_open`. A cooldown schedules a bounded probe; a successful probe closes the circuit and a failed probe reopens it. diff --git a/docs/OMNIROUTE_QUOTA_TELEMETRY.md b/docs/OMNIROUTE_QUOTA_TELEMETRY.md new file mode 100644 index 000000000000..5fe939034fd7 --- /dev/null +++ b/docs/OMNIROUTE_QUOTA_TELEMETRY.md @@ -0,0 +1,17 @@ +# OmniRoute Quota Telemetry + +OmniRoute separates provider quota telemetry from Ghostlight accounting. + +## Truthful states + +- `healthy` means a source reported usable remaining capacity. +- `approaching_limit` means a source reported remaining capacity at or below the configured threshold. +- `exhausted` is emitted only when a source reports zero capacity or usage at its limit. +- `unavailable` means a supported source failed to return data. +- `unknown` means no supported source exists or no provider limit is known. + +Unknown is not exhausted and does not disable a provider. + +Sources are preferred in this order: official provider API, authenticated usage API, explicitly mapped response headers, administrator configuration, local estimates, unknown. Local estimates are never presented as provider billing data. + +Response headers are parsed only through an explicit provider mapping. Generic header names are not assumed globally. diff --git a/docs/OMNIROUTE_ROUTING_POLICY.md b/docs/OMNIROUTE_ROUTING_POLICY.md new file mode 100644 index 000000000000..ee7ec45d876c --- /dev/null +++ b/docs/OMNIROUTE_ROUTING_POLICY.md @@ -0,0 +1,11 @@ +# OmniRoute Routing Policy + +Routing preserves the existing capability and combo selection logic, then applies allocation, health, circuit, quota, latency, reliability, model preference, and cost preference factors. + +The adaptive score is explainable and returns both the selected candidate and all ranked candidates. Exhausted quota, denied allocation, and open circuits are ineligible. Unknown quota remains eligible with a neutral quota factor. + +Route preview is deterministic and performs zero upstream model requests: + +`POST /api/omniroute/route/preview` + +The response includes candidate scores, factors, reasons, the selected provider, and `liveRequestExecuted: false`. diff --git a/package.json b/package.json index 951d417e733e..c8e80ab133b3 100644 --- a/package.json +++ b/package.json @@ -101,6 +101,7 @@ "build:secure": "OMNIROUTE_BUILD_PROFILE=minimal node scripts/build/build-next-isolated.mjs", "build:backend": "cross-env OMNIROUTE_BUILD_BACKEND_ONLY=1 node scripts/build/build-next-isolated.mjs", "build:cli": "node --import tsx scripts/build/prepublish.ts", + "omniroute:verify": "node scripts/check/omniroute-verify.mjs", "build:release": "rm -rf .build dist && OMNIROUTE_BUILD_SHA=$(git rev-parse --short HEAD) npm run build && npm run build:cli && node scripts/build/write-build-sha.mjs", "build:native:tproxy": "cd src/mitm/tproxy/native && npx --yes node-gyp rebuild", "start": "node scripts/dev/run-next.mjs start", diff --git a/scripts/check/omniroute-verify.mjs b/scripts/check/omniroute-verify.mjs new file mode 100644 index 000000000000..8082f499da82 --- /dev/null +++ b/scripts/check/omniroute-verify.mjs @@ -0,0 +1,69 @@ +#!/usr/bin/env node + +import { CLI_TOKEN_HEADER, getCliToken } from "../../bin/cli/utils/cliToken.mjs"; + +const baseUrl = (process.env.OMNIROUTE_BASE_URL || "http://127.0.0.1:20128").replace(/\/$/, ""); +const apiKey = process.env.OMNIROUTE_API_KEY || ""; +const timeoutMs = 5000; + +async function get(path) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + let hardTimer; + const hardTimeout = new Promise((_, reject) => { + hardTimer = setTimeout( + () => reject(new Error(`request timeout after ${timeoutMs}ms`)), + timeoutMs + 100 + ); + }); + try { + const response = await Promise.race([ + fetch(`${baseUrl}${path}`, { + headers: { + ...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}), + [CLI_TOKEN_HEADER]: await getCliToken(), + }, + signal: controller.signal, + }), + hardTimeout, + ]); + const body = await response.json().catch(() => null); + return { ok: response.ok, status: response.status, body }; + } finally { + clearTimeout(timer); + clearTimeout(hardTimer); + } +} + +function check(label, passed, detail = "") { + console.log(`${label}: ${passed ? "PASS" : "FAIL"}${detail ? ` (${detail})` : ""}`); + return passed; +} + +console.log("OmniRoute Verification"); +console.log(`Gateway: ${baseUrl}`); +const results = []; + +try { + const models = await get("/v1/models"); + results.push(check("Gateway", models.ok, `HTTP ${models.status}`)); + const modelCount = Array.isArray(models.body?.data) ? models.body.data.length : 0; + results.push(check("Catalog", modelCount > 0, `${modelCount} models`)); + + const pools = await get("/api/quota/pools"); + const poolRows = Array.isArray(pools.body?.pools) ? pools.body.pools : []; + const allocations = poolRows.reduce((sum, pool) => sum + (pool.allocations?.length || 0), 0); + results.push(check("Pools", pools.ok, `${poolRows.length}`)); + results.push(check("Allocations", pools.ok && allocations >= poolRows.length, `${allocations}`)); + + const status = await get("/api/omniroute/status"); + results.push(check("Status API", status.ok, `HTTP ${status.status}`)); + results.push(check("No live request", status.body?.liveRequestExecuted === false)); +} catch (error) { + results.push( + check("Verification", false, error instanceof Error ? error.message : String(error)) + ); +} + +console.log(`Live upstream requests: 0`); +if (results.some((passed) => !passed)) process.exitCode = 1; diff --git a/src/app/api/omniroute/route/preview/route.ts b/src/app/api/omniroute/route/preview/route.ts new file mode 100644 index 000000000000..0401ec97676c --- /dev/null +++ b/src/app/api/omniroute/route/preview/route.ts @@ -0,0 +1,36 @@ +import { NextResponse } from "next/server"; +import { z } from "zod"; +import { requireManagementAuth } from "@/lib/api/requireManagementAuth"; +import { rankCandidates } from "@/lib/routing/adaptiveRouting"; + +const candidateSchema = z.object({ + providerId: z.string().min(1), + modelId: z.string().min(1), + capabilityScore: z.number().min(0).max(1), + allocation: z.enum(["allow", "warn", "deny"]), + healthScore: z.number().min(0).max(1), + circuit: z.enum(["closed", "open", "half_open"]), + quota: z.enum(["healthy", "approaching_limit", "exhausted", "unavailable", "unknown"]), + latencyMs: z.number().nonnegative().optional(), + errorRate: z.number().min(0).max(1).optional(), + modelPreference: z.number().min(0).max(1).optional(), + costPreference: z.number().min(0).max(1).optional(), +}); + +const requestSchema = z.object({ candidates: z.array(candidateSchema).min(1).max(100) }); + +/** Deterministic routing preview. It never calls an upstream provider. */ +export async function POST(request: Request): Promise { + const authError = await requireManagementAuth(request); + if (authError) return authError; + const parsed = requestSchema.safeParse(await request.json().catch(() => null)); + if (!parsed.success) return NextResponse.json({ error: parsed.error.message }, { status: 400 }); + + const result = rankCandidates(parsed.data); + return NextResponse.json({ + request: { candidateCount: parsed.data.candidates.length }, + ...result, + selected: result.selected?.providerId ?? null, + liveRequestExecuted: false, + }); +} diff --git a/src/app/api/omniroute/status/route.ts b/src/app/api/omniroute/status/route.ts new file mode 100644 index 000000000000..2709e6f160e1 --- /dev/null +++ b/src/app/api/omniroute/status/route.ts @@ -0,0 +1,21 @@ +import { NextResponse } from "next/server"; +import { requireManagementAuth } from "@/lib/api/requireManagementAuth"; +import { buildOmniRouteStatus } from "@/lib/omnirouteStatus"; + +export const dynamic = "force-dynamic"; + +/** Read-only operational status; never performs an upstream model request. */ +export async function GET(request: Request): Promise { + const authError = await requireManagementAuth(request); + if (authError) return authError; + + try { + return NextResponse.json({ + generatedAt: new Date().toISOString(), + liveRequestExecuted: false, + ...(await buildOmniRouteStatus()), + }); + } catch { + return NextResponse.json({ error: "Failed to build OmniRoute status" }, { status: 500 }); + } +} diff --git a/src/app/api/quota/pools/route.ts b/src/app/api/quota/pools/route.ts index b669b1f4e5ef..81fee4211a3a 100644 --- a/src/app/api/quota/pools/route.ts +++ b/src/app/api/quota/pools/route.ts @@ -16,7 +16,7 @@ import { NextResponse } from "next/server"; import { buildErrorBody } from "@omniroute/open-sse/utils/error"; import { requireManagementAuth } from "@/lib/api/requireManagementAuth"; import { PoolCreateSchema } from "@/shared/schemas/quota"; -import { listPools, createPool } from "@/lib/localDb"; +import { listPools, createPool, ensurePool } from "@/lib/localDb"; import { logAuditEvent, getAuditRequestContext } from "@/lib/compliance/index"; export const dynamic = "force-dynamic"; @@ -48,17 +48,28 @@ export async function POST(request: Request): Promise { return NextResponse.json(buildErrorBody(400, parsed.error.message), { status: 400 }); } - const pool = createPool(parsed.data); + const ensure = new URL(request.url).searchParams.get("ensure") === "true"; + const ensured = ensure ? ensurePool(parsed.data) : null; + const pool = ensured?.pool ?? createPool(parsed.data); const ctx = getAuditRequestContext(request); logAuditEvent({ - action: "quota.pool.created", + action: ensured?.updated ? "quota.pool.updated" : "quota.pool.created", target: pool.id, - metadata: { connectionId: pool.connectionId, name: pool.name }, + metadata: { + connectionId: pool.connectionId, + name: pool.name, + ensure, + created: ensured?.created ?? true, + updated: ensured?.updated ?? false, + }, ipAddress: ctx.ipAddress ?? undefined, requestId: ctx.requestId, }); - return NextResponse.json({ pool }, { status: 201 }); + return NextResponse.json( + { pool, ...(ensured ? { created: ensured.created, updated: ensured.updated } : {}) }, + { status: ensured?.created === false ? 200 : 201 } + ); } catch (err) { const message = err instanceof Error ? err.message : "Failed to create pool"; return NextResponse.json(buildErrorBody(500, message), { status: 500 }); diff --git a/src/lib/db/quotaPools.ts b/src/lib/db/quotaPools.ts index 033f62d0480d..fbe50d3e33c4 100644 --- a/src/lib/db/quotaPools.ts +++ b/src/lib/db/quotaPools.ts @@ -50,7 +50,7 @@ async function removeQuotaCombosGuarded(poolId: string): Promise { } catch (err) { console.warn( "[quota-pools] removeQuotaCombosForPool failed (non-fatal):", - (err as Error)?.message, + (err as Error)?.message ); } } @@ -109,6 +109,12 @@ export interface PoolUpdate { connectionIds?: string[]; } +export interface EnsurePoolResult { + pool: QuotaPool; + created: boolean; + updated: boolean; +} + // --------------------------------------------------------------------------- // Internal helpers // --------------------------------------------------------------------------- @@ -145,7 +151,7 @@ function assertSingleProvider(connectionIds: string[]): void { const providers = rows.map((r) => r.provider).filter(Boolean); if (new Set(providers).size > 1) { throw new Error( - `A quota pool must use a single provider (got: ${[...new Set(providers)].join(", ")})`, + `A quota pool must use a single provider (got: ${[...new Set(providers)].join(", ")})` ); } } @@ -199,7 +205,7 @@ interface PoolConnectionRow { function getConnectionIds(poolId: string, fallbackConnectionId: string): string[] { const rows = getDb() .prepare( - "SELECT connection_id FROM quota_pool_connections WHERE pool_id = ? ORDER BY created_at ASC", + "SELECT connection_id FROM quota_pool_connections WHERE pool_id = ? ORDER BY created_at ASC" ) .all(poolId); if (rows.length > 0) { @@ -230,7 +236,7 @@ function batchBuildPools(rows: PoolRow[]): QuotaPool[] { // Batch allocations: 1 query for all pools const allocRows = db .prepare( - `SELECT pool_id, api_key_id, weight, cap_value, cap_unit, policy FROM quota_allocations WHERE pool_id IN (${ph})`, + `SELECT pool_id, api_key_id, weight, cap_value, cap_unit, policy FROM quota_allocations WHERE pool_id IN (${ph})` ) .all(...poolIds); const allocsByPool = new Map(); @@ -246,7 +252,7 @@ function batchBuildPools(rows: PoolRow[]): QuotaPool[] { // Batch connections: 1 query for all pools const connRows = db .prepare<{ pool_id: string; connection_id: string }>( - `SELECT pool_id, connection_id FROM quota_pool_connections WHERE pool_id IN (${ph}) ORDER BY created_at ASC`, + `SELECT pool_id, connection_id FROM quota_pool_connections WHERE pool_id IN (${ph}) ORDER BY created_at ASC` ) .all(...poolIds); const connsByPool = new Map(); @@ -273,7 +279,7 @@ function batchBuildPools(rows: PoolRow[]): QuotaPool[] { function getAllocations(poolId: string): PoolAllocation[] { const rows = getDb() .prepare( - "SELECT pool_id, api_key_id, weight, cap_value, cap_unit, policy FROM quota_allocations WHERE pool_id = ?", + "SELECT pool_id, api_key_id, weight, cap_value, cap_unit, policy FROM quota_allocations WHERE pool_id = ?" ) .all(poolId); return rows.map(rowToAllocation); @@ -344,7 +350,7 @@ export function listPools(options?: { limit?: number; offset?: number }): { export function getPool(id: string): QuotaPool | null { const row = getDb() .prepare( - "SELECT id, connection_id, name, group_id, created_at FROM quota_pools WHERE id = ?", + "SELECT id, connection_id, name, group_id, created_at FROM quota_pools WHERE id = ?" ) .get(id); if (!row) return null; @@ -378,12 +384,12 @@ export function createPool(input: PoolCreate): QuotaPool { const doCreate = database.transaction(() => { database .prepare( - "INSERT INTO quota_pools (id, connection_id, name, group_id, created_at) VALUES (?, ?, ?, ?, ?)", + "INSERT INTO quota_pools (id, connection_id, name, group_id, created_at) VALUES (?, ?, ?, ?, ?)" ) .run(id, primaryConnectionId, input.name, groupId, now); const insertConn = database.prepare( - "INSERT OR IGNORE INTO quota_pool_connections (pool_id, connection_id) VALUES (?, ?)", + "INSERT OR IGNORE INTO quota_pool_connections (pool_id, connection_id) VALUES (?, ?)" ); for (const connId of members) { insertConn.run(id, connId); @@ -392,7 +398,7 @@ export function createPool(input: PoolCreate): QuotaPool { if (input.allocations && input.allocations.length > 0) { const insertAlloc = database.prepare( `INSERT INTO quota_allocations (pool_id, api_key_id, weight, cap_value, cap_unit, policy) - VALUES (?, ?, ?, ?, ?, ?)`, + VALUES (?, ?, ?, ?, ?, ?)` ); for (const alloc of input.allocations) { insertAlloc.run( @@ -401,7 +407,7 @@ export function createPool(input: PoolCreate): QuotaPool { alloc.weight, alloc.capValue ?? null, alloc.capUnit ?? null, - alloc.policy, + alloc.policy ); } } @@ -416,7 +422,7 @@ export function createPool(input: PoolCreate): QuotaPool { group_id: groupId, created_at: now, }, - getAllocations(id), + getAllocations(id) ); // Phase B2: fire-and-forget combo sync; failures are logged but never thrown. @@ -427,6 +433,51 @@ export function createPool(input: PoolCreate): QuotaPool { return result; } +function allocationFingerprint(allocations: PoolAllocation[] = []): string { + return JSON.stringify( + allocations + .map((allocation) => ({ + apiKeyId: allocation.apiKeyId, + weight: allocation.weight, + capValue: allocation.capValue ?? null, + capUnit: allocation.capUnit ?? null, + policy: allocation.policy, + })) + .sort((left, right) => left.apiKeyId.localeCompare(right.apiKeyId)) + ); +} + +/** Idempotent pool management for automation and bounded CLI callers. */ +export function ensurePool(input: PoolCreate): EnsurePoolResult { + const members = + input.connectionIds && input.connectionIds.length > 0 + ? input.connectionIds + : [input.connectionId]; + const groupId = input.groupId || "group-demo"; + const existing = listPools().items.find((pool) => { + return pool.name === input.name && pool.groupId === groupId; + }); + + if (!existing) return { pool: createPool(input), created: true, updated: false }; + + const allocationsChanged = + input.allocations !== undefined && + allocationFingerprint(existing.allocations) !== allocationFingerprint(input.allocations); + const membersChanged = + existing.connectionIds.length !== members.length || + existing.connectionIds.some((id) => !members.includes(id)); + if (!allocationsChanged && !membersChanged) { + return { pool: existing, created: false, updated: false }; + } + + const update: PoolUpdate = { connectionIds: members }; + if (input.allocations !== undefined) update.allocations = input.allocations; + if (input.groupId !== undefined) update.groupId = input.groupId; + const updated = updatePool(existing.id, update); + if (!updated) throw new Error(`Quota pool disappeared during ensure: ${existing.id}`); + return { pool: updated, created: false, updated: true }; +} + /** * Update an existing pool's name, allocations, and/or member connections. * Returns updated pool, or null if pool not found. @@ -439,7 +490,7 @@ export function updatePool(id: string, input: PoolUpdate): QuotaPool | null { const database = getDb(); const existing = database .prepare( - "SELECT id, connection_id, name, group_id, created_at FROM quota_pools WHERE id = ?", + "SELECT id, connection_id, name, group_id, created_at FROM quota_pools WHERE id = ?" ) .get(id); if (!existing) return null; @@ -465,7 +516,7 @@ export function updatePool(id: string, input: PoolUpdate): QuotaPool | null { // Replace join rows. database.prepare("DELETE FROM quota_pool_connections WHERE pool_id = ?").run(id); const insertConn = database.prepare( - "INSERT OR IGNORE INTO quota_pool_connections (pool_id, connection_id) VALUES (?, ?)", + "INSERT OR IGNORE INTO quota_pool_connections (pool_id, connection_id) VALUES (?, ?)" ); for (const connId of input.connectionIds) { insertConn.run(id, connId); @@ -480,7 +531,7 @@ export function updatePool(id: string, input: PoolUpdate): QuotaPool | null { database.prepare("DELETE FROM quota_allocations WHERE pool_id = ?").run(id); const insertAlloc = database.prepare( `INSERT INTO quota_allocations (pool_id, api_key_id, weight, cap_value, cap_unit, policy) - VALUES (?, ?, ?, ?, ?, ?)`, + VALUES (?, ?, ?, ?, ?, ?)` ); for (const alloc of input.allocations) { insertAlloc.run( @@ -489,7 +540,7 @@ export function updatePool(id: string, input: PoolUpdate): QuotaPool | null { alloc.weight, alloc.capValue ?? null, alloc.capUnit ?? null, - alloc.policy, + alloc.policy ); } } @@ -597,7 +648,7 @@ export function upsertAllocations(poolId: string, allocations: PoolAllocation[]) // without requiring a manual re-save. Persists the normalized weights. const totalWeight = allocations.reduce( (s, a) => s + (Number.isFinite(a.weight) ? a.weight : 0), - 0, + 0 ); const normalizedAllocations = totalWeight === 0 && allocations.length > 0 @@ -608,7 +659,7 @@ export function upsertAllocations(poolId: string, allocations: PoolAllocation[]) // Defensive: fall back to [poolId] (single-pool semantics) if pool not found. const targetPool = database .prepare( - "SELECT id, connection_id, name, group_id, created_at FROM quota_pools WHERE id = ?", + "SELECT id, connection_id, name, group_id, created_at FROM quota_pools WHERE id = ?" ) .get(poolId); @@ -627,7 +678,7 @@ export function upsertAllocations(poolId: string, allocations: PoolAllocation[]) const doUpsert = database.transaction(() => { const insert = database.prepare( `INSERT INTO quota_allocations (pool_id, api_key_id, weight, cap_value, cap_unit, policy) - VALUES (?, ?, ?, ?, ?, ?)`, + VALUES (?, ?, ?, ?, ?, ?)` ); for (const pid of poolIdsInGroup) { database.prepare("DELETE FROM quota_allocations WHERE pool_id = ?").run(pid); @@ -638,7 +689,7 @@ export function upsertAllocations(poolId: string, allocations: PoolAllocation[]) alloc.weight, alloc.capValue ?? null, alloc.capUnit ?? null, - alloc.policy, + alloc.policy ); } } @@ -655,13 +706,13 @@ export function upsertAllocations(poolId: string, allocations: PoolAllocation[]) * Returns pairs of { poolId, allocation }. */ export function listAllocationsForApiKey( - apiKeyId: string, + apiKeyId: string ): Array<{ poolId: string; allocation: PoolAllocation }> { const rows = getDb() .prepare( `SELECT pool_id, api_key_id, weight, cap_value, cap_unit, policy FROM quota_allocations - WHERE api_key_id = ?`, + WHERE api_key_id = ?` ) .all(apiKeyId); return rows.map((row) => ({ poolId: row.pool_id, allocation: rowToAllocation(row) })); diff --git a/src/lib/localDb.ts b/src/lib/localDb.ts index 70e7eb04c3fd..c49af936de0f 100755 --- a/src/lib/localDb.ts +++ b/src/lib/localDb.ts @@ -614,6 +614,7 @@ export { listPools, getPool, getPoolsByGroup, + ensurePool, createPool, updatePool, deletePool, diff --git a/src/lib/omnirouteStatus.ts b/src/lib/omnirouteStatus.ts new file mode 100644 index 000000000000..74b8afdbaa51 --- /dev/null +++ b/src/lib/omnirouteStatus.ts @@ -0,0 +1,87 @@ +import { getDbInstance, pingDb } from "@/lib/db/core"; +import { listPools } from "@/lib/db/quotaPools"; + +interface ProviderStatusRow { + id: string; + provider: string; + is_active: number | boolean | null; + test_status: string | null; + last_error: string | null; +} + +function readProviderStatusRows(): ProviderStatusRow[] { + const db = getDbInstance(); + return db + .prepare( + "SELECT id, provider, is_active, test_status, last_error FROM provider_connections" + ) + .all(); +} + +export async function buildOmniRouteStatus() { + const [connections, circuitModule, quotaMonitorModule] = await Promise.all([ + Promise.resolve(readProviderStatusRows()), + import("@/shared/utils/circuitBreaker").catch(() => null), + import("../../open-sse/services/quotaMonitor").catch(() => null), + ]); + const pools = listPools().items; + const circuitStatuses = circuitModule?.getAllCircuitBreakerStatuses() ?? null; + const quotaSummary = quotaMonitorModule?.getQuotaMonitorSummary() ?? null; + const active = connections.filter( + (connection) => connection.is_active !== 0 && connection.is_active !== false + ); + const disabled = connections.filter( + (connection) => connection.is_active === 0 || connection.is_active === false + ); + const healthy = active.filter((connection) => connection.test_status === "active"); + + return { + gateway: pingDb() ? "healthy" : "degraded", + catalog: { available: true }, + providers: { + configured: connections.length, + active: active.length, + healthy: healthy.length, + disabled: disabled.length, + connections: connections.map((connection) => ({ + id: connection.id, + provider: connection.provider, + active: connection.is_active !== 0 && connection.is_active !== false, + health: + connection.is_active === 0 || connection.is_active === false + ? "disabled" + : connection.test_status === "active" + ? "healthy" + : "unknown", + failureState: connection.last_error ? "recent_error" : "none", + })), + }, + pools: { + count: pools.length, + allocations: pools.reduce((count, pool) => count + pool.allocations.length, 0), + items: pools.map((pool) => ({ + id: pool.id, + name: pool.name, + connectionIds: pool.connectionIds, + allocationCount: pool.allocations.length, + })), + }, + quotaMonitoring: { + authoritative: quotaSummary?.active ?? 0, + headerBased: 0, + configured: 0, + unsupported: quotaSummary ? Math.max(0, active.length - quotaSummary.active) : null, + status: quotaSummary?.active ? "partial" : "unknown", + }, + circuits: circuitStatuses + ? { + open: circuitStatuses.filter((c) => c.state === "OPEN").length, + halfOpen: circuitStatuses.filter((c) => c.state === "HALF_OPEN").length, + closed: circuitStatuses.filter((c) => c.state === "CLOSED").length, + source: "persisted circuit breaker registry", + } + : { status: "unknown", source: "resilience subsystem unavailable" }, + usage: { source: "usage_history and call_logs", liveRequestsExecuted: false }, + budgets: { source: "internal governance limits", upstreamQuotaClaims: false }, + }; +} diff --git a/src/lib/quota/providerCapabilities.ts b/src/lib/quota/providerCapabilities.ts new file mode 100644 index 000000000000..cc5ec32a7673 --- /dev/null +++ b/src/lib/quota/providerCapabilities.ts @@ -0,0 +1,37 @@ +export interface ProviderCapabilities { + providerId: string; + quotaApi: boolean; + usageApi: boolean; + rateLimitHeaders: boolean; + streaming: boolean; + toolUse: boolean; + coding: boolean; + vision: boolean; + longContext: boolean; +} + +const registry = new Map(); + +export function registerProviderCapabilities(capabilities: ProviderCapabilities): void { + registry.set(capabilities.providerId, { ...capabilities }); +} + +export function getProviderCapabilities(providerId: string): ProviderCapabilities { + return ( + registry.get(providerId) ?? { + providerId, + quotaApi: false, + usageApi: false, + rateLimitHeaders: false, + streaming: false, + toolUse: false, + coding: false, + vision: false, + longContext: false, + } + ); +} + +export function listProviderCapabilities(): ProviderCapabilities[] { + return [...registry.values()].map((capabilities) => ({ ...capabilities })); +} diff --git a/src/lib/quota/providerQuotaTelemetry.ts b/src/lib/quota/providerQuotaTelemetry.ts new file mode 100644 index 000000000000..cd9a29f2ad16 --- /dev/null +++ b/src/lib/quota/providerQuotaTelemetry.ts @@ -0,0 +1,248 @@ +/** Provider-neutral quota telemetry contracts and header normalization. */ + +export type QuotaDimensionName = + | "requests" + | "tokens" + | "input_tokens" + | "output_tokens" + | "credits" + | "currency" + | "daily_requests" + | "weekly_requests" + | "monthly_requests" + | "rate_limit" + | "unknown"; + +export type QuotaValueSource = + "provider_api" | "response_headers" | "configured" | "estimated" | "unknown"; + +export type QuotaConfidence = "authoritative" | "high" | "medium" | "low" | "unknown"; + +export interface QuotaValue { + dimension: QuotaDimensionName; + limit?: number; + used?: number; + remaining?: number; + resetAt?: string; + unit?: string; + source: QuotaValueSource; + confidence: QuotaConfidence; +} + +export type ProviderQuotaStatus = + "healthy" | "approaching_limit" | "exhausted" | "unavailable" | "unknown"; + +export interface ProviderQuotaState { + providerId: string; + connectionId: string; + supported: boolean; + fetchedAt: string; + values: QuotaValue[]; + status: ProviderQuotaStatus; + error?: string; +} + +export interface ProviderConnectionForQuota { + id: string; + provider: string; + [key: string]: unknown; +} + +export interface ProviderQuotaMonitor { + providerId: string; + supportedDimensions(): Promise; + fetchQuotaState(connection: ProviderConnectionForQuota): Promise; +} + +export type QuotaSourceKind = + "provider_api" | "response_headers" | "configured" | "estimated" | "unknown"; + +export interface QuotaSourceAdapter { + kind: QuotaSourceKind; + supports(providerId: string): boolean; + read(connection: ProviderConnectionForQuota): Promise; +} + +const SOURCE_PRIORITY: QuotaSourceKind[] = [ + "provider_api", + "response_headers", + "configured", + "estimated", + "unknown", +]; + +function sourceRank(source: QuotaSourceKind): number { + return SOURCE_PRIORITY.indexOf(source); +} + +function finite(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value); +} + +function statusForValues(values: QuotaValue[], approachingThreshold: number): ProviderQuotaStatus { + if (values.length === 0) return "unknown"; + + const exhausted = values.some( + (value) => + (finite(value.remaining) && value.remaining <= 0) || + (finite(value.used) && finite(value.limit) && value.used >= value.limit) + ); + if (exhausted) return "exhausted"; + + const approaching = values.some((value) => { + if (!finite(value.remaining) || !finite(value.limit) || value.limit <= 0) return false; + return value.remaining / value.limit <= approachingThreshold; + }); + return approaching ? "approaching_limit" : "healthy"; +} + +export function unknownQuotaState( + providerId: string, + connectionId: string, + fetchedAt = new Date().toISOString(), + error?: string +): ProviderQuotaState { + return { + providerId, + connectionId, + supported: false, + fetchedAt, + values: [], + status: "unknown", + ...(error ? { error } : {}), + }; +} + +/** + * Reads the best available source without treating missing provider data as exhaustion. + * Sources are selected per dimension, so a provider API can coexist with header data. + */ +export async function collectQuotaState( + connection: ProviderConnectionForQuota, + adapters: QuotaSourceAdapter[], + options: { approachingThreshold?: number; fetchedAt?: string } = {} +): Promise { + const fetchedAt = options.fetchedAt ?? new Date().toISOString(); + const byDimension = new Map(); + let supported = false; + let lastError: string | undefined; + + for (const adapter of adapters) { + if (!adapter.supports(connection.provider)) continue; + supported = true; + try { + const values = await adapter.read(connection); + for (const value of values) { + const current = byDimension.get(value.dimension); + if (!current || sourceRank(value.source) < sourceRank(current.source)) { + byDimension.set(value.dimension, value); + } + } + } catch (error) { + lastError = error instanceof Error ? error.message : String(error); + } + } + + const values = [...byDimension.values()]; + return { + providerId: connection.provider, + connectionId: connection.id, + supported, + fetchedAt, + values, + status: + values.length > 0 + ? statusForValues(values, options.approachingThreshold ?? 0.2) + : supported + ? "unavailable" + : "unknown", + ...(lastError ? { error: lastError } : {}), + }; +} + +export interface RateLimitHeaderMapping { + limit?: string; + remaining?: string; + reset?: string; + retryAfter?: string; + dimension?: QuotaDimensionName; + unit?: string; +} + +export interface RateLimitSnapshot { + providerId: string; + connectionId: string; + capturedAt: string; + requestLimit?: number; + requestsRemaining?: number; + resetAt?: string; + retryAfterSeconds?: number; + source: "response_headers"; +} + +function headerValue(headers: Headers | Record, name?: string) { + if (!name) return undefined; + if (headers instanceof Headers) + return headers.get(name) ?? headers.get(name.toLowerCase()) ?? undefined; + const lower = name.toLowerCase(); + const key = Object.keys(headers).find((candidate) => candidate.toLowerCase() === lower); + return key ? headers[key] : undefined; +} + +function numberHeader(value: string | undefined): number | undefined { + if (value === undefined || value.trim() === "") return undefined; + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : undefined; +} + +function resetHeaderToIso(value: string | undefined): string | undefined { + const parsed = numberHeader(value); + if (parsed === undefined) + return value && !Number.isNaN(Date.parse(value)) ? new Date(value).toISOString() : undefined; + const milliseconds = parsed > 10_000_000_000 ? parsed : parsed * 1000; + return new Date(milliseconds).toISOString(); +} + +export function parseRateLimitHeaders( + headers: Headers | Record, + providerId: string, + connectionId: string, + mapping: RateLimitHeaderMapping, + capturedAt = new Date().toISOString() +): { value: QuotaValue; snapshot: RateLimitSnapshot } | null { + const limit = numberHeader(headerValue(headers, mapping.limit)); + const remaining = numberHeader(headerValue(headers, mapping.remaining)); + const resetAt = resetHeaderToIso(headerValue(headers, mapping.reset)); + const retryAfterSeconds = numberHeader(headerValue(headers, mapping.retryAfter)); + if ( + limit === undefined && + remaining === undefined && + !resetAt && + retryAfterSeconds === undefined + ) { + return null; + } + + const dimension = mapping.dimension ?? "rate_limit"; + return { + value: { + dimension, + ...(limit !== undefined ? { limit } : {}), + ...(remaining !== undefined ? { remaining } : {}), + ...(resetAt ? { resetAt } : {}), + ...(mapping.unit ? { unit: mapping.unit } : {}), + source: "response_headers", + confidence: "high", + }, + snapshot: { + providerId, + connectionId, + capturedAt, + ...(limit !== undefined ? { requestLimit: limit } : {}), + ...(remaining !== undefined ? { requestsRemaining: remaining } : {}), + ...(resetAt ? { resetAt } : {}), + ...(retryAfterSeconds !== undefined ? { retryAfterSeconds } : {}), + source: "response_headers", + }, + }; +} diff --git a/src/lib/resilience/adaptiveCircuit.ts b/src/lib/resilience/adaptiveCircuit.ts new file mode 100644 index 000000000000..4d0282a93daf --- /dev/null +++ b/src/lib/resilience/adaptiveCircuit.ts @@ -0,0 +1,60 @@ +export type AdaptiveCircuitState = "closed" | "open" | "half_open"; + +export interface AdaptiveCircuit { + state: AdaptiveCircuitState; + failureCount: number; + successCount: number; + lastFailureAt?: string; + openedAt?: string; + halfOpenAt?: string; + nextProbeAt?: string; + reason?: string; +} + +export function createAdaptiveCircuit(): AdaptiveCircuit { + return { state: "closed", failureCount: 0, successCount: 0 }; +} + +export function observeCircuit( + current: AdaptiveCircuit, + event: "failure" | "success" | "probe", + options: { now?: Date; failureThreshold?: number; cooldownMs?: number; reason?: string } = {} +): AdaptiveCircuit { + const now = options.now ?? new Date(); + const failureThreshold = options.failureThreshold ?? 3; + const cooldownMs = options.cooldownMs ?? 60_000; + const next = { ...current }; + + if (event === "failure") { + next.failureCount += 1; + next.successCount = 0; + next.lastFailureAt = now.toISOString(); + next.reason = options.reason; + if (next.state === "half_open" || next.failureCount >= failureThreshold) { + next.state = "open"; + next.openedAt = now.toISOString(); + next.nextProbeAt = new Date(now.getTime() + cooldownMs).toISOString(); + next.halfOpenAt = undefined; + } + return next; + } + + if (event === "probe") { + if (next.state === "open" && (!next.nextProbeAt || now >= new Date(next.nextProbeAt))) { + next.state = "half_open"; + next.halfOpenAt = now.toISOString(); + } + return next; + } + + next.successCount += 1; + if (next.state === "half_open" || next.successCount >= 1) { + next.state = "closed"; + next.failureCount = 0; + next.openedAt = undefined; + next.halfOpenAt = undefined; + next.nextProbeAt = undefined; + next.reason = undefined; + } + return next; +} diff --git a/src/lib/resilience/failureClassification.ts b/src/lib/resilience/failureClassification.ts new file mode 100644 index 000000000000..8bea63b82b3e --- /dev/null +++ b/src/lib/resilience/failureClassification.ts @@ -0,0 +1,75 @@ +export type ProviderFailureType = + | "authentication_error" + | "rate_limit" + | "quota_exhausted" + | "timeout" + | "network_error" + | "provider_5xx" + | "invalid_request" + | "model_unavailable" + | "permission_error" + | "unknown"; + +export interface ProviderFailure { + type: ProviderFailureType; + retryable: boolean; + providerId: string; + connectionId?: string; + statusCode?: number; + retryAfter?: number; + message: string; +} + +export function classifyProviderFailure(input: { + providerId: string; + connectionId?: string; + statusCode?: number; + code?: string; + message?: string; + retryAfter?: number; +}): ProviderFailure { + const message = input.message?.trim() || "Provider request failed"; + const normalized = `${input.code ?? ""} ${message}`.toLowerCase(); + const status = input.statusCode; + let type: ProviderFailureType = "unknown"; + let retryable = false; + + if ( + status === 401 || + status === 403 || + /invalid.*(key|token)|unauthori[sz]ed|forbidden/.test(normalized) + ) { + type = + status === 403 || normalized.includes("permission") + ? "permission_error" + : "authentication_error"; + } else if (status === 408 || status === 504 || /timeout|timed out|etimedout/.test(normalized)) { + type = "timeout"; + retryable = true; + } else if (status === 429 || /rate.?limit|too many requests|retry.?after/.test(normalized)) { + type = /quota|insufficient balance|credits exhausted|balance is \$0/.test(normalized) + ? "quota_exhausted" + : "rate_limit"; + retryable = type === "rate_limit"; + } else if (status !== undefined && status >= 500) { + type = "provider_5xx"; + retryable = true; + } else if (status === 400 || /invalid request|malformed|unsupported parameter/.test(normalized)) { + type = "invalid_request"; + } else if (status === 404 || /model unavailable|model not found|unknown model/.test(normalized)) { + type = "model_unavailable"; + } else if (/network|econnreset|econnrefused|enotfound|fetch failed|socket/.test(normalized)) { + type = "network_error"; + retryable = true; + } + + return { + type, + retryable, + providerId: input.providerId, + ...(input.connectionId ? { connectionId: input.connectionId } : {}), + ...(status !== undefined ? { statusCode: status } : {}), + ...(input.retryAfter !== undefined ? { retryAfter: input.retryAfter } : {}), + message, + }; +} diff --git a/src/lib/routing/adaptiveRouting.ts b/src/lib/routing/adaptiveRouting.ts new file mode 100644 index 000000000000..cfbaffbdf1ab --- /dev/null +++ b/src/lib/routing/adaptiveRouting.ts @@ -0,0 +1,153 @@ +import type { ProviderFailure } from "@/lib/resilience/failureClassification"; +import type { ProviderQuotaStatus } from "@/lib/quota/providerQuotaTelemetry"; + +export type AllocationDecision = "allow" | "warn" | "deny"; +export type CircuitState = "closed" | "open" | "half_open"; + +export interface RoutingCandidate { + providerId: string; + modelId: string; + capabilityScore: number; + allocation: AllocationDecision; + healthScore: number; + circuit: CircuitState; + quota: ProviderQuotaStatus; + latencyMs?: number; + errorRate?: number; + modelPreference?: number; + costPreference?: number; +} + +export interface RoutingExplanation { + providerId: string; + modelId: string; + score: number; + eligible: boolean; + reasons: string[]; + factors: Record; +} + +export interface RankedRoutingResult { + selected: RoutingExplanation | null; + candidates: RoutingExplanation[]; +} + +function clamp(value: number, fallback = 0): number { + return Number.isFinite(value) ? Math.max(0, Math.min(1, value)) : fallback; +} + +function quotaFactor(quota: ProviderQuotaStatus): number { + switch (quota) { + case "exhausted": + return 0; + case "approaching_limit": + return 0.65; + case "unavailable": + return 0.9; + case "unknown": + return 1; + case "healthy": + return 1; + default: + return 1; + } +} + +function latencyFactor(latencyMs?: number): number { + if (!Number.isFinite(latencyMs) || latencyMs === undefined) return 1; + return Math.max(0.4, 1 - Math.min(latencyMs, 30_000) / 50_000); +} + +export function scoreCandidate(candidate: RoutingCandidate): RoutingExplanation { + const capability = clamp(candidate.capabilityScore); + const allocation = + candidate.allocation === "deny" ? 0 : candidate.allocation === "warn" ? 0.85 : 1; + const health = clamp(candidate.healthScore, 0.5); + const reliability = 1 - clamp(candidate.errorRate ?? 0); + const latency = latencyFactor(candidate.latencyMs); + const preference = clamp(candidate.modelPreference, 0.5); + const cost = clamp(candidate.costPreference, 1); + const quota = quotaFactor(candidate.quota); + const circuit = candidate.circuit === "open" ? 0 : candidate.circuit === "half_open" ? 0.5 : 1; + const score = Number( + ( + capability * + allocation * + health * + reliability * + latency * + preference * + cost * + quota * + circuit + ).toFixed(6) + ); + const reasons = [ + capability >= 0.8 ? "capability match" : "partial capability match", + candidate.allocation === "allow" + ? "allocation permitted" + : candidate.allocation === "warn" + ? "allocation permitted with warning" + : "allocation denied", + health >= 0.8 ? "provider healthy" : "provider health degraded", + `circuit ${candidate.circuit}`, + candidate.quota === "unknown" + ? "quota state unknown but not exhausted" + : `quota ${candidate.quota}`, + ]; + if (candidate.latencyMs !== undefined) + reasons.push(`latency ${Math.round(candidate.latencyMs)}ms`); + if (candidate.errorRate !== undefined) + reasons.push(`${Math.round(candidate.errorRate * 100)}% recent errors`); + return { + providerId: candidate.providerId, + modelId: candidate.modelId, + score, + eligible: + score > 0 && + candidate.allocation !== "deny" && + candidate.circuit !== "open" && + candidate.quota !== "exhausted", + reasons, + factors: { + capability, + allocation, + health, + reliability, + latency, + preference, + cost, + quota, + circuit, + }, + }; +} + +export function rankCandidates(candidates: RoutingCandidate[]): RankedRoutingResult { + const ranked = candidates.map(scoreCandidate).sort((a, b) => b.score - a.score); + return { selected: ranked.find((candidate) => candidate.eligible) ?? null, candidates: ranked }; +} + +export interface FailoverPolicy { + maxProviderAttempts: number; + allowCrossProviderFallback: boolean; + retryRateLimited: boolean; + retryTimeouts: boolean; +} + +export const DEFAULT_FAILOVER_POLICY: FailoverPolicy = { + maxProviderAttempts: 3, + allowCrossProviderFallback: true, + retryRateLimited: true, + retryTimeouts: true, +}; + +export function shouldFailover( + failure: ProviderFailure, + policy = DEFAULT_FAILOVER_POLICY +): boolean { + if (!policy.allowCrossProviderFallback || !failure.retryable) return false; + if (failure.type === "rate_limit") return policy.retryRateLimited; + if (failure.type === "timeout") return policy.retryTimeouts; + return failure.type === "network_error" || failure.type === "provider_5xx"; +} diff --git a/src/lib/usage/budgetGuard.ts b/src/lib/usage/budgetGuard.ts new file mode 100644 index 000000000000..034c3c9da390 --- /dev/null +++ b/src/lib/usage/budgetGuard.ts @@ -0,0 +1,66 @@ +export type BudgetPeriod = "daily" | "weekly" | "monthly"; +export type BudgetDecision = "allow" | "warn" | "deny"; + +export interface InternalBudgetLimit { + id: string; + scope: "global" | "provider" | "model" | "pool"; + providerId?: string; + modelId?: string; + poolId?: string; + period: BudgetPeriod; + limitType: "currency" | "tokens" | "requests"; + limitValue: number; + warningThreshold: number; + enabled: boolean; +} + +export interface BudgetUsage { + currency: number; + tokens: number; + requests: number; +} + +export interface BudgetEvaluation { + decision: BudgetDecision; + limit?: InternalBudgetLimit; + used: number; + remaining?: number; + reason: string; +} + +export function evaluateBudget( + limit: InternalBudgetLimit | undefined, + usage: BudgetUsage +): BudgetEvaluation { + if (!limit || !limit.enabled) + return { decision: "allow", used: 0, reason: "No enabled internal budget applies." }; + const used = usage[limit.limitType]; + if (!Number.isFinite(used) || limit.limitValue <= 0) { + return { + decision: "deny", + limit, + used: 0, + remaining: 0, + reason: "Internal budget configuration is invalid.", + }; + } + const remaining = Math.max(0, limit.limitValue - used); + if (used >= limit.limitValue) + return { decision: "deny", limit, used, remaining, reason: "Internal budget exhausted." }; + if (used / limit.limitValue >= limit.warningThreshold) { + return { + decision: "warn", + limit, + used, + remaining, + reason: `Internal budget is ${Math.round((used / limit.limitValue) * 100)}% consumed.`, + }; + } + return { + decision: "allow", + limit, + used, + remaining, + reason: "Internal budget permits the request.", + }; +} diff --git a/src/lib/usage/modelPricingRegistry.ts b/src/lib/usage/modelPricingRegistry.ts new file mode 100644 index 000000000000..fae70367f8bc --- /dev/null +++ b/src/lib/usage/modelPricingRegistry.ts @@ -0,0 +1,52 @@ +export interface ModelPricing { + providerId: string; + modelId: string; + inputPricePerMillionTokens?: number; + outputPricePerMillionTokens?: number; + source?: string; + effectiveAt?: string; +} + +export class ModelPricingRegistry { + private readonly entries = new Map(); + + private key(providerId: string, modelId: string): string { + return `${providerId.trim().toLowerCase()}\0${modelId.trim().toLowerCase()}`; + } + + set(pricing: ModelPricing): void { + this.entries.set(this.key(pricing.providerId, pricing.modelId), { ...pricing }); + } + + get(providerId: string, modelId: string): ModelPricing | undefined { + return this.entries.get(this.key(providerId, modelId)); + } + + estimate( + providerId: string, + modelId: string, + inputTokens = 0, + outputTokens = 0 + ): number | undefined { + const pricing = this.get(providerId, modelId); + if (!pricing) return undefined; + const input = + pricing.inputPricePerMillionTokens === undefined + ? 0 + : (inputTokens * pricing.inputPricePerMillionTokens) / 1_000_000; + const output = + pricing.outputPricePerMillionTokens === undefined + ? 0 + : (outputTokens * pricing.outputPricePerMillionTokens) / 1_000_000; + if ( + pricing.inputPricePerMillionTokens === undefined && + pricing.outputPricePerMillionTokens === undefined + ) + return undefined; + return Number((input + output).toFixed(12)); + } + + list(): ModelPricing[] { + return [...this.entries.values()].map((entry) => ({ ...entry })); + } +} diff --git a/src/lib/usage/usageLedger.ts b/src/lib/usage/usageLedger.ts new file mode 100644 index 000000000000..4f5f47c12cc4 --- /dev/null +++ b/src/lib/usage/usageLedger.ts @@ -0,0 +1,78 @@ +import type { ModelPricingRegistry } from "./modelPricingRegistry.js"; + +export type UsageStatus = "success" | "failed" | "rate_limited" | "timeout" | "cancelled"; + +export interface UsageRecord { + id: string; + providerId: string; + connectionId?: string; + modelId: string; + poolId?: string; + allocation?: number; + inputTokens?: number; + outputTokens?: number; + totalTokens?: number; + estimatedCostUsd?: number; + latencyMs: number; + status: UsageStatus; + createdAt: string; +} + +export interface UsageRecordInput extends Omit { + totalTokens?: number; + estimatedCostUsd?: number; +} + +export function createUsageRecord( + input: UsageRecordInput, + pricing?: ModelPricingRegistry +): UsageRecord { + const inputTokens = Number.isFinite(input.inputTokens) ? input.inputTokens : undefined; + const outputTokens = Number.isFinite(input.outputTokens) ? input.outputTokens : undefined; + const totalTokens = + input.totalTokens ?? + (inputTokens !== undefined || outputTokens !== undefined + ? (inputTokens ?? 0) + (outputTokens ?? 0) + : undefined); + const estimatedCostUsd = + input.estimatedCostUsd ?? + (pricing && inputTokens !== undefined && outputTokens !== undefined + ? pricing.estimate(input.providerId, input.modelId, inputTokens, outputTokens) + : undefined); + return { + ...input, + ...(inputTokens !== undefined ? { inputTokens } : {}), + ...(outputTokens !== undefined ? { outputTokens } : {}), + ...(totalTokens !== undefined ? { totalTokens } : {}), + ...(estimatedCostUsd !== undefined ? { estimatedCostUsd } : {}), + }; +} + +export function summarizeUsage(records: UsageRecord[]) { + return records.reduce( + (summary, record) => { + summary.requests += 1; + summary.successes += record.status === "success" ? 1 : 0; + summary.failures += record.status === "success" ? 0 : 1; + summary.inputTokens += record.inputTokens ?? 0; + summary.outputTokens += record.outputTokens ?? 0; + summary.totalTokens += record.totalTokens ?? 0; + summary.estimatedCostUsd = + summary.estimatedCostUsd === undefined || record.estimatedCostUsd === undefined + ? undefined + : summary.estimatedCostUsd + record.estimatedCostUsd; + summary.latencyMs += record.latencyMs; + return summary; + }, + { + requests: 0, + successes: 0, + failures: 0, + inputTokens: 0, + outputTokens: 0, + totalTokens: 0, + estimatedCostUsd: 0 as number | undefined, + latencyMs: 0, + } + ); +} diff --git a/tests/unit/adaptive-circuit-budget-ledger.test.ts b/tests/unit/adaptive-circuit-budget-ledger.test.ts new file mode 100644 index 000000000000..3ea22a4829b4 --- /dev/null +++ b/tests/unit/adaptive-circuit-budget-ledger.test.ts @@ -0,0 +1,77 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { observeCircuit, createAdaptiveCircuit } from "@/lib/resilience/adaptiveCircuit"; +import { evaluateBudget } from "@/lib/usage/budgetGuard"; +import { ModelPricingRegistry } from "@/lib/usage/modelPricingRegistry"; +import { createUsageRecord, summarizeUsage } from "@/lib/usage/usageLedger"; + +test("adaptive circuit opens, probes, and closes after recovery", () => { + const now = new Date("2026-01-01T00:00:00.000Z"); + let circuit = createAdaptiveCircuit(); + circuit = observeCircuit(circuit, "failure", { + now, + failureThreshold: 2, + cooldownMs: 1000, + reason: "timeout", + }); + circuit = observeCircuit(circuit, "failure", { + now: new Date(now.getTime() + 10), + failureThreshold: 2, + cooldownMs: 1000, + reason: "timeout", + }); + assert.equal(circuit.state, "open"); + circuit = observeCircuit(circuit, "probe", { now: new Date(now.getTime() + 1011) }); + assert.equal(circuit.state, "half_open"); + circuit = observeCircuit(circuit, "success", { now: new Date(now.getTime() + 1002) }); + assert.equal(circuit.state, "closed"); + assert.equal(circuit.failureCount, 0); +}); + +test("internal budget returns allow, warn, and deny without upstream quota claims", () => { + const limit = { + id: "b", + scope: "global" as const, + period: "daily" as const, + limitType: "currency" as const, + limitValue: 10, + warningThreshold: 0.75, + enabled: true, + }; + assert.equal(evaluateBudget(limit, { currency: 2, tokens: 0, requests: 0 }).decision, "allow"); + assert.equal(evaluateBudget(limit, { currency: 8, tokens: 0, requests: 0 }).decision, "warn"); + assert.equal(evaluateBudget(limit, { currency: 10, tokens: 0, requests: 0 }).decision, "deny"); +}); + +test("unknown pricing remains unknown while configured pricing is estimated", () => { + const registry = new ModelPricingRegistry(); + assert.equal(registry.estimate("codex", "unknown", 1000, 1000), undefined); + registry.set({ + providerId: "codex", + modelId: "gpt-5", + inputPricePerMillionTokens: 1, + outputPricePerMillionTokens: 2, + source: "admin", + }); + assert.equal(registry.estimate("codex", "gpt-5", 1000, 1000), 0.003); + const record = createUsageRecord( + { + id: "r1", + providerId: "codex", + modelId: "gpt-5", + inputTokens: 100, + outputTokens: 50, + latencyMs: 10, + status: "success", + createdAt: "2026-01-01T00:00:00.000Z", + }, + registry + ); + assert.equal(record.totalTokens, 150); + assert.equal(record.estimatedCostUsd, 0.0002); + const summary = summarizeUsage([record]); + assert.equal(summary.requests, 1); + assert.equal(summary.successes, 1); + assert.equal(summary.estimatedCostUsd, 0.0002); +}); diff --git a/tests/unit/cli-machine-token.test.ts b/tests/unit/cli-machine-token.test.ts index 8b8d6f762a56..97591e967f09 100644 --- a/tests/unit/cli-machine-token.test.ts +++ b/tests/unit/cli-machine-token.test.ts @@ -9,12 +9,13 @@ test("cliToken.mjs pode ser importado sem erro", async () => { assert.equal(mod.CLI_TOKEN_HEADER, "x-omniroute-cli-token"); }); -test("getCliToken retorna string de 32 chars ou string vazia", async () => { +test("getCliToken retorna string de 64 chars ou string vazia", async () => { const { getCliToken } = await import("../../bin/cli/utils/cliToken.mjs"); const token = await getCliToken(); assert.ok(typeof token === "string"); - // Pode ser "" se node-machine-id falhar, ou 32 chars se funcionar. - assert.ok(token === "" || token.length === 32, `expected 0 or 32 chars, got ${token.length}`); + // Pode ser "" se node-machine-id falhar, ou 64 chars se funcionar + // (HMAC-SHA256 digest hex — see #10148 cliToken hardening). + assert.ok(token === "" || token.length === 64, `expected 0 or 64 chars, got ${token.length}`); }); test("getCliToken retorna mesmo valor em chamadas repetidas (cache)", async () => { @@ -28,7 +29,8 @@ test("getCliToken produz apenas hex lowercase se não-vazio", async () => { const { getCliToken } = await import("../../bin/cli/utils/cliToken.mjs"); const token = await getCliToken(); if (token.length > 0) { - assert.match(token, /^[0-9a-f]{32}$/); + // HMAC-SHA256 digest hex = 64 chars (#10148 cliToken hardening). + assert.match(token, /^[0-9a-f]{64}$/); } }); @@ -70,17 +72,13 @@ test("isLoopback rejeita IP público", async () => { test("token derivado de machine-id diferente produz hash diferente", () => { const SALT = "omniroute-cli-auth-v1"; - const hash = (mid: string) => - crypto - .createHash("sha256") - .update(mid + SALT) - .digest("hex") - .substring(0, 32); + // Mirror the production derivation (#10148): HMAC-SHA256(machineId, SALT) hex. + const hash = (mid: string) => crypto.createHmac("sha256", mid).update(SALT).digest("hex"); const t1 = hash("machine-id-host-A"); const t2 = hash("machine-id-host-B"); assert.notEqual(t1, t2); - assert.match(t1, /^[0-9a-f]{32}$/); - assert.match(t2, /^[0-9a-f]{32}$/); + assert.match(t1, /^[0-9a-f]{64}$/); + assert.match(t2, /^[0-9a-f]{64}$/); }); test("OMNIROUTE_DISABLE_CLI_TOKEN desabilita auth (estrutura verificada)", async () => { diff --git a/tests/unit/quota-telemetry-adaptive-routing.test.ts b/tests/unit/quota-telemetry-adaptive-routing.test.ts new file mode 100644 index 000000000000..2f205a62de3a --- /dev/null +++ b/tests/unit/quota-telemetry-adaptive-routing.test.ts @@ -0,0 +1,149 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + collectQuotaState, + parseRateLimitHeaders, + unknownQuotaState, +} from "@/lib/quota/providerQuotaTelemetry"; +import { classifyProviderFailure } from "@/lib/resilience/failureClassification"; +import { rankCandidates, shouldFailover } from "@/lib/routing/adaptiveRouting"; + +test("unknown quota remains unknown and does not disable routing", async () => { + const state = await collectQuotaState({ id: "c1", provider: "codex" }, []); + assert.equal(state.supported, false); + assert.equal(state.status, "unknown"); + assert.deepEqual(state.values, []); + + const ranked = rankCandidates([ + { + providerId: "codex", + modelId: "gpt-5", + capabilityScore: 1, + allocation: "allow", + healthScore: 1, + circuit: "closed", + quota: state.status, + }, + ]); + assert.equal(ranked.selected?.providerId, "codex"); +}); + +test("quota source priority prefers authoritative values per dimension", async () => { + const state = await collectQuotaState({ id: "c1", provider: "codex" }, [ + { + kind: "estimated", + supports: () => true, + read: async () => [ + { dimension: "requests", remaining: 1, source: "estimated", confidence: "low" }, + ], + }, + { + kind: "provider_api", + supports: () => true, + read: async () => [ + { + dimension: "requests", + limit: 100, + remaining: 80, + source: "provider_api", + confidence: "authoritative", + }, + ], + }, + ]); + assert.equal(state.status, "healthy"); + assert.equal(state.values[0]?.source, "provider_api"); + assert.equal(state.values[0]?.remaining, 80); +}); + +test("response headers normalize only through an explicit provider mapping", () => { + const parsed = parseRateLimitHeaders( + { "x-limit": "100", "x-remaining": "12", "x-reset": "1700000000" }, + "provider", + "connection", + { limit: "x-limit", remaining: "x-remaining", reset: "x-reset", dimension: "requests" } + ); + assert.equal(parsed?.value.dimension, "requests"); + assert.equal(parsed?.value.remaining, 12); + assert.equal(parsed?.snapshot.requestLimit, 100); + assert.equal(parseRateLimitHeaders({}, "p", "c", { limit: "missing" }), null); +}); + +test("exhausted quota and open circuits are excluded; approaching quota is penalized", () => { + const result = rankCandidates([ + { + providerId: "exhausted", + modelId: "m", + capabilityScore: 1, + allocation: "allow", + healthScore: 1, + circuit: "closed", + quota: "exhausted", + }, + { + providerId: "open", + modelId: "m", + capabilityScore: 1, + allocation: "allow", + healthScore: 1, + circuit: "open", + quota: "unknown", + }, + { + providerId: "approaching", + modelId: "m", + capabilityScore: 1, + allocation: "allow", + healthScore: 1, + circuit: "closed", + quota: "approaching_limit", + }, + { + providerId: "healthy", + modelId: "m", + capabilityScore: 1, + allocation: "allow", + healthScore: 1, + circuit: "closed", + quota: "unknown", + }, + ]); + assert.equal(result.selected?.providerId, "healthy"); + assert.equal( + result.candidates.find((candidate) => candidate.providerId === "exhausted")?.eligible, + false + ); + assert.ok( + (result.candidates.find((candidate) => candidate.providerId === "approaching")?.score ?? 1) < 1 + ); +}); + +test("failure classification retries transient failures but not authentication errors", () => { + const timeout = classifyProviderFailure({ + providerId: "codex", + statusCode: 504, + message: "upstream timeout", + }); + const auth = classifyProviderFailure({ + providerId: "codex", + statusCode: 401, + message: "invalid token", + }); + assert.equal(timeout.type, "timeout"); + assert.equal(shouldFailover(timeout), true); + assert.equal(auth.type, "authentication_error"); + assert.equal(shouldFailover(auth), false); +}); + +test("unknown state helper is explicit", () => { + const state = unknownQuotaState("p", "c", "2026-01-01T00:00:00.000Z"); + assert.deepEqual(state, { + providerId: "p", + connectionId: "c", + supported: false, + fetchedAt: "2026-01-01T00:00:00.000Z", + values: [], + status: "unknown", + }); +});