Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions src/app/api/usage/[connectionId]/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { getUsageForProvider } from "@omniroute/open-sse/services/usage.ts";
import { getExecutor } from "@omniroute/open-sse/executors/index.ts";
import { syncToCloud } from "@/lib/cloudSync";
import { runWithProxyContext } from "@omniroute/open-sse/utils/proxyFetch.ts";
import { setQuotaCache } from "@/domain/quotaCache";

/**
* Sync to cloud if enabled
Expand Down Expand Up @@ -147,6 +148,12 @@ export async function GET(request: Request, { params }: { params: Promise<{ conn
const usage = await runWithProxyContext(proxyInfo?.proxy || null, () =>
getUsageForProvider(connection)
);

// Populate quota cache for quota-aware account selection
if (usage?.quotas) {
setQuotaCache(connectionId, connection.provider, usage.quotas);
}

return Response.json(usage);
} catch (error) {
console.error("[Usage API] Error fetching usage:", error);
Expand Down
248 changes: 248 additions & 0 deletions src/domain/quotaCache.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,248 @@
/**
* Quota Cache — Domain Layer
*
* In-memory cache of provider quota data per connectionId.
* Populated by:
* - Dashboard usage endpoint (GET /api/usage/[connectionId])
* - 429 responses marking account as exhausted
*
* Background refresh runs every 1 minute:
* - Active accounts (quota > 0%): refetch every 5 minutes
* - Exhausted accounts: refetch every 20 minutes (or immediately after resetAt passes)
*
* @module domain/quotaCache
*/

import { getUsageForProvider } from "@omniroute/open-sse/services/usage.ts";
import { getProviderConnectionById, resolveProxyForConnection } from "@/lib/localDb";
import { runWithProxyContext } from "@omniroute/open-sse/utils/proxyFetch.ts";

// ─── Types ──────────────────────────────────────────────────────────────────

interface QuotaInfo {
remainingPercentage: number;
resetAt: string | null;
}

interface QuotaCacheEntry {
connectionId: string;
provider: string;
quotas: Record<string, QuotaInfo>;
fetchedAt: number;
exhausted: boolean;
nextResetAt: string | null;
}

// ─── Constants ──────────────────────────────────────────────────────────────

const ACTIVE_TTL_MS = 5 * 60 * 1000; // 5 minutes for active accounts
const EXHAUSTED_TTL_MS = 5 * 60 * 1000; // 5 minutes for 429-sourced entries (no resetAt)
const EXHAUSTED_REFRESH_MS = 20 * 60 * 1000; // 20 minutes: recheck exhausted accounts
const REFRESH_INTERVAL_MS = 60 * 1000; // Background tick every 1 minute

// ─── State ──────────────────────────────────────────────────────────────────

const cache = new Map<string, QuotaCacheEntry>();
let refreshTimer: ReturnType<typeof setInterval> | null = null;

// ─── Helpers ────────────────────────────────────────────────────────────────

function isExhausted(quotas: Record<string, QuotaInfo>): boolean {
const entries = Object.values(quotas);
if (entries.length === 0) return false;
return entries.every((q) => q.remainingPercentage <= 0);
}

function earliestResetAt(quotas: Record<string, QuotaInfo>): string | null {
let earliest: string | null = null;
for (const q of Object.values(quotas)) {
if (q.resetAt && (!earliest || new Date(q.resetAt) < new Date(earliest))) {
earliest = q.resetAt;
}
}
return earliest;
}

function normalizeQuotas(rawQuotas: Record<string, any>): Record<string, QuotaInfo> {
const result: Record<string, QuotaInfo> = {};
for (const [key, q] of Object.entries(rawQuotas)) {
if (q && typeof q === "object") {
result[key] = {
remainingPercentage:
q.remainingPercentage ??
(q.total ? Math.round(((q.total - (q.used || 0)) / q.total) * 100) : 100),
resetAt: q.resetAt || null,
};
}
}
return result;
}

// ─── Public API ─────────────────────────────────────────────────────────────

/**
* Store quota data for a connection (called by usage endpoint and background refresh).
*/
export function setQuotaCache(
connectionId: string,
provider: string,
rawQuotas: Record<string, any>
) {
const quotas = normalizeQuotas(rawQuotas);
const exhausted = isExhausted(quotas);
cache.set(connectionId, {
connectionId,
provider,
quotas,
fetchedAt: Date.now(),
exhausted,
nextResetAt: exhausted ? earliestResetAt(quotas) : null,
});
}

/**
* Get cached quota entry (returns null if not cached).
*/
export function getQuotaCache(connectionId: string): QuotaCacheEntry | null {
return cache.get(connectionId) || null;
}

/**
* Check if an account's quota is exhausted based on cached data.
* Returns false if no cache entry exists (unknown = assume available).
*/
export function isAccountQuotaExhausted(connectionId: string): boolean {
const entry = cache.get(connectionId);
if (!entry) return false;

// If exhausted and we have a resetAt that has passed, consider it available
if (entry.exhausted && entry.nextResetAt) {
if (new Date(entry.nextResetAt).getTime() <= Date.now()) {
return false; // Reset time passed, assume available until refresh confirms
}
}

// Check TTL based on state
const age = Date.now() - entry.fetchedAt;
if (entry.exhausted) {
// Exhausted entries without resetAt use fixed TTL
if (!entry.nextResetAt && age > EXHAUSTED_TTL_MS) return false;
// Exhausted entries with resetAt stay valid until resetAt or refresh
return true;
}

// Active entries expire after ACTIVE_TTL
if (age > ACTIVE_TTL_MS) return false;

return entry.exhausted;

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

WARNING: Dead code — this line returns entry.exhausted but at this point in the function, entry.exhausted must be false (we already returned for exhausted entries at line 131). Since active entries within TTL should return false, this could be simplified to return false; for clarity, or removed entirely as the logic flow already guarantees the correct return value.

}

/**
* Mark an account as quota-exhausted from a 429 response (no quota data available).
* Uses 5-minute fixed TTL since we don't know the actual resetAt.
*/
export function markAccountExhaustedFrom429(connectionId: string, provider: string) {
cache.set(connectionId, {
connectionId,
provider,
quotas: {},
fetchedAt: Date.now(),
exhausted: true,
nextResetAt: null,
});
}

// ─── Background Refresh ─────────────────────────────────────────────────────

async function refreshEntry(entry: QuotaCacheEntry) {
try {
const connection = await getProviderConnectionById(entry.connectionId);
if (!connection || connection.authType !== "oauth" || !connection.isActive) {
cache.delete(entry.connectionId);
return;
}

const proxyInfo = await resolveProxyForConnection(entry.connectionId);
const usage = await runWithProxyContext(proxyInfo?.proxy || null, () =>
getUsageForProvider(connection)
);

if (usage?.quotas) {
setQuotaCache(entry.connectionId, entry.provider, usage.quotas);
}
} catch {
// Refresh failed silently — keep stale entry
}
}

async function backgroundRefreshTick() {
const now = Date.now();

for (const entry of cache.values()) {
const age = now - entry.fetchedAt;

if (entry.exhausted) {
// If resetAt has passed, refetch immediately
if (entry.nextResetAt && new Date(entry.nextResetAt).getTime() <= now) {
refreshEntry(entry);
continue;
}
// Recheck exhausted accounts every 20 minutes
if (age >= EXHAUSTED_REFRESH_MS) {
refreshEntry(entry);
}
} else {
// Refresh active accounts every 5 minutes
if (age >= ACTIVE_TTL_MS) {
refreshEntry(entry);
}
}
}
}
Comment on lines +178 to +201

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The current implementation of backgroundRefreshTick uses a fire-and-forget pattern for refreshEntry calls inside the loop. While this is functional due to the internal error handling in refreshEntry, it can make the code harder to reason about.

To improve clarity and robustness, I suggest collecting all the refresh tasks into an array of promises and then using Promise.allSettled to handle them. This makes the parallel, non-blocking execution explicit and is a more standard pattern for managing multiple concurrent asynchronous operations.

async function backgroundRefreshTick() {
  const now = Date.now();
  const refreshPromises: Promise<void>[] = [];

  for (const entry of cache.values()) {
    const age = now - entry.fetchedAt;

    if (entry.exhausted) {
      // If resetAt has passed, refetch immediately
      if (entry.nextResetAt && new Date(entry.nextResetAt).getTime() <= now) {
        refreshPromises.push(refreshEntry(entry));
        continue;
      }
      // Recheck exhausted accounts every 20 minutes
      if (age >= EXHAUSTED_REFRESH_MS) {
        refreshPromises.push(refreshEntry(entry));
      }
    } else {
      // Refresh active accounts every 5 minutes
      if (age >= ACTIVE_TTL_MS) {
        refreshPromises.push(refreshEntry(entry));
      }
    }
  }

  if (refreshPromises.length > 0) {
    // Execute refreshes in parallel but don't block the tick.
    // Using Promise.allSettled to ensure all promises are handled.
    Promise.allSettled(refreshPromises);
  }
}


/**
* Start the background refresh timer.
*/
export function startBackgroundRefresh() {
if (refreshTimer) return;
refreshTimer = setInterval(backgroundRefreshTick, REFRESH_INTERVAL_MS);
// Don't prevent process exit
if (refreshTimer && typeof refreshTimer === "object" && "unref" in refreshTimer) {
(refreshTimer as any).unref();
}
}

/**
* Stop the background refresh timer.
*/
export function stopBackgroundRefresh() {
if (refreshTimer) {
clearInterval(refreshTimer);
refreshTimer = null;
}
}

/**
* Get cache stats (for debugging/dashboard).
*/
export function getQuotaCacheStats() {
const entries: Array<{
connectionId: string;
provider: string;
exhausted: boolean;
nextResetAt: string | null;
ageMs: number;
}> = [];

for (const entry of cache.values()) {
entries.push({
connectionId: entry.connectionId.slice(0, 8) + "...",
provider: entry.provider,
exhausted: entry.exhausted,
nextResetAt: entry.nextResetAt,
ageMs: Date.now() - entry.fetchedAt,
});
}

return { total: cache.size, entries };
}
4 changes: 4 additions & 0 deletions src/instrumentation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ export async function register() {
const { initApiBridgeServer } = await import("@/lib/apiBridgeServer");
initApiBridgeServer();

// Quota cache: start background refresh for quota-aware account selection
const { startBackgroundRefresh } = await import("@/domain/quotaCache");
startBackgroundRefresh();

// Compliance: Initialize audit_log table + cleanup expired logs
try {
const { initAuditLog, cleanupExpiredLogs } = await import("@/lib/compliance/index");
Expand Down
34 changes: 24 additions & 10 deletions src/sse/handlers/chat.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,12 @@ import { sanitizeRequest } from "../../shared/utils/inputSanitizer";

// Pipeline integration — wired modules
import { getCircuitBreaker, CircuitBreakerOpenError } from "../../shared/utils/circuitBreaker";
import { isModelAvailable, setModelUnavailable } from "../../domain/modelAvailability";
import {
isModelAvailable,
setModelUnavailable,
clearModelUnavailability,
} from "../../domain/modelAvailability";
import { getQuotaCache, markAccountExhaustedFrom429 } from "../../domain/quotaCache";
import { RequestTelemetry, recordTelemetry } from "../../shared/utils/requestTelemetry";
import { generateRequestId } from "../../shared/utils/requestId";
import { recordCost } from "../../domain/costRules";
Expand Down Expand Up @@ -127,7 +132,10 @@ export async function handleChat(request: any, clientRawRequest: any = null) {
telemetry.startPhase("policy");
const policy = await enforceApiKeyPolicy(request, modelStr);
if (policy.rejection) {
log.warn("POLICY", `API key policy rejected: ${modelStr} (key=${policy.apiKeyInfo?.id || "unknown"})`);
log.warn(
"POLICY",
`API key policy rejected: ${modelStr} (key=${policy.apiKeyInfo?.id || "unknown"})`
);
return policy.rejection;
}
const apiKeyInfo = policy.apiKeyInfo;
Expand Down Expand Up @@ -243,6 +251,13 @@ async function handleSingleModelChat(
const credentials = await getProviderCredentials(provider, excludeConnectionId);

if (!credentials || credentials.allRateLimited) {
if (lastStatus === 429 || lastStatus === 503) {
setModelUnavailable(provider, model, 60000, `HTTP ${lastStatus}`);
log.info(
"AVAILABILITY",
`${provider}/${model} marked unavailable — all accounts exhausted (HTTP ${lastStatus})`
);
}
return handleNoCredentials(
credentials,
excludeConnectionId,
Expand Down Expand Up @@ -296,22 +311,21 @@ async function handleSingleModelChat(
});

if (result.success) {
if (excludeConnectionId) {
clearModelUnavailability(provider, model);
}
recordCostIfNeeded(apiKeyInfo, result);
if (telemetry) telemetry.startPhase("finalize");
if (telemetry) telemetry.endPhase();
return result.response;
}

// Pipeline: Mark model unavailable on repeated failures
if (result.status === 429 || result.status === 503) {
setModelUnavailable(provider, model, 60000, `HTTP ${result.status}`);
log.info(
"AVAILABILITY",
`${provider}/${model} marked unavailable for 60s (HTTP ${result.status})`
);
// 6. Mark quota-exhausted from 429 if no cached quota data
if (result.status === 429 && !getQuotaCache(credentials.connectionId)) {
markAccountExhaustedFrom429(credentials.connectionId, provider);
}

// 6. Fallback to next account
// 7. Fallback to next account
const { shouldFallback } = await markAccountUnavailable(
credentials.connectionId,
result.status,
Expand Down
Loading