perf(quota): scope availability read views to single accounts

changelog.d/fixes/18741-quota-availability-read-view-stall.md

@@ -0,0 +1 @@
+- **perf(quota):** Scope availability read views to single accounts and prioritize in-memory usage over repeated SQLite snapshot queries to eliminate 14-21s event loop stalls on instances with many connections; the dashboard availability view now prefers fresh provider-limits usage over a stale persisted snapshot, which is only read when no usage is available ([#18741](https://github.com/cheaper-inference/OmniRoute/pull/18741)) — thanks @bufftop25

src/domain/quotaCacheReadView.ts

@@ -8,13 +8,38 @@ export function getRequestQuotaCacheState(): State {
   return readView.getStore() ?? getQuotaCacheState();
 }
 
-/** Snapshot hydration and reset advancement stay private to this synchronous read. */
-export function withQuotaCacheReadView<T>(read: () => T): T {
+/**
+ * Snapshot hydration and reset advancement stay private to this synchronous read.
+ * With `connectionId` only that connection's cache entry is cloned (the 30-second availability
+ * poll evaluates one account at a time); without it every entry is cloned. A call nested inside
+ * an active read view reuses that view and adds the requested connection to it.
+ */
+export function withQuotaCacheReadView<T>(read: () => T, connectionId?: string): T {
+  const currentStore = readView.getStore();
+  if (currentStore) {
+    if (connectionId !== undefined && !currentStore.cache.has(connectionId)) {
+      const state = getQuotaCacheState();
+      const entry = state.cache.get(connectionId);
+      if (entry) {
+        currentStore.cache.set(connectionId, structuredClone(entry));
+      }
+    }
+    return read();
+  }
+
   const state = getQuotaCacheState();
+  // Availability evaluates one account at a time. Cloning every account for every model
+  // makes the 30-second dashboard poll quadratic; keep the same isolation with one entry.
+  const entries =
+    connectionId === undefined
+      ? [...state.cache]
+      : state.cache.has(connectionId)
+        ? [[connectionId, state.cache.get(connectionId)!] as const]
+        : [];
   return readView.run(
     {
       ...state,
-      cache: new Map([...state.cache].map(([id, entry]) => [id, structuredClone(entry)])),
+      cache: new Map(entries.map(([id, entry]) => [id, structuredClone(entry)])),
       healthyUntil: new Map(state.healthyUntil),
       refreshingSet: new Set(),
     },

src/lib/usage/availability.ts

@@ -120,10 +120,13 @@ function readCachedUsage(
   persistedUsage: Usage | undefined,
   requestedModel: string | null
 ): Usage & { cached?: ReturnType<typeof getQuotaCache> } {
-  getQuotaSnapshotFetchedAt(id);
   seedReadView(id, provider, persistedUsage || {});
   hydrateCodexQuotaCacheForRequest(connection, requestedModel);
-  const cached = getQuotaCache(id);
+  let cached = getQuotaCache(id);
+  if (!cached && !persistedUsage?.quotas) {
+    getQuotaSnapshotFetchedAt(id);
+    cached = getQuotaCache(id);
+  }
   return {
     ...(persistedUsage ||
       (cached
@@ -348,7 +351,7 @@ export function evaluateCachedQuotaAvailability(
       ),
       ...(predicates.cutoffEnabled ? { cutoffEvaluated: predicates.cutoffEvaluated } : {}),
     };
-  });
+  }, toProviderConnection(input).id);
 }
 
 export function buildConnectionQuotaAvailability(
@@ -357,71 +360,74 @@ export function buildConnectionQuotaAvailability(
   policy: QuotaPreflightPolicy
 ): ConnectionQuotaAvailability {
   const connection = toProviderConnection(input);
-  const cachedUsage = withQuotaCacheReadView(() => {
-    getQuotaSnapshotFetchedAt(connection.id);
-    const cached = getQuotaCache(connection.id);
-    return (
+  return withQuotaCacheReadView(() => {
+    seedReadView(connection.id, connection.provider, usage || {});
+    let cached = getQuotaCache(connection.id);
+    if (!cached && !usage?.quotas) {
+      getQuotaSnapshotFetchedAt(connection.id);
+      cached = getQuotaCache(connection.id);
+    }
+    const cachedUsage =
       usage ||
       (cached
         ? {
             quotas: cached.quotas,
             modelQuotas: cached.modelQuotas,
             fetchedAt: timestamp(cached.fetchedAt)!,
           }
-        : {})
+        : {});
+    const evaluated = appendSavedRouterScopes(
+      connection,
+      availabilityScopes(
+        connection.provider,
+        cachedUsage,
+        connection.providerSpecificData?.antigravityFamilyRateLimitedUntil
+      )
+    ).map((scope) => ({
+      ...scope,
+      // Routing has no request-specific selector for this display-only window.
+      decision:
+        scope.id === "code_review" &&
+        connection.provider === "codex" &&
+        scope.windows.some((name) => {
+          const q = normalizeQuotas(cachedUsage.quotas || {})[name];
+          return q && q.fractionReported !== false && q.remainingPercentage <= 0;
+        })
+          ? {
+              usable: null,
+              blockers: [],
+              allowanceReasons: [],
+              fetchedAt: cachedUsage.fetchedAt || null,
+            }
+          : scope.models
+            ? combineQuotaAlternatives(
+                scope.models.map((model) =>
+                  evaluateCachedQuotaAvailability(input, usage, policy, model)
+                ),
+                evaluateCachedQuotaAvailability(input, usage, policy, scope.model)
+              )
+            : evaluateCachedQuotaAvailability(input, usage, policy, scope.model),
+    }));
+    const candidates = evaluated.filter((scope) => scope.id !== "code_review");
+    // Ordinary requests define the account verdict. Only peer pools may use any-usable
+    // aggregation; attached scopes remain visible without promoting a blocked account.
+    const ordinary = evaluated.find((scope) => scope.kind === "account");
+    const decisions = evaluated
+      .filter((scope) => scope.kind === "group")
+      .map((scope) => scope.decision);
+    const fallback = evaluateCachedQuotaAvailability(input, usage, policy);
+    const account = ordinary ? ordinary.decision : combineQuotaAlternatives(decisions, fallback);
+    const payloadScopes = evaluated.flatMap((scope) =>
+      scope.kind === "account"
+        ? candidates.some((candidate) => candidate.kind !== "account")
+          ? [{ ...scope, kind: "scoped" as const }]
+          : []
+        : [{ ...scope, kind: scope.kind }]
     );
-  });
-  const evaluated = appendSavedRouterScopes(
-    connection,
-    availabilityScopes(
-      connection.provider,
-      cachedUsage,
-      connection.providerSpecificData?.antigravityFamilyRateLimitedUntil
-    )
-  ).map((scope) => ({
-    ...scope,
-    // Routing has no request-specific selector for this display-only window.
-    decision:
-      scope.id === "code_review" &&
-      connection.provider === "codex" &&
-      scope.windows.some((name) => {
-        const q = normalizeQuotas(cachedUsage.quotas || {})[name];
-        return q && q.fractionReported !== false && q.remainingPercentage <= 0;
-      })
-        ? {
-            usable: null,
-            blockers: [],
-            allowanceReasons: [],
-            fetchedAt: cachedUsage.fetchedAt || null,
-          }
-        : scope.models
-          ? combineQuotaAlternatives(
-              scope.models.map((model) =>
-                evaluateCachedQuotaAvailability(input, usage, policy, model)
-              ),
-              evaluateCachedQuotaAvailability(input, usage, policy, scope.model)
-            )
-          : evaluateCachedQuotaAvailability(input, usage, policy, scope.model),
-  }));
-  const candidates = evaluated.filter((scope) => scope.id !== "code_review");
-  // Ordinary requests define the account verdict. Only peer pools may use any-usable
-  // aggregation; attached scopes remain visible without promoting a blocked account.
-  const ordinary = evaluated.find((scope) => scope.kind === "account");
-  const decisions = evaluated
-    .filter((scope) => scope.kind === "group")
-    .map((scope) => scope.decision);
-  const fallback = evaluateCachedQuotaAvailability(input, usage, policy);
-  const account = ordinary ? ordinary.decision : combineQuotaAlternatives(decisions, fallback);
-  const payloadScopes = evaluated.flatMap((scope) =>
-    scope.kind === "account"
-      ? candidates.some((candidate) => candidate.kind !== "account")
-        ? [{ ...scope, kind: "scoped" as const }]
-        : []
-      : [{ ...scope, kind: scope.kind }]
-  );
-  return {
-    id: connection.id,
-    account,
-    scopes: payloadScopes,
-  };
+    return {
+      id: connection.id,
+      account,
+      scopes: payloadScopes,
+    };
+  }, connection.id);
 }

tests/unit/quota-availability-read-view-precedence-18741.test.ts

@@ -0,0 +1,72 @@
+// #18741 — buildConnectionQuotaAvailability seeds the scoped read view from the fresh
+// provider-limits usage BEFORE consulting SQLite snapshots. Pins the precedence (fresh usage
+// wins over a stale snapshot, and the snapshot is not read at all when usage carries quotas)
+// plus the unchanged fallback (no usage -> the snapshot still hydrates the decision).
+import test from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-18741-"));
+process.env.DATA_DIR = TEST_DATA_DIR;
+
+const core = await import("../../src/lib/db/core.ts");
+const snapshots = await import("../../src/lib/db/quotaSnapshots.ts");
+const availability = await import("../../src/lib/usage/availability.ts");
+
+const policy = { enabled: true, defaultThresholdPercent: 2, providerWindowDefaults: {} };
+const future = new Date(Date.now() + 48 * 3600e3).toISOString();
+
+test.after(() => {
+  core.resetDbInstance();
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+});
+
+function saveSnapshot(id: string, remainingPercentage: number, exhausted: boolean) {
+  snapshots.saveQuotaSnapshot({
+    provider: "codex",
+    connection_id: id,
+    window_key: "weekly",
+    remaining_percentage: remainingPercentage,
+    is_exhausted: exhausted ? 1 : 0,
+    next_reset_at: future,
+    window_duration_ms: 604800000,
+    raw_data: "{}",
+  });
+}
+
+function usageWith(remainingPercentage: number) {
+  return {
+    fetchedAt: new Date().toISOString(),
+    quotas: { weekly: { remainingPercentage, resetAt: future } },
+    modelQuotas: {},
+  };
+}
+
+function accountUsable(id: string, usage: ReturnType<typeof usageWith> | undefined) {
+  return availability.buildConnectionQuotaAvailability(
+    { id, provider: "codex", isActive: true },
+    usage,
+    policy
+  ).account.usable;
+}
+
+test("fresh provider-limits usage wins over a stale exhausted snapshot", () => {
+  saveSnapshot("conn-a-18741", 0, true);
+  assert.equal(accountUsable("conn-a-18741", usageWith(80)), true);
+});
+
+test("fresh exhausted usage still blocks over a healthy snapshot", () => {
+  saveSnapshot("conn-b-18741", 80, false);
+  assert.equal(accountUsable("conn-b-18741", usageWith(0)), false);
+});
+
+test("without usage the persisted snapshot still hydrates the decision", () => {
+  saveSnapshot("conn-c-18741", 0, true);
+  assert.equal(accountUsable("conn-c-18741", undefined), false);
+});
+
+test("no usage and no snapshot stays unknown, not exhausted", () => {
+  assert.notEqual(accountUsable("conn-d-18741", undefined), false);
+});

tests/unit/quota-read-view-scoped.test.ts

@@ -0,0 +1,52 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { getQuotaCacheState } from "../../src/domain/quotaCacheState.ts";
+import {
+  getRequestQuotaCacheState,
+  withQuotaCacheReadView,
+} from "../../src/domain/quotaCacheReadView.ts";
+
+test("a connection-scoped read clones only that connection and keeps mutations private", () => {
+  const state = getQuotaCacheState();
+  const saved = new Map(state.cache);
+  const original = globalThis.structuredClone;
+  let clones = 0;
+  try {
+    state.cache.clear();
+    for (let i = 0; i < 132; i++)
+      state.cache.set(`c${i}`, {
+        connectionId: `c${i}`,
+        provider: "codex",
+        quotas: { session: { remainingPercentage: 80, resetAt: null } },
+        modelQuotas: {},
+        fetchedAt: Date.now(),
+        exhausted: false,
+        nextResetAt: null,
+      });
+    globalThis.structuredClone = ((value: unknown) => {
+      clones++;
+      return original(value);
+    }) as typeof structuredClone;
+    withQuotaCacheReadView(() => {
+      const local = getRequestQuotaCacheState();
+      local.cache.get("c0")!.quotas.session.remainingPercentage = 0;
+      local.cache.delete("c0");
+      assert.equal(local.cache.has("c1"), false);
+    }, "c0");
+    assert.equal(clones, 1);
+    assert.equal(state.cache.get("c0")!.quotas.session.remainingPercentage, 80);
+    assert.equal(state.cache.size, 132);
+  } finally {
+    globalThis.structuredClone = original;
+    state.cache.clear();
+    for (const [id, entry] of saved) state.cache.set(id, entry);
+  }
+});
+
+test("an unscoped read retains the full isolated snapshot", () => {
+  const state = getQuotaCacheState();
+  withQuotaCacheReadView(() => {
+    assert.notEqual(getRequestQuotaCacheState().cache, state.cache);
+    assert.equal(getRequestQuotaCacheState().cache.size, state.cache.size);
+  });
+});