fix(api): keep catalog builds responsive and hash cache keys (#9147, #10313)

changelog.d/fixes/10313-catalog-cache-key-hash.md

@@ -0,0 +1 @@
+- fix(api): hash the API key before using it as the model-catalog cache Map key (no raw credentials in process heap) (#10313)

changelog.d/fixes/9147-catalog-eventloop-yield.md

@@ -0,0 +1 @@
+- fix(api): yield the event loop during catalog builds and bulk-load override/hidden-model tables (#9147)

src/app/api/v1/models/catalogResponse.ts

@@ -32,6 +32,7 @@ import {
   enrichCatalogModelEntry,
   type CatalogEnrichmentSnapshot,
 } from "@/lib/modelMetadataRegistry";
+import { createModelCapabilityResolutionSnapshot } from "@/lib/modelCapabilityResolutionSnapshot";
 import { isModelCatalogNamesEnabled } from "@/shared/utils/featureFlags";
 import { extractApiKey } from "@/sse/services/auth";
 import { maybeOmitCatalogModelName } from "./catalogHelpers";
@@ -45,7 +46,7 @@ import { isCodexModelCatalogClient } from "./catalogRequest";
  * returns early, but it still owes the caller these steps — the discovery mirrors in
  * particular are what let Claude Code see a quota pool's models at all.
  */
-export function applyCatalogPostFilters(
+export async function applyCatalogPostFilters(
   request: Request,
   models: Array<Record<string, any>>,
   ctx: {
@@ -54,7 +55,8 @@ export function applyCatalogPostFilters(
     aliasToProviderId: Record<string, string>;
     hideNoThinkVariants?: boolean;
   }
-): Array<Record<string, any>> {
+): Promise<Array<Record<string, any>>> {
+  const yieldTurn = (): Promise<void> => new Promise((resolve) => setImmediate(resolve));
   let finalModels = models;
 
   // variants are only generated for surviving models.
@@ -65,6 +67,11 @@ export function applyCatalogPostFilters(
     });
   }
 
+  // #9147: the variant-append passes each walk the full model list (O(n) per pass),
+  // so a catalog-scale build must not run all of them in one synchronous stretch.
+  // Yield once between the expensive passes to let the event loop breathe.
+  await yieldTurn();
+
   // Advertise Claude reasoning-effort variants (claude/<model>-{low,medium,high[,xhigh]}).
   // Derived from the already key-filtered list so a variant only appears when its real
   // model is permitted. Runs before the no-thinking pass: the gateway already routes these
@@ -139,11 +146,15 @@ export function applyCatalogPostFilters(
     );
   }
 
+  await yieldTurn();
+
   // #7694: advertise `<provider>/<model>-<tier>` variants for synced models that
   // captured `reasoning.supported_efforts` at sync time (capabilities.effort_tiers).
   // Derived from the already key-filtered list; skips codex/kimi (own suffix mechanism).
   finalModels = appendSyncedEffortVariants(finalModels);
 
+  await yieldTurn();
+
   // #4424 follow-up — drop exact-duplicate ids that slip through the per-source push
   // guards (e.g. `codex/gpt-5.5`, `veo-free/seedance` listed twice). Keyed by listing
   // identity (id, type, subtype) so the intentional same-id audio transcription/speech
@@ -207,25 +218,45 @@ export async function finalizeCatalogResponse(
   }
 
   const includeModelNames = isModelCatalogNamesEnabled();
-  const enrichedModels = disambiguateCatalogModelNames(
-    finalModels.map((model) => {
-      if (model.owned_by === "combo") {
-        return maybeOmitCatalogModelName(model, includeModelNames);
-      }
-      const enriched = enrichCatalogModelEntry(model, undefined, enrichmentSnapshot);
-      const fallbackContextLength = getContextFallback(enriched);
-      const listedModel = fallbackContextLength
-        ? { ...enriched, context_length: fallbackContextLength }
-        : enriched;
-      return maybeOmitCatalogModelName(listedModel, includeModelNames);
-    })
-  );
-  // Canonical provider-grouped publication: one contiguous block per provider,
-  // combos pinned first. Stable — preserves combo sort_order, connection priority,
-  // and equal-id audio twins. Grouped by owned_by (canonical identity), not the
-  // routing alias prefix. Applied after enrichment/disambiguation so the final
-  // serialized order is what every consumer sees; cached as part of the body.
+  // #9147: enrichment is the most expensive single stage of the catalog build —
+  // per-entry provider/model resolution plus pricing + token/context override
+  // lookups. Two fixes so a large catalog cannot pin the Node.js thread here:
+  //  (1) bulk-load the synced-capability + override tables ONCE into an in-memory
+  //      snapshot (#9199 machinery) so per-entry enrichment never hits SQLite;
+  //  (2) yield to the event loop every `YIELD_EVERY` entries so even the remaining
+  //      per-entry work is interleaved with other callers / the dashboard WS.
+  const yieldTurn = (): Promise<void> => new Promise((resolve) => setImmediate(resolve));
+  await yieldTurn();
+  const capabilityResolutionSnapshot = createModelCapabilityResolutionSnapshot();
+  const enriched: Array<Record<string, unknown>> = [];
+  const catYIELD_EVERY = 5;
+  let catEnrichCount = 0;
+  for (const model of finalModels) {
+    let listedModel: Record<string, unknown>;
+    if (model.owned_by === "combo") {
+      listedModel = maybeOmitCatalogModelName(model, includeModelNames);
+    } else {
+      const entry = enrichCatalogModelEntry(model, undefined, {
+        ...enrichmentSnapshot,
+        capabilityResolutionSnapshot,
+      });
+      const fallbackContextLength = getContextFallback(entry);
+      listedModel = fallbackContextLength
+        ? { ...entry, context_length: fallbackContextLength }
+        : entry;
+      listedModel = maybeOmitCatalogModelName(listedModel, includeModelNames);
+    }
+    enriched.push(listedModel);
+    catEnrichCount++;
+    if (catEnrichCount % catYIELD_EVERY === 0) {
+      await yieldTurn();
+    }
+  }
+  await yieldTurn();
+  const enrichedModels = disambiguateCatalogModelNames(enriched);
+  await yieldTurn();
   const orderedModels = sortCatalogModelsProviderGrouped(enrichedModels);
+  await yieldTurn();
   // Codex CLI compatibility: its model-catalog refresh (codex_models_manager) does
   // GET /v1/models?client_version=<v> and decodes a JSON object with a TOP-LEVEL
   // `models` array, so the OpenAI-standard `{object,data}` shape makes it fail with

src/lib/modelCapabilities.ts

@@ -567,9 +567,12 @@ function getContextOverride(
 /**
  * Resolve a persisted context override by canonical id, then by the exact raw
  * alias supplied by the caller. Neither lookup inherits to related models.
+ *
+ * `snapshot` is the #9147 build-local bulk load; when supplied the on-demand
+ * SQLite read is skipped and the preloaded nested map is used instead.
  */
-export function getResolvedModelContextOverride(input: CapabilityInput): number | null {
-  return getContextOverride(resolveCapabilityInput(input));
+export function getResolvedModelContextOverride(input: CapabilityInput, snapshot?: ModelCapabilityResolutionSnapshot | null): number | null {
+  return getContextOverride(resolveCapabilityInput(input), snapshot);
 }
 
 function getInputTokenCapabilityOverride(resolved: {

src/lib/modelMetadataRegistry.ts

@@ -28,6 +28,7 @@ import {
   CANONICAL_EFFORT_VALUES,
   extendCodexGpt56EffortValues,
 } from "@/shared/reasoning/effortStandardization";
+import type { ModelCapabilityResolutionSnapshot } from "@/lib/modelCapabilityResolutionSnapshot";
 
 const MODEL_METADATA_SCHEMA_VERSION = "model-metadata-v1";
 
@@ -40,6 +41,9 @@ type JsonRecord = Record<string, unknown>;
 export interface CatalogEnrichmentSnapshot {
   modelsDevPricing: PricingByProvider | null;
   providerNodeIdsByPrefix?: Readonly<Record<string, string>>;
+  /** #9147: build-local bulk load of synced capabilities + token/context overrides
+   * so per-entry enrichment never hits SQLite again (see catalogResponse.ts). */
+  capabilityResolutionSnapshot?: ModelCapabilityResolutionSnapshot | null;
 }
 
 interface CatalogDiagnosticsOptions {
@@ -200,20 +204,27 @@ export function getCatalogDiagnosticsHeaders(
 export function getCanonicalModelMetadata(input: {
   provider?: string | null;
   model?: string | null;
+  snapshot?: ModelCapabilityResolutionSnapshot | null;
 }): CanonicalModelMetadata | null {
   const modelId = asNonEmptyString(input.model);
   if (!modelId) return null;
 
-  const resolved = getResolvedModelCapabilities({
-    provider: input.provider || null,
-    model: modelId,
-  });
+  const resolved = getResolvedModelCapabilities(
+    {
+      provider: input.provider || null,
+      model: modelId,
+    },
+    undefined,
+    input.snapshot || null
+  );
   const provider = resolved.provider;
   const providerAlias = provider ? PROVIDER_ID_TO_ALIAS[provider] || provider : null;
   const registryModel = getRegistryModel(providerAlias || provider, resolved.model || modelId);
   const staticSpec = getModelSpec(resolved.model || modelId);
   const syncedCapability =
-    provider && resolved.model ? getSyncedCapability(provider, resolved.model) : null;
+    provider && resolved.model
+      ? getSyncedCapability(provider, resolved.model, input.snapshot?.synced ?? null)
+      : null;
   const canonicalStaticAlias = resolveStaticModelAlias(resolved.model || modelId);
   const modalities = buildModalities(
     resolved.modalitiesInput,
@@ -420,7 +431,11 @@ export function enrichCatalogModelEntry<T extends JsonRecord>(
       return id;
     })();
 
-  const metadata = getCanonicalModelMetadata({ provider, model });
+  const metadata = getCanonicalModelMetadata({
+    provider,
+    model,
+    snapshot: snapshot?.capabilityResolutionSnapshot ?? null,
+  });
   if (!metadata) return entry;
   const registryModel = getRegistryModel(
     metadata.providerAlias || metadata.provider,
@@ -436,7 +451,11 @@ export function enrichCatalogModelEntry<T extends JsonRecord>(
     getAuthoritativeContextWindow(metadata.model) ??
     getAuthoritativeContextWindow(model);
   const specialtySurface = isNonChatCatalogSurface(entry.type);
-  const persistedContextWindow = getResolvedModelContextOverride({ provider, model });
+  const capabilitySnapshot = snapshot?.capabilityResolutionSnapshot ?? null;
+  const persistedContextWindow = getResolvedModelContextOverride(
+    { provider, model },
+    capabilitySnapshot
+  );
   const capabilityFields = {
     ...(typeof metadata.capabilities.vision === "boolean"
       ? { vision: metadata.capabilities.vision }
@@ -528,10 +547,15 @@ export function enrichCatalogModelEntry<T extends JsonRecord>(
   }
 
   const persistedOutputLimit =
-    getModelCapabilityOverride(provider, model, "max_output_tokens") ??
-    getModelCapabilityOverride(provider, model, "max_token") ??
-    getModelCapabilityOverride(publicProvider, model, "max_output_tokens") ??
-    getModelCapabilityOverride(publicProvider, model, "max_token");
+    getModelCapabilityOverride(provider, model, "max_output_tokens", capabilitySnapshot?.maxTokenOverrides) ??
+    getModelCapabilityOverride(provider, model, "max_token", capabilitySnapshot?.maxTokenOverrides) ??
+    getModelCapabilityOverride(
+      publicProvider,
+      model,
+      "max_output_tokens",
+      capabilitySnapshot?.maxTokenOverrides
+    ) ??
+    getModelCapabilityOverride(publicProvider, model, "max_token", capabilitySnapshot?.maxTokenOverrides);
   if (persistedOutputLimit !== null) {
     nextEntry.max_output_tokens = persistedOutputLimit;
   } else if (

tests/unit/10313-catalog-cache-key-hashing.test.ts

@@ -0,0 +1,131 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-catalog-keyleak-"));
+process.env.DATA_DIR = TEST_DATA_DIR;
+process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "catalog-keyleak-test-secret";
+
+const core = await import("../../src/lib/db/core.ts");
+const apiKeysDb = await import("../../src/lib/db/apiKeys.ts");
+const v1ModelsCatalog = await import("../../src/app/api/v1/models/catalog.ts");
+const catalogCacheMod = await import("../../src/app/api/v1/models/catalogCache.ts");
+
+const SECRET = "sk-live-PROBE-10313-SUPER-SECRET-TOKEN";
+
+test.beforeEach(() => {
+  core.resetDbInstance();
+  apiKeysDb.resetApiKeyState();
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+  fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
+  v1ModelsCatalog.__resetCatalogBuilderRunsForTest();
+});
+
+test.after(() => {
+  core.resetDbInstance();
+  apiKeysDb.resetApiKeyState();
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+});
+
+function captureMapKeys(): { keys: string[]; restore: () => void } {
+  const capturedKeys: string[] = [];
+  const originalSet = Map.prototype.set;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  (Map.prototype as any).set = function (key: unknown, value: unknown) {
+    if (typeof key === "string") capturedKeys.push(key);
+    return originalSet.call(this, key, value);
+  };
+  return {
+    keys: capturedKeys,
+    release: () => {
+      Map.prototype.set = originalSet;
+    },
+  };
+}
+
+// buildCatalogCacheKey emits `prefix|isCodex|apiKeyFingerprint|configuredOnly|hideAuto|hideNoThink`
+// (6 pipe-delimited fields). Other in-flight keys (e.g. `x-request-id`) don't match.
+function isCatalogCacheKey(k: string): boolean {
+  return k.split("|").length === 6;
+}
+
+test("catalog cache Map keys must not contain the raw bearer API key (#10313)", async () => {
+  const probe = captureMapKeys();
+
+  try {
+    const request = new Request("http://localhost/v1/models", {
+      headers: { Authorization: `Bearer ${SECRET}` },
+    });
+    const res = await v1ModelsCatalog.getUnifiedModelsResponse(request);
+    assert.ok(res.status === 200 || res.status === 401 || res.status === 403);
+  } finally {
+    probe.release();
+  }
+
+  const catalogKeys = probe.keys.filter(isCatalogCacheKey);
+  assert.ok(
+    catalogKeys.length > 0,
+    "no catalog cache Map.set() calls observed — the probe did not exercise catalogCache/catalogInFlight"
+  );
+
+  const leaked = catalogKeys.filter((k) => k.includes(SECRET));
+  assert.deepEqual(
+    leaked,
+    [],
+    `catalog cache Map key retained the raw API key verbatim: ${JSON.stringify(leaked)} — ` +
+      `buildCatalogCacheKey() must hash the secret (e.g. sha256) before using it as a Map key`
+  );
+
+  assert.ok(catalogCacheMod.CATALOG_CACHE_TTL_MS_DEFAULT > 0);
+});
+
+test("cache keys embed the sha256 digest of the secret, never the raw secret (#10313)", async () => {
+  // Capture ALL Map keys emitted across BOTH requests (a 2nd identical-secret request
+  // may be a cache hit and emit no new catalog key — irrelevant here: we assert on the
+  // digest that did appear).
+  const probe = captureMapKeys();
+  try {
+    const resA = await v1ModelsCatalog.getUnifiedModelsResponse(
+      new Request("http://localhost/v1/models", {
+        headers: { Authorization: "Bearer sk-10313-DIGEST-A" },
+      })
+    );
+    const resB = await v1ModelsCatalog.getUnifiedModelsResponse(
+      new Request("http://localhost/v1/models", {
+        headers: { Authorization: "Bearer sk-10313-DIGEST-B" },
+      })
+    );
+    assert.ok(resA.status === 200 || resA.status === 401 || resA.status === 403);
+    assert.ok(resB.status === 200 || resB.status === 401 || resB.status === 403);
+  } finally {
+    probe.release();
+  }
+
+  const catalogKeys = probe.keys.filter(isCatalogCacheKey);
+  assert.ok(catalogKeys.length > 0, "expected catalog cache Map.set() calls");
+
+  // #10538 sync note: buildCatalogCacheKey() delegates to the canonical
+  // fingerprintCatalogAuthKey() (landed on release/v3.8.50 independently of #10313),
+  // which truncates the sha256 hex digest to 16 chars for a shorter Map key. Derive
+  // the expected fingerprint the same way rather than re-hardcoding the full digest.
+  const digestA = catalogCacheMod.fingerprintCatalogAuthKey("sk-10313-DIGEST-A");
+  const digestB = catalogCacheMod.fingerprintCatalogAuthKey("sk-10313-DIGEST-B");
+  const rawA = "sk-10313-DIGEST-A";
+  const rawB = "sk-10313-DIGEST-B";
+
+  // The hashed fingerprint, not the raw secret, rides in the cache keys.
+  const keysWithDigestA = catalogKeys.filter((k) => k.includes(digestA));
+  const keysWithDigestB = catalogKeys.filter((k) => k.includes(digestB));
+  assert.ok(keysWithDigestA.length > 0, `expected a cache key embedding the fingerprint of A: ${catalogKeys.join(",")}`);
+  assert.ok(keysWithDigestB.length > 0, `expected a cache key embedding the fingerprint of B: ${catalogKeys.join(",")}`);
+
+  // Raw secrets must never appear (issue #10313 root cause).
+  assert.ok(!catalogKeys.some((k) => k.includes(rawA) || k.includes(rawB)));
+
+  // Identical secrets ⇒ identical key (memoized reuse); different ⇒ distinct.
+  assert.ok(keysWithDigestA.every((k) => k === keysWithDigestA[0]), "all A keys must be identical");
+  assert.ok(keysWithDigestB.every((k) => k === keysWithDigestB[0]), "all B keys must be identical");
+  assert.notEqual(keysWithDigestA[0], keysWithDigestB[0]);
+});