perf: settings/combo read caches, async fs routes, bounded limiter settings

changelog.d/maintenance/17304-settings-combo-read-caches-async-fs-limiter-cap.md

@@ -0,0 +1 @@
+- **perf:** the hot-path settings and combo reads are served from short-TTL caches that settings/combo writes invalidate, the MITM settings, OpenAPI spec and storage-health routes use async `fs/promises` instead of blocking the event loop, and the preserved rate-limiter settings map is now a bounded 512-entry LRU ([#17304](https://github.com/diegosouzapw/OmniRoute/pull/17304)) — thanks @mayankjaiswal1707-spec

open-sse/services/rateLimitManager.ts

@@ -119,7 +119,16 @@ export const ZAI_WEB_REQUEST_QUEUE_MAX_WAIT_MS = 60_000;
 export const MAXAI_REQUEST_QUEUE_MAX_WAIT_MS = 300_000;
 
 const limiterEffectiveSettings = new WeakMap<Bottleneck, Bottleneck.ConstructorOptions>();
-const preservedReplacementSettings = new Map<string, Bottleneck.ConstructorOptions>();
+// Wedge evictions stash per-key limiter settings here for the NEXT replacement
+// (preserved reservoir carry-forward). A key that is never requested again
+// (model churn) would keep its entry for the whole process lifetime, so this
+// is LRU-capped: evicting an entry only means a replacement rebuilds from
+// defaults instead of carried settings, which is always safe.
+const preservedReplacementSettings = boundedMap<Bottleneck.ConstructorOptions>(
+  "preserved-limiter-settings",
+  512,
+  "lru"
+);
 const limiterWatchdog = new LimiterWedgeWatchdog({
   limiters,
   limiterLastUsed,
@@ -416,11 +425,10 @@ export async function initializeRateLimits() {
   applyBottleneckHeartbeatPatch();
 
   try {
-    const { getCachedProviderConnections } = await import("@/lib/db/readCache");
-    const { getSettings } = await import("@/lib/db/settings");
+    const { getCachedProviderConnections, getCachedSettings } = await import("@/lib/db/readCache");
     const [connections, settings] = await Promise.all([
       getCachedProviderConnections(),
-      getSettings(),
+      getCachedSettings(),
     ]);
     const resilience = resolveResilienceSettings(settings);
     currentRequestQueueSettings = { ...resilience.requestQueue };
@@ -1164,8 +1172,8 @@ export async function __getLimiterStateForTests(provider, connectionId, model =
  */
 async function loadPersistedLimits() {
   try {
-    const { getSettings } = await import("@/lib/db/settings");
-    const settings = await getSettings();
+    const { getCachedSettings } = await import("@/lib/db/readCache");
+    const settings = await getCachedSettings();
     const raw = settings?.learnedRateLimits;
     if (typeof raw !== "string" || raw.trim().length === 0) return;
 

open-sse/services/rateLimitManager/wedgeWatchdog.ts

@@ -14,7 +14,9 @@ interface LimiterWedgeWatchdogDependencies {
   limiters: Map<string, Bottleneck>;
   limiterLastUsed: Map<string, number>;
   limiterEffectiveSettings: WeakMap<Bottleneck, Bottleneck.ConstructorOptions>;
-  preservedReplacementSettings: Map<string, Bottleneck.ConstructorOptions>;
+  // boundedMap in production (LRU-capped), plain Map in some tests — the
+  // watchdog only reads and writes entries.
+  preservedReplacementSettings: Pick<Map<string, Bottleneck.ConstructorOptions>, "get" | "set">;
   trackBackground: (promise: Promise<unknown>) => void;
   log: (...args: unknown[]) => void;
   warn: (...args: unknown[]) => void;

open-sse/utils/usageTracking.ts

@@ -142,8 +142,8 @@ function getBufferTokens(): number {
 
 async function _loadBufferFromDb(): Promise<void> {
   try {
-    const { getSettings } = await import("@/lib/db/settings");
-    const settings = await getSettings();
+    const { getCachedSettings } = await import("@/lib/db/readCache");
+    const settings = await getCachedSettings();
     const val = settings.usageTokenBuffer;
     if (typeof val === "number" && val >= 0) {
       _cachedBuffer = val;

src/app/api/openapi/spec/route.ts

@@ -4,7 +4,7 @@
  */
 
 import { NextResponse } from "next/server";
-import fs from "fs";
+import { access, readFile, stat } from "fs/promises";
 import path from "path";
 import * as yaml from "js-yaml";
 import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/error";
@@ -116,25 +116,28 @@ export async function GET() {
   try {
     let specPath = "";
     for (const p of OPENAPI_SPEC_CANDIDATES) {
-      if (fs.existsSync(p)) {
+      try {
+        await access(p);
         specPath = p;
         break;
+      } catch {
+        // Candidate missing — try the next one.
       }
     }
 
     if (!specPath) {
       return NextResponse.json({ error: "openapi.yaml not found" }, { status: 404 });
     }
 
-    const stat = fs.statSync(specPath);
-    const mtime = stat.mtimeMs;
+    const specStat = await stat(specPath);
+    const mtime = specStat.mtimeMs;
 
     // Use cache if file hasn't changed
     if (cachedSpec && cachedSpec.mtime === mtime) {
       return NextResponse.json(cachedSpec.data);
     }
 
-    const content = fs.readFileSync(specPath, "utf-8");
+    const content = await readFile(specPath, "utf-8");
     const raw: any = yaml.load(content);
 
     // Build a structured catalog

src/app/api/settings/mitm/route.ts

@@ -1,6 +1,6 @@
 export const runtime = "nodejs";
 
-import fs from "fs";
+import fs from "fs/promises";
 import path from "path";
 import { z } from "zod";
 import { NextResponse } from "next/server";
@@ -72,6 +72,15 @@ function getStatsPath() {
   return path.join(getMitmDir(), "stats.json");
 }
 
+async function fileExists(filePath: string): Promise<boolean> {
+  try {
+    await fs.access(filePath);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
 function defaultTargets(port = DEFAULT_PORT): MitmTargetRoute[] {
   const allHosts = [
     ANTIGRAVITY_MITM_PROFILE.targetHost,
@@ -99,9 +108,9 @@ function defaultTargets(port = DEFAULT_PORT): MitmTargetRoute[] {
   ];
 }
 
-function readConfig(): MitmConfig {
+async function readConfig(): Promise<MitmConfig> {
   try {
-    JSON.parse(fs.readFileSync(getConfigPath(), "utf8"));
+    JSON.parse(await fs.readFile(getConfigPath(), "utf8"));
     return {
       port: DEFAULT_PORT,
       targets: defaultTargets(DEFAULT_PORT),
@@ -114,15 +123,15 @@ function readConfig(): MitmConfig {
   }
 }
 
-function writeConfig() {
+async function writeConfig() {
   const mitmDir = getMitmDir();
-  fs.mkdirSync(mitmDir, { recursive: true });
-  fs.writeFileSync(getConfigPath(), JSON.stringify({ port: DEFAULT_PORT }, null, 2));
+  await fs.mkdir(mitmDir, { recursive: true });
+  await fs.writeFile(getConfigPath(), JSON.stringify({ port: DEFAULT_PORT }, null, 2));
 }
 
-function readStats(): MitmStats {
+async function readStats(): Promise<MitmStats> {
   try {
-    const raw = JSON.parse(fs.readFileSync(getStatsPath(), "utf8"));
+    const raw = JSON.parse(await fs.readFile(getStatsPath(), "utf8"));
     return {
       startedAt: typeof raw.startedAt === "string" ? raw.startedAt : null,
       totalRequests: Number(raw.totalRequests || 0),
@@ -146,16 +155,16 @@ function readStats(): MitmStats {
 async function buildMitmResponse() {
   const { getMitmStatus, getCachedPassword } = await import("@/mitm/manager.runtime");
   const status = await getMitmStatus();
-  const config = readConfig();
-  const stats = readStats();
+  const config = await readConfig();
+  const stats = await readStats();
 
   return {
     running: status.running,
     pid: status.pid || null,
     dnsConfigured: status.dnsConfigured || false,
     // The cert of the ACTIVE model (ca.crt under the root-CA model,
     // server.crt on a legacy-leaf install) — never a stale legacy leaf.
-    certExists: status.certExists || fs.existsSync(resolveActiveMitmCert().certPath),
+    certExists: status.certExists || (await fileExists(resolveActiveMitmCert().certPath)),
     hasCachedPassword: !!getCachedPassword(),
     port: config.port,
     targets: config.targets,
@@ -171,10 +180,10 @@ export async function GET(request: Request) {
     const { searchParams } = new URL(request.url);
     if (searchParams.get("download") === "cert") {
       const { certPath, mode } = resolveActiveMitmCert();
-      if (!fs.existsSync(certPath)) {
+      if (!(await fileExists(certPath))) {
         return NextResponse.json({ error: activeCertMissingMessage(mode) }, { status: 404 });
       }
-      return new NextResponse(fs.readFileSync(certPath), {
+      return new NextResponse(await fs.readFile(certPath), {
         headers: {
           "Content-Type": "application/x-pem-file",
           "Content-Disposition": `attachment; filename="${activeCertDownloadName(mode)}"`,
@@ -202,15 +211,15 @@ export async function PUT(request: Request) {
       return NextResponse.json({ error: parsed.error.flatten() }, { status: 400 });
     }
 
-    const config = readConfig();
+    const config = await readConfig();
     if (parsed.data.port !== undefined && parsed.data.port !== DEFAULT_PORT) {
       return NextResponse.json({ error: MITM_PORT_ERROR }, { status: 400 });
     }
 
     if (parsed.data.port !== undefined) {
       config.port = DEFAULT_PORT;
       config.targets = defaultTargets(config.port);
-      writeConfig();
+      await writeConfig();
     }
 
     if (typeof parsed.data.enabled === "boolean") {

src/app/api/storage/health/route.ts

@@ -1,6 +1,6 @@
 import { NextResponse } from "next/server";
 import path from "path";
-import fs from "fs";
+import { readdir, stat } from "fs/promises";
 import { resolveDataDir } from "@/lib/dataPaths";
 import {
   getAppLogRetentionDays,
@@ -24,10 +24,8 @@ export async function GET() {
     // Get DB file size
     let sizeBytes = 0;
     try {
-      if (fs.existsSync(dbFilePath)) {
-        const stat = fs.statSync(dbFilePath);
-        sizeBytes = stat.size;
-      }
+      const fileStat = await stat(dbFilePath);
+      sizeBytes = fileStat.size;
     } catch {
       /* ignore */
     }
@@ -36,17 +34,14 @@ export async function GET() {
     let lastBackupAt = null;
     let backupCount = 0;
     try {
-      if (fs.existsSync(backupsDir)) {
-        const files = fs
-          .readdirSync(backupsDir)
-          .filter((f) => f.startsWith("db_") && f.endsWith(".sqlite"))
-          .sort()
-          .reverse();
-        backupCount = files.length;
-        if (files.length > 0) {
-          const latestStat = fs.statSync(path.join(backupsDir, files[0]));
-          lastBackupAt = latestStat.mtime.toISOString();
-        }
+      const files = (await readdir(backupsDir))
+        .filter((f) => f.startsWith("db_") && f.endsWith(".sqlite"))
+        .sort()
+        .reverse();
+      backupCount = files.length;
+      if (files.length > 0) {
+        const latestStat = await stat(path.join(backupsDir, files[0]));
+        lastBackupAt = latestStat.mtime.toISOString();
       }
     } catch {
       /* ignore */

src/lib/db/modelComboMappings.ts

@@ -9,6 +9,7 @@ import type {
   UpdateModelComboMappingInput,
 } from "@/domain/persistence/comboRepositories";
 import { routingConfigRepositories } from "./repositories/routingConfigRepositories";
+import { invalidateDbCache } from "./readCache";
 
 export type { ModelComboMapping } from "@/domain/persistence/comboRepositories";
 
@@ -25,21 +26,29 @@ export function getModelComboMappingById(id: string): Promise<ModelComboMapping
   return repository.findById(id);
 }
 
-export function createModelComboMapping(
+export async function createModelComboMapping(
   data: CreateModelComboMappingInput
 ): Promise<ModelComboMapping> {
-  return repository.create(data);
+  const mapping = await repository.create(data);
+  // Bust the version-keyed combo lookup caches (readCache) so
+  // resolveComboForModel / combo-name policy checks see the new mapping.
+  invalidateDbCache("combos");
+  return mapping;
 }
 
-export function updateModelComboMapping(
+export async function updateModelComboMapping(
   id: string,
   data: UpdateModelComboMappingInput
 ): Promise<ModelComboMapping | null> {
-  return repository.update(id, data);
+  const mapping = await repository.update(id, data);
+  if (mapping) invalidateDbCache("combos");
+  return mapping;
 }
 
-export function deleteModelComboMapping(id: string): Promise<boolean> {
-  return repository.deleteById(id);
+export async function deleteModelComboMapping(id: string): Promise<boolean> {
+  const deleted = await repository.deleteById(id);
+  if (deleted) invalidateDbCache("combos");
+  return deleted;
 }
 
 export function resolveComboForModel(model: string): Promise<Record<string, unknown> | null> {

src/lib/db/readCache.ts

@@ -249,6 +249,38 @@ export function getCombosCacheVersion(): number {
   return combosCacheVersion;
 }
 
+// ──────────────── Combo Name / Mapping Read Cache ────────────────
+//
+// apiKeyPolicy.resolveRequestedComboName and model.ts.getCombo issue up to 3
+// getComboByName queries per request, and resolveComboForModel runs for every
+// non-combo model. Entries are keyed by combosCacheVersion, so combo or
+// mapping writes (invalidateDbCache("combos")) bust them immediately — same
+// contract as the handler-level caches above. 10s TTL mirrors
+// getCombosCachedForChat.
+const COMBO_LOOKUP_TTL_MS = 10_000;
+const comboByNameCache = new TTLCache<Record<string, unknown> | null>(COMBO_LOOKUP_TTL_MS, 500);
+const comboForModelCache = new TTLCache<Record<string, unknown> | null>(COMBO_LOOKUP_TTL_MS, 500);
+
+/** Cached getComboByName — version-keyed, so combo writes take effect at once. */
+export async function getCachedComboByName(name: string): Promise<Record<string, unknown> | null> {
+  const key = `${getCombosCacheVersion()}:${name}`;
+  return comboByNameCache.load(key, async () => {
+    const { getComboByName } = await import("@/lib/db/combos");
+    return getComboByName(name);
+  });
+}
+
+/** Cached resolveComboForModel — same version-keyed contract as above. */
+export async function getCachedComboForModel(
+  model: string
+): Promise<Record<string, unknown> | null> {
+  const key = `${getCombosCacheVersion()}:${model}`;
+  return comboForModelCache.load(key, async () => {
+    const { resolveComboForModel } = await import("@/lib/db/modelComboMappings");
+    return resolveComboForModel(model);
+  });
+}
+
 // ──────────────── Model Catalog Cache Invalidation Signal ────────────────
 //
 // #6408 added a request-shape-keyed (prefix/isCodex/apiKey/configuredOnly) TTL

src/shared/utils/apiAuth.ts

@@ -8,7 +8,7 @@
  */
 
 import { getOrCreateBootstrapToken, peekBootstrapToken } from "@/lib/auth/bootstrapToken";
-import { getSettings } from "@/lib/db/settings";
+import { getCachedSettings } from "@/lib/db/readCache";
 import {
   AUTHZ_HEADER_PEER_LOCALITY,
   BOOTSTRAP_TOKEN_HEADER,
@@ -495,7 +495,11 @@ export async function isAuthRequired(
   options?: AuthRequiredOptions
 ): Promise<boolean> {
   try {
-    const settings = await getSettings();
+    // Cached read (5s TTL): updateSettings() busts it synchronously, so
+    // requireLogin flips take effect on the very next request. Writers that
+    // bypass updateSettings() (the CLI settings-store.mjs in another process)
+    // can lag by up to the 5s TTL.
+    const settings = await getCachedSettings();
     if (settings.requireLogin === false) return false;
 
     if (

src/shared/utils/apiKeyPolicy.ts

@@ -11,9 +11,8 @@
 import { extractApiKey } from "@/sse/services/auth";
 import { resolveClientApiKey } from "@/sse/services/clientApiKeyResolver";
 import { getApiKeyMetadata, isModelAllowedForKey, getApiKeyById } from "@/lib/db/apiKeys";
-import { getComboByName } from "@/lib/db/combos";
+import { getCachedComboByName, getCachedComboForModel } from "@/lib/db/readCache";
 import { isDashboardSessionAuthenticated } from "./apiAuth";
-import { resolveComboForModel } from "@/lib/db/modelComboMappings";
 import { checkBudget } from "@/domain/costRules";
 import { checkKeyQuota } from "@/domain/keyQuota";
 import { checkTokenLimits } from "@omniroute/open-sse/services/tokenLimitCounter.ts";
@@ -263,16 +262,16 @@ function policyErrorResponse(
 }
 
 async function resolveRequestedComboName(modelStr: string): Promise<string | null> {
-  const exact = await getComboByName(modelStr);
+  const exact = await getCachedComboByName(modelStr);
   if (exact && typeof exact.name === "string") return exact.name;
 
   if (modelStr.startsWith("combo/")) {
     const withoutPrefix = modelStr.slice(6);
-    const prefixed = await getComboByName(withoutPrefix);
+    const prefixed = await getCachedComboByName(withoutPrefix);
     if (prefixed && typeof prefixed.name === "string") return prefixed.name;
   }
 
-  const mapped = await resolveComboForModel(modelStr);
+  const mapped = await getCachedComboForModel(modelStr);
   const mappedName = normalizeComboAccessName(mapped?.name);
   return mappedName;
 }