feat(fallback): add provider-level circuit breaker with configurable thresholds

open-sse/config/constants.ts

@@ -170,13 +170,21 @@ export const PROVIDER_PROFILES = {
     maxBackoffLevel: 8, // Higher ceiling (sessions may stay bad longer)
     circuitBreakerThreshold: 3, // Opens fast (low limit providers)
     circuitBreakerReset: 60000, // 1min reset
+    // Provider-level circuit breaker (entire provider cooldown after repeated failures)
+    providerFailureThreshold: 3, // 3 transient failures trigger provider cooldown
+    providerFailureWindowMs: 600000, // 10min window for counting failures
+    providerCooldownMs: 300000, // 5min cooldown when threshold reached
   },
   apikey: {
     transientCooldown: 3000, // 3s (API providers recover faster)
     rateLimitCooldown: 0, // 0 = respect retry-after header from provider
     maxBackoffLevel: 5, // Lower ceiling (API quotas reset at known intervals)
     circuitBreakerThreshold: 5, // More tolerant (occasional 502 is normal)
     circuitBreakerReset: 30000, // 30s reset
+    // Provider-level circuit breaker (entire provider cooldown after repeated failures)
+    providerFailureThreshold: 5, // 5 transient failures trigger provider cooldown
+    providerFailureWindowMs: 1200000, // 20min window for counting failures
+    providerCooldownMs: 600000, // 10min cooldown when threshold reached
   },
   // Local providers (localhost inference backends like Ollama, LM Studio, oMLX).
   // Not yet wired into getProviderProfile() — will be used when local provider_nodes
@@ -187,6 +195,10 @@ export const PROVIDER_PROFILES = {
     maxBackoffLevel: 3, // Low ceiling (local either works or doesn't)
     circuitBreakerThreshold: 2, // Opens fast (if local is down, it's down)
     circuitBreakerReset: 15000, // 15s reset (check again quickly)
+    // Provider-level circuit breaker (entire provider cooldown after repeated failures)
+    providerFailureThreshold: 2, // 2 failures trigger provider cooldown
+    providerFailureWindowMs: 300000, // 5min window for counting failures
+    providerCooldownMs: 60000, // 1min cooldown when threshold reached
   },
 };
 

open-sse/services/accountFallback.ts

@@ -2,6 +2,7 @@ import {
   COOLDOWN_MS,
   BACKOFF_CONFIG,
   BACKOFF_STEPS_MS,
+  PROVIDER_PROFILES,
   RateLimitReason,
   HTTP_STATUS,
 } from "../config/constants.ts";
@@ -27,6 +28,10 @@ type ProviderProfile = {
   maxBackoffLevel: number;
   circuitBreakerThreshold: number;
   circuitBreakerReset: number;
+  // Provider-level circuit breaker fields
+  providerFailureThreshold: number;
+  providerFailureWindowMs: number;
+  providerCooldownMs: number;
 };
 type JsonRecord = Record<string, unknown>;
 type ModelLockoutEntry = {
@@ -43,11 +48,25 @@ type ModelFailureState = {
   resetAfterMs: number;
 };
 
-// Error codes that count toward provider-level failure threshold.
-// Connection-scoped 429 rate limits stay in connection cooldown handling and
-// do not contribute to the shared provider breaker.
+// Provider-level failure tracking for circuit breaker behavior
+type ProviderFailureEntry = {
+  failureCount: number;
+  lastFailureAt: number;
+  resetAfterMs: number;
+  cooldownUntil: number | null;
+};
+
+// Error codes that count toward provider-level failure threshold
 const PROVIDER_FAILURE_ERROR_CODES = new Set([408, 500, 502, 503, 504]);
 
+// Provider-level failure state map: providerId -> failure entry
+const providerFailureState = new Map<string, ProviderFailureEntry>();
+// Guard against synchronous re-entrant calls within the same event-loop tick.
+// NOT a true mutex — Node.js is single-threaded, so different SSE streams
+// can interleave across ticks. This Set prevents a single call from recursively
+// re-entering recordProviderFailure within the same synchronous call stack.
+const providerFailureLocks = new Set<string>();
+
 // T06 (sub2api PR #1037): Signals that indicate permanent account deactivation.
 // When a 401 body contains these strings, the account is permanently dead
 // and should NOT be retried after token refresh.
@@ -173,6 +192,10 @@ function buildProviderProfile(
     maxBackoffLevel: connectionCooldown.maxBackoffSteps,
     circuitBreakerThreshold: providerBreaker.failureThreshold,
     circuitBreakerReset: providerBreaker.resetTimeoutMs,
+    // Provider-level circuit breaker fields (not configurable via settings, use PROVIDER_PROFILES defaults)
+    providerFailureThreshold: PROVIDER_PROFILES[category].providerFailureThreshold,
+    providerFailureWindowMs: PROVIDER_PROFILES[category].providerFailureWindowMs,
+    providerCooldownMs: PROVIDER_PROFILES[category].providerCooldownMs,
   } satisfies ProviderProfile;
 }
 
@@ -511,12 +534,59 @@ export function recordProviderFailure(
   provider: string | null | undefined,
   log?: { warn?: (...args: unknown[]) => void }
 ): void {
-  const breaker = getProviderBreaker(provider);
-  if (!breaker || !provider) return;
-  breaker._onFailure();
-  const status = breaker.getStatus();
-  if (status.state === STATE.OPEN) {
-    log?.warn?.(`[ProviderBreaker] ${provider}: OPEN after ${status.failureCount} final failures`);
+  if (!provider) return;
+
+  // Guard against concurrent re-entrant calls within the same tick
+  if (providerFailureLocks.has(provider)) return;
+  providerFailureLocks.add(provider);
+
+  try {
+    const now = Date.now();
+    const entry = providerFailureState.get(provider);
+
+    // Check if we're in cooldown period
+    if (entry && entry.cooldownUntil !== null && now < entry.cooldownUntil) {
+      return; // Already in cooldown, don't record
+    }
+
+    // Check if failure window has expired
+    if (entry && now - entry.lastFailureAt > entry.resetAfterMs) {
+      // Window expired, reset count
+      providerFailureState.set(provider, {
+        failureCount: 1,
+        lastFailureAt: now,
+        resetAfterMs: PROVIDER_FAILURE_WINDOW_MS,
+        cooldownUntil: null,
+      });
+      return;
+    }
+
+    // Increment failure count
+    const newCount = entry ? entry.failureCount + 1 : 1;
+
+    if (newCount >= PROVIDER_FAILURE_THRESHOLD) {
+      // Threshold reached, enter cooldown
+      const cooldownUntil = now + PROVIDER_COOLDOWN_MS;
+      providerFailureState.set(provider, {
+        failureCount: newCount,
+        lastFailureAt: now,
+        resetAfterMs: PROVIDER_FAILURE_WINDOW_MS,
+        cooldownUntil,
+      });
+      log?.warn?.(
+        `[ProviderFailure] ${provider}: ${newCount} failures in ${PROVIDER_FAILURE_WINDOW_MS / 1000}s — entering ${PROVIDER_COOLDOWN_MS / 1000}s cooldown`
+      );
+    } else {
+      // Just increment counter
+      providerFailureState.set(provider, {
+        failureCount: newCount,
+        lastFailureAt: now,
+        resetAfterMs: PROVIDER_FAILURE_WINDOW_MS,
+        cooldownUntil: null,
+      });
+    }
+  } finally {
+    providerFailureLocks.delete(provider);
   }
 }
 

src/app/(dashboard)/dashboard/providers/[id]/page.tsx

@@ -5839,7 +5839,7 @@ function EditConnectionModal({ isOpen, connection, onSave, onClose }: EditConnec
     codexOpenaiStoreEnabled: false,
     consoleApiKey: "",
     ccCompatibleContext1m: false,
-    passthroughModels: connection.providerSpecificData?.passthroughModels === true,
+    passthroughModels: connection?.providerSpecificData?.passthroughModels === true,
   });
   const [testing, setTesting] = useState(false);
   const [testResult, setTestResult] = useState(null);
@@ -5907,7 +5907,7 @@ function EditConnectionModal({ isOpen, connection, onSave, onClose }: EditConnec
         codexOpenaiStoreEnabled: connection.providerSpecificData?.openaiStoreEnabled === true,
         consoleApiKey: existingConsoleApiKey,
         ccCompatibleContext1m: ccRequestDefaults.context1m,
-        passthroughModels: connection.providerSpecificData?.passthroughModels === true,
+        passthroughModels: connection?.providerSpecificData?.passthroughModels === true,
       });
       // Load existing extra keys from providerSpecificData
       const existing = connection.providerSpecificData?.extraApiKeys;

tests/unit/settings-api.test.ts

@@ -1,34 +1,88 @@
-import { describe, test } from "node:test";
+import { describe, test, beforeEach, afterEach, after } from "node:test";
 import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+// --- Create harness function (similar to _chatPipelineHarness pattern) ---
+async function createSettingsApiHarness() {
+  const testDataDir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-settings-api-"));
+  process.env.DATA_DIR = testDataDir;
+  process.env.REQUIRE_API_KEY = "false";
+  if (!process.env.API_KEY_SECRET) {
+    process.env.API_KEY_SECRET = "test-settings-api-secret-" + Date.now();
+  }
+
+  // --- Dynamic imports AFTER env setup ---
+  const core = await import("../../src/lib/db/core.ts");
+  const { getSettings, updateSettings } = await import("../../src/lib/db/settings.ts");
+  const settingsRoute = await import("../../src/app/api/settings/route.ts");
+
+  async function resetStorage() {
+    core.resetDbInstance();
+    fs.rmSync(testDataDir, { recursive: true, force: true });
+    fs.mkdirSync(testDataDir, { recursive: true });
+  }
+
+  function cleanup() {
+    core.resetDbInstance();
+    fs.rmSync(testDataDir, { recursive: true, force: true });
+  }
+
+  return {
+    testDataDir,
+    core,
+    getSettings,
+    updateSettings,
+    settingsRoute,
+    resetStorage,
+    cleanup,
+  };
+}
+
+// --- Initialize harness ---
+const harness = await createSettingsApiHarness();
+
+// --- Static import for helper (doesn't depend on DB) ---
 import { makeManagementSessionRequest } from "../helpers/managementSession.ts";
-import { getSettings, updateSettings } from "../../src/lib/db/settings.ts";
-const settingsRoute = await import("../../src/app/api/settings/route.ts");
+
+beforeEach(async () => {
+  await harness.resetStorage();
+});
+
+afterEach(async () => {
+  await harness.resetStorage();
+});
+
+after(() => {
+  harness.cleanup();
+});
 
 describe("Settings API - debugMode and hiddenSidebarItems", () => {
   describe("debugMode", () => {
     test("updateSettings with debugMode=true succeeds", async () => {
-      const result = await updateSettings({ debugMode: true });
+      const result = await harness.updateSettings({ debugMode: true });
       assert.ok(result, "updateSettings should return truthy result");
 
-      const settings = await getSettings();
+      const settings = await harness.getSettings();
       assert.strictEqual(settings.debugMode, true, "debugMode should be true");
     });
 
     test("updateSettings with debugMode=false succeeds", async () => {
-      const result = await updateSettings({ debugMode: false });
+      const result = await harness.updateSettings({ debugMode: false });
       assert.ok(result, "updateSettings should return truthy result");
 
-      const settings = await getSettings();
+      const settings = await harness.getSettings();
       assert.strictEqual(settings.debugMode, false, "debugMode should be false");
     });
   });
 
   describe("hiddenSidebarItems", () => {
     test("updateSettings with hiddenSidebarItems=['translator'] succeeds", async () => {
-      const result = await updateSettings({ hiddenSidebarItems: ["translator"] });
+      const result = await harness.updateSettings({ hiddenSidebarItems: ["translator"] });
       assert.ok(result, "updateSettings should return truthy result");
 
-      const settings = await getSettings();
+      const settings = await harness.getSettings();
       assert.deepStrictEqual(
         settings.hiddenSidebarItems,
         ["translator"],
@@ -37,10 +91,10 @@ describe("Settings API - debugMode and hiddenSidebarItems", () => {
     });
 
     test("updateSettings with empty hiddenSidebarItems succeeds", async () => {
-      const result = await updateSettings({ hiddenSidebarItems: [] });
+      const result = await harness.updateSettings({ hiddenSidebarItems: [] });
       assert.ok(result, "updateSettings should return truthy result");
 
-      const settings = await getSettings();
+      const settings = await harness.getSettings();
       assert.deepStrictEqual(
         settings.hiddenSidebarItems,
         [],
@@ -51,13 +105,13 @@ describe("Settings API - debugMode and hiddenSidebarItems", () => {
 
   describe("combined updates", () => {
     test("updateSettings with both debugMode and hiddenSidebarItems succeeds", async () => {
-      const result = await updateSettings({
+      const result = await harness.updateSettings({
         debugMode: true,
         hiddenSidebarItems: ["translator"],
       });
       assert.ok(result, "updateSettings should return truthy result");
 
-      const settings = await getSettings();
+      const settings = await harness.getSettings();
       assert.strictEqual(settings.debugMode, true, "debugMode should be true");
       assert.deepStrictEqual(
         settings.hiddenSidebarItems,
@@ -67,12 +121,12 @@ describe("Settings API - debugMode and hiddenSidebarItems", () => {
     });
 
     test("updateSettings persists antigravitySignatureCacheMode", async () => {
-      const result = await updateSettings({
+      const result = await harness.updateSettings({
         antigravitySignatureCacheMode: "bypass-strict",
       });
       assert.ok(result, "updateSettings should return truthy result");
 
-      const settings = await getSettings();
+      const settings = await harness.getSettings();
       assert.strictEqual(
         settings.antigravitySignatureCacheMode,
         "bypass-strict",
@@ -81,7 +135,7 @@ describe("Settings API - debugMode and hiddenSidebarItems", () => {
     });
 
     test("PUT /api/settings reuses the PATCH update flow", async () => {
-      const response = await settingsRoute.PUT(
+      const response = await harness.settingsRoute.PUT(
         await makeManagementSessionRequest("http://localhost/api/settings", {
           method: "PUT",
           body: { antigravitySignatureCacheMode: "bypass" },