From d87343e614f205912d199e7af7db3405e309716b Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Sun, 30 Aug 2026 10:40:08 -0400 Subject: [PATCH] fix(search): treat HTTP 432 and plan limit errors as transient cooldown - Classify HTTP 432 and plan limit exhaustion strings as non-permanent QUOTA_EXHAUSTED - Add PLAN_LIMIT_EXCEEDED: 432 constant and 1-hour quota cooldown fallback - Trigger markAccountUnavailable in search executeProviderFetch strictly for quota/retriable statuses - Enable search multi-key auto-rotation to healthy connections without deactivating accounts - Add unit and regression tests including negative coverage for 400/401/403/404 Signed-off-by: Minxi Hou --- open-sse/config/constants.ts | 1 + open-sse/handlers/search/searchProxy.ts | 26 ++ open-sse/services/accountFallback.ts | 21 +- open-sse/services/quotaTextCooldowns.ts | 7 +- .../search-432-plan-limit-cooldown.test.ts | 288 ++++++++++++++++++ 5 files changed, 341 insertions(+), 2 deletions(-) create mode 100644 tests/unit/search-432-plan-limit-cooldown.test.ts diff --git a/open-sse/config/constants.ts b/open-sse/config/constants.ts index d99da4725b05..ebe568d2d3b5 100644 --- a/open-sse/config/constants.ts +++ b/open-sse/config/constants.ts @@ -175,6 +175,7 @@ export const HTTP_STATUS = { REQUEST_TIMEOUT: 408, GONE: 410, RATE_LIMITED: 429, + PLAN_LIMIT_EXCEEDED: 432, SERVER_ERROR: 500, BAD_GATEWAY: 502, SERVICE_UNAVAILABLE: 503, diff --git a/open-sse/handlers/search/searchProxy.ts b/open-sse/handlers/search/searchProxy.ts index ec2d855858fc..ffc7872299af 100644 --- a/open-sse/handlers/search/searchProxy.ts +++ b/open-sse/handlers/search/searchProxy.ts @@ -11,9 +11,27 @@ import { saveCallLog } from "@/lib/usageDb"; import { sanitizeErrorMessage } from "../../utils/error.ts"; import { formatSearchProviderFailure } from "./providerFailure.ts"; +import { HTTP_STATUS } from "../../config/constants.ts"; +import { isSubscriptionQuotaText } from "../../services/quotaTextCooldowns.ts"; import type { SearchProviderConfig } from "../../config/searchRegistry.ts"; import type { SearchResult } from "../search.ts"; +const SEARCH_COOLDOWN_STATUSES = new Set([ + HTTP_STATUS.PAYMENT_REQUIRED, + HTTP_STATUS.REQUEST_TIMEOUT, + HTTP_STATUS.RATE_LIMITED, + HTTP_STATUS.PLAN_LIMIT_EXCEEDED, + HTTP_STATUS.SERVER_ERROR, + HTTP_STATUS.BAD_GATEWAY, + HTTP_STATUS.SERVICE_UNAVAILABLE, + HTTP_STATUS.GATEWAY_TIMEOUT, +]); + +export function shouldCoolDownSearchConnection(status: number, errorText: string): boolean { + if (SEARCH_COOLDOWN_STATUSES.has(status)) return true; + return isSubscriptionQuotaText(errorText.toLowerCase()); +} + /** Resolved proxy binding for a single provider attempt. */ export interface ResolvedSearchProxy { proxy: unknown; @@ -196,6 +214,14 @@ export async function executeProviderFetch( if (log) { log.error("SEARCH", `${config.id} error ${response.status}: ${errorText.slice(0, 200)}`); } + if (connectionId && shouldCoolDownSearchConnection(response.status, errorText)) { + try { + const { markAccountUnavailable } = await import("@/sse/services/auth.ts"); + await markAccountUnavailable(connectionId, response.status, errorText, config.id, null); + } catch { + /* non-critical - background cooldown mark must not break search response */ + } + } logCall({ status: response.status, duration: Date.now() - startTime, diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index 4a29105e6af3..b5e5fe7b34c1 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -77,6 +77,7 @@ import { buildSubscriptionQuotaFallback, buildWeeklyQuotaFallback, buildSessionQuotaFallback, + SUBSCRIPTION_QUOTA_COOLDOWN_MS, } from "./quotaTextCooldowns.ts"; import { parseDayGranularityResetMs, shouldPreserveQuotaSignals } from "./quotaResetParsing.ts"; import { evictLockoutOverflow } from "./accountFallback/lockoutEviction.ts"; @@ -1560,7 +1561,7 @@ export function classifyError( if (status === HTTP_STATUS.UNAUTHORIZED || status === HTTP_STATUS.FORBIDDEN) { return RateLimitReason.AUTH_ERROR; } - if (status === HTTP_STATUS.PAYMENT_REQUIRED) { + if (status === HTTP_STATUS.PAYMENT_REQUIRED || status === HTTP_STATUS.PLAN_LIMIT_EXCEEDED) { return RateLimitReason.QUOTA_EXHAUSTED; } if (status === HTTP_STATUS.RATE_LIMITED) { @@ -2131,6 +2132,24 @@ export function checkFallbackError( return buildRetryableFallback(RateLimitReason.SERVER_ERROR); } + // 432 -- plan limit reached (e.g. Tavily, Context7, and search upstreams) + if (status === HTTP_STATUS.PLAN_LIMIT_EXCEEDED) { + const subResult = buildSubscriptionQuotaFallback( + errorStr, + () => getUpstreamRetryHint()?.retryAfterMs ?? null, + parseRetryFromErrorText, + provider + ); + if (subResult) return subResult; + const cooldownMs = getUpstreamRetryHint()?.retryAfterMs ?? SUBSCRIPTION_QUOTA_COOLDOWN_MS; + return { + shouldFallback: true, + cooldownMs, + baseCooldownMs: cooldownMs, + reason: RateLimitReason.QUOTA_EXHAUSTED, + }; + } + // 400 — context overflow / malformed request / model access denied if (status === HTTP_STATUS.BAD_REQUEST) { // Check structured error codes first (more reliable, no false positives) diff --git a/open-sse/services/quotaTextCooldowns.ts b/open-sse/services/quotaTextCooldowns.ts index 2717d128dc9d..e5a13cf62f52 100644 --- a/open-sse/services/quotaTextCooldowns.ts +++ b/open-sse/services/quotaTextCooldowns.ts @@ -36,6 +36,11 @@ export function isSubscriptionQuotaText(lower: string, provider?: string | null) lower.includes("claude pro usage limit") || lower.includes("you've reached your usage limit") || lower.includes("you have reached your usage limit") || + lower.includes("exceeds your plan") || + lower.includes("plan limit") || + lower.includes("plan's set usage limit") || + lower.includes("plan limit exceeded") || + lower.includes("usage limit exceeded") || // Native Claude OAuth uses this otherwise-generic 429 wording for an // exhausted subscription window. Keep it provider-scoped: other upstreams // can use the same phrase for a short RPM throttle. @@ -43,7 +48,7 @@ export function isSubscriptionQuotaText(lower: string, provider?: string | null) ); } -const SUBSCRIPTION_QUOTA_COOLDOWN_MS = 60 * 60 * 1000; // 1 hour +export const SUBSCRIPTION_QUOTA_COOLDOWN_MS = 60 * 60 * 1000; // 1 hour /** * Builds the QUOTA_EXHAUSTED fallback for the subscription-quota text above. diff --git a/tests/unit/search-432-plan-limit-cooldown.test.ts b/tests/unit/search-432-plan-limit-cooldown.test.ts new file mode 100644 index 000000000000..074054678d03 --- /dev/null +++ b/tests/unit/search-432-plan-limit-cooldown.test.ts @@ -0,0 +1,288 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import http from "node:http"; + +interface TestConnectionRecord { + id?: string | number; + isActive?: boolean; + testStatus?: string; + rateLimitedUntil?: string | null; +} + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-test-search-432-")); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "search-432-test-secret"; +process.env.DISABLE_SQLITE_AUTO_BACKUP = "true"; + +const core = await import("../../src/lib/db/core.ts"); +const accountFallback = await import("../../open-sse/services/accountFallback.ts"); +const { RateLimitReason } = await import("../../open-sse/config/constants.ts"); +const quotaTextCooldowns = await import("../../open-sse/services/quotaTextCooldowns.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); +const auth = await import("../../src/sse/services/auth.ts"); +const connectionRecovery = await import("../../src/lib/quota/connectionRecovery.ts"); +const searchProxy = await import("../../open-sse/handlers/search/searchProxy.ts"); +const { closeCallLogSaves } = await import("../../src/lib/usage/callLogs.ts"); + +test.after(async () => { + await closeCallLogSaves(500).catch(() => {}); + try { + core.resetDbInstance(); + } catch {} + try { + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } catch {} +}); + +test("Tavily 432 plan limit body is detected as transient plan usage limit", () => { + const tavily432Body = JSON.stringify({ + error: "This request exceeds your plan's set usage limit. Please upgrade your plan or contact support@tavily.com", + }); + + const isMatched = quotaTextCooldowns.isSubscriptionQuotaText(tavily432Body.toLowerCase(), "tavily-search"); + assert.equal(isMatched, true, "Should recognize Tavily plan limit error message"); +}); + +test("checkFallbackError classifies status 432 and plan limit text as non-permanent quota_exhausted", () => { + const tavily432Body = JSON.stringify({ + error: "This request exceeds your plan's set usage limit. Please upgrade your plan or contact support@tavily.com", + }); + + const result = accountFallback.checkFallbackError( + 432, + tavily432Body, + "tavily-search", + null, + undefined, + undefined, + 0 + ); + + assert.equal(result.shouldFallback, true); + assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED); + assert.equal(result.permanent, undefined); + assert.equal(result.creditsExhausted, undefined); + assert.ok(result.cooldownMs > 0, "Should have a positive cooldown duration"); +}); + +test("markAccountUnavailable sets transient unavailable status without deactivating the connection", async () => { + const conn = await providersDb.createProviderConnection({ + provider: "tavily-search", + authType: "apikey", + name: "tavily-plan-limit-test", + apiKey: "tvly-test-key-12345", + isActive: true, + testStatus: "active", + }); + const connId = String(conn.id); + + const errorText = JSON.stringify({ + error: "This request exceeds your plan's set usage limit.", + }); + + await auth.markAccountUnavailable(connId, 432, errorText, "tavily-search", null); + + const updatedRaw = (await providersDb.getProviderConnections({ + provider: "tavily-search", + })) as TestConnectionRecord[]; + const updated = (Array.isArray(updatedRaw) ? updatedRaw : []).find( + (c) => String(c.id) === connId + ); + + assert.ok(updated, "Connection should exist in DB"); + assert.equal(updated.isActive, true, "Connection must remain isActive=1"); + assert.equal(updated.testStatus, "unavailable", "Connection should be marked transient unavailable"); + assert.ok(updated.rateLimitedUntil, "rateLimitedUntil must be populated"); + + const untilMs = new Date(updated.rateLimitedUntil).getTime(); + assert.ok(untilMs > Date.now(), "rateLimitedUntil should be in the future"); +}); + +test("executeProviderFetch calls markAccountUnavailable on 432 error response when connectionId is present", async () => { + let serverPort = 0; + const server = http.createServer((_req, res) => { + res.writeHead(432, { "Content-Type": "application/json", Connection: "close" }); + res.end(JSON.stringify({ + error: "This request exceeds your plan's set usage limit. Please upgrade your plan or contact support@tavily.com", + })); + }); + + await new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => { + const addr = server.address(); + if (addr && typeof addr === "object") serverPort = addr.port; + resolve(); + }); + }); + + const conn = await providersDb.createProviderConnection({ + provider: "tavily-search", + authType: "apikey", + name: "tavily-fetch-432-test", + apiKey: "tvly-test-key-fetch-432", + isActive: true, + testStatus: "active", + }); + const connId = String(conn.id); + + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 5000); + timer.unref?.(); + + try { + const { SEARCH_PROVIDERS } = await import("../../open-sse/config/searchRegistry.ts"); + const result = await searchProxy.executeProviderFetch({ + config: SEARCH_PROVIDERS["tavily-search"], + url: `http://127.0.0.1:${serverPort}/search`, + init: { method: "POST", headers: { "Content-Type": "application/json" } }, + controller, + timer, + query: "test query", + searchType: "web", + maxResults: 5, + startTime: Date.now(), + connectionId: connId, + proxy: null, + proxyLevel: "none", + normalize: () => ({ results: [], totalResults: 0 }), + }); + + assert.equal(result.success, false); + assert.equal(result.status, 432); + + const updatedRaw = (await providersDb.getProviderConnections({ + provider: "tavily-search", + })) as TestConnectionRecord[]; + const updated = (Array.isArray(updatedRaw) ? updatedRaw : []).find( + (c) => String(c.id) === connId + ); + + assert.ok(updated); + assert.equal(updated.isActive, true, "isActive should stay true"); + assert.equal(updated.testStatus, "unavailable", "Connection should become unavailable"); + assert.ok(updated.rateLimitedUntil, "rateLimitedUntil should be set"); + } finally { + clearTimeout(timer); + await new Promise((resolve) => { + server.closeAllConnections?.(); + server.close(() => resolve()); + }); + } +}); + +test("executeProviderFetch does NOT mark account unavailable on non-quota client errors (400, 401, 403, 404)", async () => { + for (const statusCode of [400, 401, 403, 404]) { + let serverPort = 0; + const server = http.createServer((_req, res) => { + res.writeHead(statusCode, { "Content-Type": "application/json", Connection: "close" }); + res.end(JSON.stringify({ error: `Generic client error ${statusCode}` })); + }); + + await new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => { + const addr = server.address(); + if (addr && typeof addr === "object") serverPort = addr.port; + resolve(); + }); + }); + + const conn = await providersDb.createProviderConnection({ + provider: "tavily-search", + authType: "apikey", + name: `tavily-fetch-${statusCode}-test`, + apiKey: `tvly-test-key-fetch-${statusCode}`, + isActive: true, + testStatus: "active", + }); + const connId = String(conn.id); + + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 5000); + timer.unref?.(); + + try { + const { SEARCH_PROVIDERS } = await import("../../open-sse/config/searchRegistry.ts"); + const result = await searchProxy.executeProviderFetch({ + config: SEARCH_PROVIDERS["tavily-search"], + url: `http://127.0.0.1:${serverPort}/search`, + init: { method: "POST", headers: { "Content-Type": "application/json" } }, + controller, + timer, + query: "test query", + searchType: "web", + maxResults: 5, + startTime: Date.now(), + connectionId: connId, + proxy: null, + proxyLevel: "none", + normalize: () => ({ results: [], totalResults: 0 }), + }); + + assert.equal(result.success, false); + assert.equal(result.status, statusCode); + + const updatedRaw = (await providersDb.getProviderConnections({ + provider: "tavily-search", + })) as TestConnectionRecord[]; + const updated = (Array.isArray(updatedRaw) ? updatedRaw : []).find( + (c) => String(c.id) === connId + ); + + assert.ok(updated); + assert.equal(updated.isActive, true, `isActive must remain true on ${statusCode}`); + assert.equal(updated.testStatus, "active", `testStatus must remain 'active' on ${statusCode}`); + assert.ok(!updated.rateLimitedUntil, `rateLimitedUntil must be null/empty on ${statusCode}`); + } finally { + clearTimeout(timer); + await new Promise((resolve) => { + server.closeAllConnections?.(); + server.close(() => resolve()); + }); + } + } +}); + +test("cooldown on key1 allows getProviderCredentials to auto-rotate to healthy key2", async () => { + const conn1 = await providersDb.createProviderConnection({ + provider: "tavily-search", + authType: "apikey", + name: "tavily-key-1", + apiKey: "tvly-key-1", + priority: 1, + isActive: true, + testStatus: "active", + }); + const conn2 = await providersDb.createProviderConnection({ + provider: "tavily-search", + authType: "apikey", + name: "tavily-key-2", + apiKey: "tvly-key-2", + priority: 2, + isActive: true, + testStatus: "active", + }); + + // Mark key1 as unavailable due to 432 + await auth.markAccountUnavailable(String(conn1.id), 432, "plan limit reached", "tavily-search", null); + + // Next credential resolution should skip key1 and return key2 + const selected = await auth.getProviderCredentials("tavily-search"); + assert.ok(selected, "Should return available credentials"); + assert.equal(String(selected.connectionId), String(conn2.id), "Should rotate to healthy key2"); +}); + +test("connectionRecovery restores elapsed unavailable connections", () => { + const pastTime = new Date(Date.now() - 5000).toISOString(); + const connInput = { + id: "test-conn-1", + testStatus: "unavailable", + rateLimitedUntil: pastTime, + lastErrorAt: pastTime, + }; + + const isRecoverable = connectionRecovery.isRecoverableCooldownConnection(connInput, Date.now()); + assert.equal(isRecoverable, true, "Elapsed unavailable connection should be recoverable"); +});