fix(resilience): retry Codex pre-output transport failures on the same account (#9708)

changelog.d/fixes/9708-codex-same-account-retry.md

@@ -0,0 +1 @@
+- **fix(resilience):** retry a retryable Codex pre-output 502/503/504/507 once on the same account (2–3s jitter) before cooling the connection, and stop translating that mixed pool into an all-accounts quota `429` ([#9708](https://github.com/diegosouzapw/OmniRoute/issues/9708))

src/sse/handlers/chat.ts

@@ -176,6 +176,10 @@ import {
   resolveCooldownAwareRetrySettings,
   waitForCooldownAwareRetry,
 } from "../services/cooldownAwareRetry";
+import {
+  shouldRetrySameAccountTransport,
+  sameAccountTransportRetryDelayMs,
+} from "../services/sameAccountTransportRetry";
 import { constrainConnectionsToQuota, resolveQuotaKeyScope } from "../../lib/quota/quotaKey";
 import { checkConnectionCapacity } from "../utils/backpressure";
 import {
@@ -333,7 +337,11 @@ function isManagedComboUnsupported(
 
 const managedComboRejection = () =>
   buildManagedLeaseErrorResponse(
-    new LeaseContextError(409, "LEASE_UNSUPPORTED_ROUTE", "Managed leases do not support this route")
+    new LeaseContextError(
+      409,
+      "LEASE_UNSUPPORTED_ROUTE",
+      "Managed leases do not support this route"
+    )
   );
 
 const comboPromoteDeps = { updateCombo, info: log.info, warn: log.warn };
@@ -1534,6 +1542,7 @@ async function handleSingleModelChat(
   // re-attempt to exactly one for the whole request. Declared outside both retry
   // loops so it can never reset and loop.
   let streamEarlyEofRetries = 0;
+  const sameAccountTransportRetries = new Map<string, number>();
   const occupancySessionKey =
     runtimeOptions.sessionAffinityKey ?? runtimeOptions.sessionId ?? `request:${randomUUID()}`;
   let initialPreselectedCredentials = runtimeOptions.preselectedCredentials;
@@ -1818,7 +1827,8 @@ async function handleSingleModelChat(
             comboStrategy,
             isCombo,
             comboStepId: runtimeOptions.comboStepId ?? null,
-            comboExecutionKey: runtimeOptions.comboExecutionKey ?? runtimeOptions.comboStepId ?? null,
+            comboExecutionKey:
+              runtimeOptions.comboExecutionKey ?? runtimeOptions.comboStepId ?? null,
             extendedContext,
             modelApiFormat: apiFormat,
             modelTargetFormat: targetFormat,
@@ -2201,6 +2211,35 @@ async function handleSingleModelChat(
         }
       }
 
+      // #9708: retry a retryable pre-output transport failure once on the same
+      // account (jittered 2-3s) before cooling the connection. A first 503/507
+      // must not rotate away from a still-healthy Codex prompt-cache partition.
+      const transportAttempts = sameAccountTransportRetries.get(credentials.connectionId) || 0;
+      if (
+        shouldRetrySameAccountTransport({
+          status: result.status,
+          errorText: errorStr,
+          errorCode: result.errorCode,
+          errorType: result.errorType,
+          attempt: transportAttempts,
+          hasForcedConnection,
+        })
+      ) {
+        sameAccountTransportRetries.set(credentials.connectionId, transportAttempts + 1);
+        const waitMs = sameAccountTransportRetryDelayMs();
+        log.warn(
+          "RETRY",
+          `${provider}/${model} retryable pre-output ${result.status} — retrying same account once after ${waitMs}ms`
+        );
+        const completed = await waitForCooldownAwareRetry(waitMs, requestSignal);
+        if (!completed) {
+          releaseOAuthSession();
+          return errorResponse(499, "Request aborted");
+        }
+        preselectedCredentials = credentials;
+        continue;
+      }
+
       // 8. Fallback to next account
       // A3 guard: if 401 and connection has extra keys, skip connection-level disable
       // (key-level failure already recorded in chatCore.ts via T07)

src/sse/services/auth.ts

@@ -61,6 +61,10 @@ import {
 } from "@omniroute/open-sse/services/quotaPreflight.ts";
 import { resolveResilienceSettings } from "@/lib/resilience/settings";
 import { resolveModelLockoutSettings } from "@/lib/resilience/modelLockoutSettings";
+import {
+  buildMixedAvailabilityError,
+  isTransportCooldownErrorCode,
+} from "../services/sameAccountTransportRetry";
 import { syncHealthFromDB, type KeyHealth } from "@omniroute/open-sse/services/apiKeyRotator.ts";
 import {
   classifyProviderError,
@@ -1049,7 +1053,9 @@ async function getProviderSearchPool(provider: string): Promise<string[]> {
       if (!nodeId) continue;
       if (
         nodePrefix &&
-        (nodePrefix === provider || nodePrefix === canonicalProvider || nodePrefix === canonicalAlias)
+        (nodePrefix === provider ||
+          nodePrefix === canonicalProvider ||
+          nodePrefix === canonicalAlias)
       ) {
         searchPool.add(nodeId);
       }
@@ -1686,6 +1692,32 @@ export async function getProviderCredentials(
     }
 
     if (policyEligibleConnections.length === 0 && availableConnections.length > 0) {
+      const transportUnavailable = connections.filter(
+        (connection) =>
+          connectionFilterStatus.get(connection.id) === "rateLimited" &&
+          isTransportCooldownErrorCode(connection.errorCode)
+      );
+      if (transportUnavailable.length > 0) {
+        const mixed = buildMixedAvailabilityError({
+          provider,
+          quotaFilteredCount: blockedByPolicy.length,
+          transportUnavailableCount: transportUnavailable.length,
+          transportStatus: Number(transportUnavailable[0]?.errorCode) || 503,
+        });
+        const retryAfter =
+          getEarliestFutureDate(
+            transportUnavailable.map((connection) => connection.rateLimitedUntil || null)
+          ) || new Date(Date.now() + 3000).toISOString();
+        invalidateManagedLease(options, "HEALTH_OR_COOLDOWN");
+        return {
+          allRateLimited: true,
+          retryAfter,
+          retryAfterHuman: formatRetryAfter(retryAfter),
+          lastError: mixed.lastError,
+          lastErrorCode: mixed.lastErrorCode,
+        };
+      }
+
       const earliestResetAt = getEarliestFutureDate(blockedByPolicy.map((entry) => entry.resetAt));
       const earliestResetMs = parseFutureDateMs(earliestResetAt);
 

src/sse/services/sameAccountTransportRetry.ts

@@ -0,0 +1,108 @@
+/**
+ * Same-account retry for retryable pre-output transport failures (#9708).
+ *
+ * A 503/507 (connection reset, retry-buffer overflow, early EOF before useful
+ * output) must not immediately cool the account and rotate. One jittered
+ * same-account retry absorbs brief proxy blips and keeps Codex prompt-cache
+ * affinity. A second failure then takes a short cooldown and may rotate.
+ */
+
+export const SAME_ACCOUNT_TRANSPORT_RETRY_MAX = 1;
+export const SAME_ACCOUNT_TRANSPORT_RETRY_MIN_DELAY_MS = 2000;
+export const SAME_ACCOUNT_TRANSPORT_RETRY_JITTER_MS = 1000;
+
+const RETRYABLE_TRANSPORT_STATUSES = new Set([502, 503, 504, 507]);
+
+const RETRYABLE_TRANSPORT_TEXT = [
+  /upstream connect error/i,
+  /disconnect\/reset before headers/i,
+  /remote connection failure/i,
+  /connection reset/i,
+  /exceeded request buffer limit/i,
+  /early eof/i,
+  /econnreset/i,
+  /socket hang up/i,
+  /und_err_socket/i,
+];
+
+const NON_RETRYABLE_ERROR_TYPES = new Set(["lease_error", "account_semaphore_capacity"]);
+
+export function isRetryableTransportStatus(status: unknown): boolean {
+  const numeric = Number(status);
+  return Number.isFinite(numeric) && RETRYABLE_TRANSPORT_STATUSES.has(numeric);
+}
+
+export function isRetryablePreOutputTransportError(
+  status: unknown,
+  errorText: string | null | undefined,
+  errorCode?: string | null,
+  errorType?: string | null
+): boolean {
+  if (errorType && NON_RETRYABLE_ERROR_TYPES.has(errorType)) return false;
+  if (errorCode && String(errorCode).startsWith("LEASE_")) return false;
+
+  const text = String(errorText || "");
+  const numericStatus = Number(status);
+  if (numericStatus === 429 || numericStatus === 401 || numericStatus === 400) return false;
+  if (/quota (threshold|exhausted)|credits exhausted/i.test(text)) return false;
+  if (/invalid_request|prompt is too long|context.?length|unsupported model/i.test(text)) {
+    return false;
+  }
+
+  const statusRetryable = isRetryableTransportStatus(status);
+  const textRetryable = RETRYABLE_TRANSPORT_TEXT.some((pattern) => pattern.test(text));
+  const codeRetryable =
+    errorCode === "STREAM_EARLY_EOF" ||
+    errorCode === "proxy_unreachable" ||
+    errorCode === "PROXY_UNREACHABLE";
+
+  return statusRetryable || textRetryable || codeRetryable;
+}
+
+export function sameAccountTransportRetryDelayMs(random: () => number = Math.random): number {
+  const draw = random();
+  const unit = Number.isFinite(draw) ? Math.min(Math.max(draw, 0), 1) : 0;
+  return Math.round(
+    SAME_ACCOUNT_TRANSPORT_RETRY_MIN_DELAY_MS + SAME_ACCOUNT_TRANSPORT_RETRY_JITTER_MS * unit
+  );
+}
+
+export function shouldRetrySameAccountTransport(options: {
+  status: unknown;
+  errorText?: string | null;
+  errorCode?: string | null;
+  errorType?: string | null;
+  attempt: number;
+  hasForcedConnection?: boolean;
+  hasEmittedOutput?: boolean;
+}): boolean {
+  if (options.hasForcedConnection) return false;
+  if (options.hasEmittedOutput) return false;
+  if (options.attempt >= SAME_ACCOUNT_TRANSPORT_RETRY_MAX) return false;
+  return isRetryablePreOutputTransportError(
+    options.status,
+    options.errorText,
+    options.errorCode,
+    options.errorType
+  );
+}
+
+export function isTransportCooldownErrorCode(errorCode: unknown): boolean {
+  return isRetryableTransportStatus(errorCode);
+}
+
+export function buildMixedAvailabilityError(options: {
+  provider: string;
+  quotaFilteredCount: number;
+  transportUnavailableCount: number;
+  transportStatus?: number | null;
+}): { status: number; lastError: string; lastErrorCode: number } {
+  const quota = Math.max(0, options.quotaFilteredCount);
+  const transport = Math.max(0, options.transportUnavailableCount);
+  const upstreamStatus = options.transportStatus || 503;
+  return {
+    status: 503,
+    lastErrorCode: 503,
+    lastError: `No ${options.provider} accounts currently available: ${quota} quota-filtered, ${transport} temporarily unavailable after upstream ${upstreamStatus}`,
+  };
+}