fix(executors): rotate to the next account on network throws, not just 429

changelog.d/fixes/10393-opencode-rotate-network-throw.md

@@ -0,0 +1 @@
+- **fix(executors):** OpencodeExecutor and MimocodeExecutor now rotate to the next account on network exceptions (timeout, connection refused/reset) when the failed account has a dedicated proxy, not only on 429 — a throw on one account no longer fails the whole request when other accounts remain. Accounts sharing the default egress (no proxy) fail fast instead of retrying the same outage against every account. The shared rotation mechanics (`pickAccount`/`markCooldown`/`markSuccess`) are now extracted into `accountRotation.ts`, fixing an identical unconditional-cooldown gap that pre-dated this PR in MimocodeExecutor ([#10393](https://github.com/diegosouzapw/OmniRoute/pull/10393))

open-sse/executors/accountRotation.ts

@@ -0,0 +1,109 @@
+/**
+ * Shared multi-account rotation mechanics for noauth executors that round-robin
+ * across several "accounts" (fingerprints), each with an optional dedicated
+ * proxy — currently `OpencodeExecutor` and `MimocodeExecutor`.
+ *
+ * Extracted after both executors independently implemented the same
+ * pickAccount/markCooldown/markSuccess skeleton with the same exponential
+ * backoff, and independently needed the same fix for the same latent bug (a
+ * network exception was treated as account-scoped rotation fodder even for
+ * accounts sharing the default egress — see `isNetworkErrorRotatable`).
+ */
+
+// Reuses the repo's established "transient, not clearly attributable" failure
+// cooldown (already used by accountFallback.ts for network-error dedup, see
+// its "one transient blip opens the whole-provider breaker" comment) instead
+// of inventing a separate constant — same magnitude the codebase already
+// applies whether the failure is a 429 or a network-level throw.
+import { TRANSIENT_COOLDOWN_MS, COOLDOWN_MS } from "../config/errorConfig.ts";
+
+/** Per-account proxy configuration, persisted by NoAuthAccountCard under
+ * `providerSpecificData.accountProxies` (keyed by the account id, which the UI
+ * stores in `providerSpecificData.fingerprints`). */
+export interface AccountProxyConfig {
+  fingerprint: string;
+  proxy: {
+    type: string;
+    host: string;
+    port: number;
+    username?: string;
+    password?: string;
+    relayAuth?: string;
+  } | null;
+}
+
+/** The subset of per-account state the rotation mechanics need. Executors may
+ * carry additional fields (e.g. mimocode's `jwt`/`expiresAt`) — this is the
+ * minimum shape `pickAccount`/`markCooldown`/`markSuccess` operate on. */
+export interface RotatableAccount {
+  fingerprint: string;
+  cooldownUntil: number;
+  consecutiveFails: number;
+  proxy: AccountProxyConfig["proxy"];
+}
+
+const COOLDOWN_BASE_MS = TRANSIENT_COOLDOWN_MS;
+const COOLDOWN_MAX_MS = COOLDOWN_MS.transientMax;
+
+export function isAccountReady(account: RotatableAccount): boolean {
+  return account.cooldownUntil <= Date.now();
+}
+
+/** Round-robin pick, skipping accounts not `isReady`; falls back to the next
+ * index (even if not ready) so a caller always gets an account rather than
+ * hanging when every account is unavailable. Mutates `state.nextAccountIdx`.
+ *
+ * `isReady` defaults to the plain cooldown check (`isAccountReady`); pass a
+ * custom predicate when readiness depends on more than cooldown (e.g.
+ * mimocode's JWT-freshness-aware variant). */
+export function pickAccount<T extends RotatableAccount>(
+  accounts: T[],
+  state: { nextAccountIdx: number },
+  isReady: (account: T) => boolean = isAccountReady
+): T {
+  for (let i = 0; i < accounts.length; i++) {
+    const idx = (state.nextAccountIdx + i) % accounts.length;
+    const acct = accounts[idx];
+    if (isReady(acct)) {
+      state.nextAccountIdx = (idx + 1) % accounts.length;
+      return acct;
+    }
+  }
+  const fallbackIdx = state.nextAccountIdx % accounts.length;
+  state.nextAccountIdx = (state.nextAccountIdx + 1) % accounts.length;
+  return accounts[fallbackIdx];
+}
+
+export function markCooldown(account: RotatableAccount): void {
+  account.consecutiveFails++;
+  const backoff = Math.min(
+    COOLDOWN_BASE_MS * Math.pow(2, account.consecutiveFails - 1),
+    COOLDOWN_MAX_MS
+  );
+  account.cooldownUntil = Date.now() + backoff + Math.random() * 1000;
+}
+
+export function markSuccess(account: RotatableAccount): void {
+  account.consecutiveFails = 0;
+}
+
+/** Mask an account id for logs (UI calls it a fingerprint). */
+export function maskAccountId(fingerprint: string): string {
+  if (!fingerprint) return "direct";
+  return `${fingerprint.slice(0, 8)}…`;
+}
+
+/**
+ * Whether a network exception (timeout, connection refused/reset) on this
+ * account should trigger rotation to the next account, vs propagating.
+ *
+ * Only true when the account has its own egress (a configured proxy) — that's
+ * the case a dead/unreachable proxy genuinely justifies rotating away from.
+ * Accounts sharing the default egress (no proxy) can all fail at once on a
+ * real network outage: rotating there would just retry the same failure
+ * against every account while poisoning each one's cooldown for a cause that
+ * isn't theirs.
+ */
+export function isNetworkErrorRotatable(account: RotatableAccount): boolean {
+  return account.proxy !== null;
+}

open-sse/executors/mimocode.ts

@@ -27,13 +27,21 @@ import { createProxyDispatcher } from "../utils/proxyDispatcher.ts";
 import { RATE_LIMIT_TEXT_PATTERNS } from "../services/accountFallback.ts";
 import { buildErrorBody, sanitizeErrorMessage } from "../utils/error.ts";
 import { fetch as undiciFetch, type Dispatcher } from "undici";
+import {
+  type AccountProxyConfig as SharedAccountProxyConfig,
+  type RotatableAccount,
+  pickAccount as pickRotatableAccount,
+  markCooldown as markAccountCooldown,
+  markSuccess as markAccountSuccess,
+  maskAccountId,
+  isNetworkErrorRotatable,
+} from "./accountRotation.ts";
+import { isNetworkRotationSharedEgressGuardEnabled } from "@/shared/utils/featureFlags";
 
 const BOOTSTRAP_PATH = "/api/free-ai/bootstrap";
 const CHAT_PATH = "/api/free-ai/openai/chat";
 const JWT_REFRESH_BUFFER_MS = 5 * 60 * 1000;
 const BOOTSTRAP_TIMEOUT_MS = 15_000;
-const COOLDOWN_BASE_MS = 5_000;
-const COOLDOWN_MAX_MS = 60_000;
 
 const MIMO_SOURCE = "mimocode-cli-free";
 
@@ -82,24 +90,12 @@ const USER_AGENTS = [
 // ── Account State ──────────────────────────────────────────────────────────
 
 /** Per-account proxy configuration, passed through providerSpecificData.accountProxies. */
-export interface AccountProxyConfig {
-  fingerprint: string;
-  proxy: {
-    type: string;
-    host: string;
-    port: number;
-    username?: string;
-    password?: string;
-    relayAuth?: string;
-  } | null;
-}
+export type AccountProxyConfig = SharedAccountProxyConfig;
 
-interface AccountState {
+interface AccountState extends RotatableAccount {
   fingerprint: string;
   jwt: string;
   expiresAt: number;
-  cooldownUntil: number;
-  consecutiveFails: number;
   /**
    * #3837/#5521: the account's resolved proxy, or `null` when none is configured.
    * Always present (never `undefined`) so callers can read `acct.proxy` directly —
@@ -223,7 +219,10 @@ function rewriteModelName(model: string): string {
 
 export class MimocodeExecutor extends BaseExecutor {
   private accounts: AccountState[] = [];
-  private nextAccountIdx = 0;
+  // Not `private`: passed as the mutable rotation cursor to the shared
+  // pickAccount() helper, which needs a plain `{ nextAccountIdx }` shape —
+  // TS's private-member nominal check rejects `this` there otherwise.
+  nextAccountIdx = 0;
   private baseUrl: string;
   private proxyUrlMap = new Map<string, string>();
   private static encoder = new TextEncoder();
@@ -342,30 +341,15 @@ export class MimocodeExecutor extends BaseExecutor {
   }
 
   private pickAccount(): AccountState {
-    for (let i = 0; i < this.accounts.length; i++) {
-      const idx = (this.nextAccountIdx + i) % this.accounts.length;
-      const acct = this.accounts[idx];
-      if (isAccountReady(acct)) {
-        this.nextAccountIdx = (idx + 1) % this.accounts.length;
-        return acct;
-      }
-    }
-    const fallbackIdx = this.nextAccountIdx % this.accounts.length;
-    this.nextAccountIdx = (this.nextAccountIdx + 1) % this.accounts.length;
-    return this.accounts[fallbackIdx];
+    return pickRotatableAccount(this.accounts, this, isAccountReady);
   }
 
   private markCooldown(account: AccountState): void {
-    account.consecutiveFails++;
-    const backoff = Math.min(
-      COOLDOWN_BASE_MS * Math.pow(2, account.consecutiveFails - 1),
-      COOLDOWN_MAX_MS
-    );
-    account.cooldownUntil = Date.now() + backoff + Math.random() * 1000;
+    markAccountCooldown(account);
   }
 
   private markSuccess(account: AccountState): void {
-    account.consecutiveFails = 0;
+    markAccountSuccess(account);
   }
 
   /**
@@ -592,9 +576,25 @@ export class MimocodeExecutor extends BaseExecutor {
 
     this.syncAccountsFromCredentials(input.credentials);
 
+    const sharedEgressGuardEnabled = isNetworkRotationSharedEgressGuardEnabled();
+    // Set once a proxy-less account's network throw reveals the shared egress
+    // is down — subsequent proxy-less accounts this request are skipped
+    // without a network call, but proxied accounts (independent egress) are
+    // still tried normally. See NETWORK_ROTATION_SHARED_EGRESS_GUARD.
+    let sharedEgressDown = false;
+
     // Try each account, skip cooldown ones
     for (let attempt = 0; attempt < this.accounts.length; attempt++) {
       const account = this.pickAccount();
+
+      if (sharedEgressGuardEnabled && sharedEgressDown && !account.proxy) {
+        log?.warn?.(
+          "MIMOCODE",
+          `skipping account ${maskAccountId(account.fingerprint)} (no dedicated proxy, shared egress already down this request)`
+        );
+        continue;
+      }
+
       try {
         const headers = this.buildHeaders(input.credentials, stream);
         const resp = await this.fetchWithAuthRetry(url, headers, reqBody, signal, account, log);
@@ -623,16 +623,60 @@ export class MimocodeExecutor extends BaseExecutor {
           transformedBody: reqBody,
         };
       } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        const masked = maskAccountId(account.fingerprint);
+
+        // Mirrors OpencodeExecutor's rotation guard: a network exception is only account-scoped
+        // when this account has its OWN egress (a configured proxy). Without
+        // one, accounts share the default egress — the failure isn't
+        // attributable to this account, and trying the next one would just
+        // retry the same outage while poisoning its cooldown for a cause
+        // that isn't theirs. Fail fast instead of exhausting every account.
+        if (!isNetworkErrorRotatable(account)) {
+          if (sharedEgressGuardEnabled) {
+            this.markCooldown(account);
+            sharedEgressDown = true;
+            log?.warn?.(
+              "MIMOCODE",
+              `network error on account ${masked} (no dedicated proxy, shared egress), cooldown applied — trying next available account… (${msg})`
+            );
+            continue;
+          }
+          log?.warn?.(
+            "MIMOCODE",
+            `network error on account ${masked} (no dedicated proxy, shared egress) — not rotating (${msg})`
+          );
+          return {
+            response: new Response(
+              encoder.encode(
+                JSON.stringify(
+                  buildErrorBody(502, msg, undefined, {
+                    type: "upstream_error",
+                    code: "EXECUTOR_ERROR",
+                  })
+                )
+              ),
+              { status: 502, headers: { "Content-Type": "application/json" } }
+            ),
+            url,
+            headers: this.buildHeaders(input.credentials, stream),
+            transformedBody: body,
+          };
+        }
+
         this.markCooldown(account);
+        log?.warn?.("MIMOCODE", `network error on account ${masked}, rotating to next… (${msg})`);
         if (attempt === this.accounts.length - 1) {
-          const msg = err instanceof Error ? err.message : String(err);
           log?.error?.("MIMOCODE", `Executor error: ${msg}`);
           return {
             response: new Response(
               encoder.encode(
-                JSON.stringify({
-                  error: { message: msg, type: "upstream_error", code: "EXECUTOR_ERROR" },
-                })
+                JSON.stringify(
+                  buildErrorBody(502, msg, undefined, {
+                    type: "upstream_error",
+                    code: "EXECUTOR_ERROR",
+                  })
+                )
               ),
               { status: 502, headers: { "Content-Type": "application/json" } }
             ),