fix(codex): lift nested child cooldowns on parent clear and on snapshot headroom

changelog.d/fixes/12951-codex-scope-cooldown-clear.md

@@ -0,0 +1 @@
+- **fix(codex):** dashboard "clear cooldown" and the CAS recovery path now drop nested `codexScopeRateLimitedUntil` maps in the same write that nulls `rate_limited_until`, and fresh quota snapshots with headroom lift fallback-sourced scope cooldowns parked by quota preflight ([#12817](https://github.com/diegosouzapw/OmniRoute/issues/12817), [#12860](https://github.com/diegosouzapw/OmniRoute/issues/12860), [#12951](https://github.com/diegosouzapw/OmniRoute/pull/12951)) The dashboard reset-credit button (`consumeCodexResetCredit`) now exercises that same snapshot path after a successful redeem, so a filled quota bar is enough to unpark a leftover fallback Codex child without another manual clear.

open-sse/services/codexAccount/quota.ts

@@ -56,7 +56,9 @@ export async function persistCodexChildQuotaResponse(params: {
           rateLimitedUntil,
           rateLimitSource: exhaustedWindow ? ("quota_reset" as const) : ("fallback" as const),
         }
-      : {}),
+      : params.status === 200
+        ? { rateLimitedUntil: null }
+        : {}),
   });
   if (!providerSpecificData) return null;
 

src/lib/db/providers.ts

@@ -39,6 +39,7 @@ import { pickCodexConnectionForUser } from "@/lib/oauth/utils/codexConnectionSel
 import { isMicrosoftDesignerWebRetiredProviderId } from "@/shared/constants/designerWebRetirement";
 import { reconcileCodexUsageHistory } from "./providers/usageIdentityReconciliation";
 import { isRuntimeRetiredProviderId } from "@/shared/constants/providerRetirement";
+import { applyCodexChildCooldownClearOnUpdate } from "./providers/codexAccountState";
 
 /**
  * normalizeProviderSpecificData + the Codex fingerprint-seed invariant: Codex
@@ -951,11 +952,14 @@ export async function updateProviderConnection(id: string, data: JsonRecord) {
     ...data,
     updatedAt: new Date().toISOString(),
   };
-  merged.providerSpecificData = normalizeConnectionProviderSpecificData(
-    toStringOrNull(merged.provider),
-    merged.providerSpecificData,
-    merged,
-    existingCamel.providerSpecificData
+  merged.providerSpecificData = applyCodexChildCooldownClearOnUpdate(
+    data,
+    normalizeConnectionProviderSpecificData(
+      toStringOrNull(merged.provider),
+      merged.providerSpecificData,
+      merged,
+      existingCamel.providerSpecificData
+    )
   );
   // Mirror the sanitization the create path applies — keep the returned
   // object in lockstep with what we persist.
@@ -1024,65 +1028,13 @@ export async function updateProviderConnection(id: string, data: JsonRecord) {
 export {
   updateCodexScopedQuotaState,
   updateCodexScopeCooldown,
+  applyCodexChildCooldownClearOnUpdate,
+  stripCodexChildCooldownFields,
+  stripCodexChildCooldownsFromConnection,
+  hasCodexScopeCooldown,
+  liftCodexScopeCooldownOnHeadroom,
 } from "./providers/codexAccountState";
 
-/**
- * Atomic conditional clear of recoverable error state on a connection row.
- *
- * Returns true when the row was cleared, false when a concurrent writer
- * (markAccountUnavailable, connectionRecovery tick, test, etc.) changed the
- * row between the caller's snapshot read and this UPDATE — in which case the
- * clear is skipped to preserve the freshest error state. Closes the TOCTOU
- * window in the quota-recovery path.
- *
- * CAS token = (test_status, last_error_at, rate_limited_until).
- * markAccountUnavailable always bumps last_error_at on every cooldown/error
- * write, so an unchanged last_error_at reliably indicates no concurrent write.
- */
-export async function clearConnectionErrorIfUnchanged(
-  id: string,
-  expected: {
-    testStatus: string | null | undefined;
-    lastErrorAt: string | null | undefined;
-    rateLimitedUntil: string | null | undefined;
-  }
-): Promise<boolean> {
-  const db = getDbInstance() as unknown as DbLike;
-  const result = db
-    .prepare(
-      `
-    UPDATE provider_connections SET
-      test_status = 'active',
-      last_error = NULL,
-      last_error_at = NULL,
-      last_error_type = NULL,
-      last_error_source = NULL,
-      error_code = NULL,
-      rate_limited_until = NULL,
-      backoff_level = 0,
-      updated_at = ?
-    WHERE id = ?
-      AND IFNULL(test_status, '') = ?
-      AND IFNULL(last_error_at, '') = ?
-      AND IFNULL(rate_limited_until, '') = ?
-    `
-    )
-    .run(
-      new Date().toISOString(),
-      id,
-      expected.testStatus ?? "",
-      expected.lastErrorAt ?? "",
-      expected.rateLimitedUntil ?? ""
-    );
-  const applied = (result.changes ?? 0) > 0;
-  if (applied) {
-    backupDbFile("pre-write");
-    invalidateDbCache("connections");
-    bumpProxyConfigGeneration();
-  }
-  return applied;
-}
-
 /**
  * Lightweight stat bump — updates lastUsedAt and consecutiveUseCount without
  * SELECT, re-encrypt, cache invalidation, or file backup.
@@ -1183,4 +1135,5 @@ export {
   formatResetCountdown,
   isConnectionRateLimited,
   getRateLimitedConnections,
+  clearConnectionErrorIfUnchanged,
 } from "./providers/rateLimit";

src/lib/db/providers/codexAccountState.ts

@@ -18,10 +18,105 @@ interface DbLike {
 type CodexScopedQuotaPatch = {
   quotaState?: JsonRecord;
   exhaustedWindow?: "5h" | "7d" | null;
-  rateLimitedUntil?: string;
+  rateLimitedUntil?: string | null;
   rateLimitSource?: "fallback" | "quota_reset";
 };
 
+const CODEX_CHILD_COOLDOWN_KEYS = [
+  "codexScopeRateLimitedUntil",
+  "codexScopeRateLimitSource",
+] as const;
+
+function omitEmptyRecord(record: JsonRecord): JsonRecord | undefined {
+  return Object.keys(record).length > 0 ? record : undefined;
+}
+
+/** Drop nested Codex child cooldowns; keep quota snapshots and unrelated keys. */
+export function stripCodexChildCooldownFields(psd: JsonRecord): JsonRecord {
+  if (!connectionHasCodexChildCooldown(psd)) return psd;
+  const next = { ...psd };
+  for (const key of CODEX_CHILD_COOLDOWN_KEYS) delete next[key];
+  return next;
+}
+
+/** PUT/CAS payload that clears the parent column must also drop nested maps. */
+export function applyCodexChildCooldownClearOnUpdate<T extends JsonRecord | undefined>(
+  data: JsonRecord,
+  psd: T
+): T {
+  if (psd == null) return psd;
+  if (!Object.hasOwn(data, "rateLimitedUntil")) return psd;
+  if (data.rateLimitedUntil != null && data.rateLimitedUntil !== "") return psd;
+  return stripCodexChildCooldownFields(psd) as T;
+}
+
+function connectionHasCodexChildCooldown(psd: JsonRecord): boolean {
+  return "codexScopeRateLimitedUntil" in psd || "codexScopeRateLimitSource" in psd;
+}
+
+/**
+ * Persist a full-parent cooldown lift into the nested Codex child maps.
+ * When `alsoClearTopLevel` is set, the parent `rate_limited_until` column is
+ * nulled in the same transaction so a crash between the two writes cannot
+ * leave a nested child map behind a cleared parent column.
+ */
+export function stripCodexChildCooldownsFromConnection(
+  id: string,
+  options?: { alsoClearTopLevel?: boolean }
+): void {
+  if (typeof id !== "string" || id.length === 0) return;
+  const db = getDbInstance() as unknown as DbLike;
+  const alsoClearTopLevel = options?.alsoClearTopLevel === true;
+  const candidate = db
+    .prepare("SELECT provider FROM provider_connections WHERE id = ?")
+    .get(id);
+  const isCodex = toRecord(candidate).provider === "codex";
+  if (!alsoClearTopLevel && !isCodex) return;
+
+  backupDbFile("pre-write");
+  const wrote = db.transaction(() => {
+    const existing = db
+      .prepare(
+        "SELECT provider, provider_specific_data FROM provider_connections WHERE id = ?"
+      )
+      .get(id);
+    if (!existing) return false;
+    const existingRecord = toRecord(rowToCamel(existing));
+    const providerSpecificData = toRecord(existingRecord.providerSpecificData);
+    const stripNested =
+      existingRecord.provider === "codex" &&
+      connectionHasCodexChildCooldown(providerSpecificData);
+    if (!alsoClearTopLevel && !stripNested) return false;
+
+    const now = new Date().toISOString();
+    if (alsoClearTopLevel && stripNested) {
+      db.prepare(
+        `UPDATE provider_connections
+         SET rate_limited_until = NULL,
+             provider_specific_data = ?,
+             updated_at = ?
+         WHERE id = ?`
+      ).run(JSON.stringify(stripCodexChildCooldownFields(providerSpecificData)), now, id);
+      return true;
+    }
+    if (alsoClearTopLevel) {
+      db.prepare(
+        `UPDATE provider_connections
+         SET rate_limited_until = NULL, updated_at = ?
+         WHERE id = ?`
+      ).run(now, id);
+      return true;
+    }
+    db.prepare(
+      `UPDATE provider_connections
+       SET provider_specific_data = ?, updated_at = ?
+       WHERE id = ?`
+    ).run(JSON.stringify(stripCodexChildCooldownFields(providerSpecificData)), now, id);
+    return true;
+  })();
+  if (wrote) invalidateDbCache("connections");
+}
+
 /**
  * Atomically merge one virtual Codex child's quota evidence into its persisted parent.
  * The transaction reads the latest row so sibling child state cannot be lost.
@@ -70,28 +165,35 @@ export async function updateCodexScopedQuotaState(
       }
     }
 
-    if (patch.rateLimitedUntil) {
-      const scopeCooldowns = toRecord(providerSpecificData.codexScopeRateLimitedUntil);
-      const sourceByScope = toRecord(providerSpecificData.codexScopeRateLimitSource);
-      const existingCooldownMs =
-        typeof scopeCooldowns[scope] === "string"
-          ? new Date(scopeCooldowns[scope] as string).getTime()
-          : NaN;
-      const existingIsAuthoritative =
-        sourceByScope[scope] === "quota_reset" &&
-        patch.rateLimitSource !== "quota_reset" &&
-        Number.isFinite(existingCooldownMs) &&
-        existingCooldownMs > Date.now();
-      nextProviderSpecificData.codexScopeRateLimitedUntil = {
-        ...scopeCooldowns,
-        [scope]: existingIsAuthoritative ? scopeCooldowns[scope] : patch.rateLimitedUntil,
-      };
-      nextProviderSpecificData.codexScopeRateLimitSource = {
-        ...sourceByScope,
-        [scope]: existingIsAuthoritative
+    if (patch.rateLimitedUntil !== undefined) {
+      const scopeCooldowns = { ...toRecord(providerSpecificData.codexScopeRateLimitedUntil) };
+      const sourceByScope = { ...toRecord(providerSpecificData.codexScopeRateLimitSource) };
+      if (patch.rateLimitedUntil) {
+        const existingCooldownMs =
+          typeof scopeCooldowns[scope] === "string"
+            ? new Date(scopeCooldowns[scope] as string).getTime()
+            : NaN;
+        const existingIsAuthoritative =
+          sourceByScope[scope] === "quota_reset" &&
+          patch.rateLimitSource !== "quota_reset" &&
+          Number.isFinite(existingCooldownMs) &&
+          existingCooldownMs > Date.now();
+        scopeCooldowns[scope] = existingIsAuthoritative
+          ? scopeCooldowns[scope]
+          : patch.rateLimitedUntil;
+        sourceByScope[scope] = existingIsAuthoritative
           ? sourceByScope[scope]
-          : (patch.rateLimitSource ?? "fallback"),
-      };
+          : (patch.rateLimitSource ?? "fallback");
+      } else {
+        delete scopeCooldowns[scope];
+        delete sourceByScope[scope];
+      }
+      const nextCooldowns = omitEmptyRecord(scopeCooldowns);
+      const nextSources = omitEmptyRecord(sourceByScope);
+      if (nextCooldowns) nextProviderSpecificData.codexScopeRateLimitedUntil = nextCooldowns;
+      else delete nextProviderSpecificData.codexScopeRateLimitedUntil;
+      if (nextSources) nextProviderSpecificData.codexScopeRateLimitSource = nextSources;
+      else delete nextProviderSpecificData.codexScopeRateLimitSource;
     }
 
     db.prepare(
@@ -106,6 +208,107 @@ export async function updateCodexScopedQuotaState(
   return persisted;
 }
 
+/** Grace window absorbing clock skew against the upstream quota server. */
+const QUOTA_RESET_CLOCK_SKEW_GRACE_MS = 30_000;
+
+/** Cheap probe: does this connection+scope currently carry a child cooldown? */
+export function hasCodexScopeCooldown(id: string, scope: "codex" | "spark"): boolean {
+  if (typeof id !== "string" || id.length === 0) return false;
+  const db = getDbInstance() as unknown as DbLike;
+  const row = db
+    .prepare("SELECT provider, provider_specific_data FROM provider_connections WHERE id = ?")
+    .get(id);
+  if (!row) return false;
+  const record = toRecord(rowToCamel(row));
+  if (record.provider !== "codex") return false;
+  const psd = toRecord(record.providerSpecificData);
+  return Boolean(toRecord(psd.codexScopeRateLimitedUntil)[scope]);
+}
+
+/**
+ * #12860: When fresh quota snapshot data demonstrates headroom on a scope,
+ * lift any fallback-sourced cooldown (e.g. parked by quota preflight).
+ * Cooldowns sourced from upstream 429 quota_reset retain their authority
+ * until their reset timestamp has elapsed.
+ */
+export function liftCodexScopeCooldownOnHeadroom(
+  id: string,
+  scope: "codex" | "spark"
+): boolean {
+  if (typeof id !== "string" || id.length === 0) return false;
+  const db = getDbInstance() as unknown as DbLike;
+
+  // Cheap eligibility probe so the backup only runs when a write is plausible.
+  // The transaction below re-reads under lock and remains the authority.
+  if (!hasCodexScopeCooldown(id, scope)) return false;
+
+  backupDbFile("pre-write");
+  const wrote = db.transaction(() => {
+    const existing = db
+      .prepare(
+        "SELECT provider, provider_specific_data FROM provider_connections WHERE id = ?"
+      )
+      .get(id);
+    if (!existing) return false;
+    const existingRecord = toRecord(rowToCamel(existing));
+    if (existingRecord.provider !== "codex") return false;
+    const currentPsd = toRecord(existingRecord.providerSpecificData);
+    const currentCooldowns = { ...toRecord(currentPsd.codexScopeRateLimitedUntil) };
+    if (!currentCooldowns[scope]) return false;
+
+    const currentSources = { ...toRecord(currentPsd.codexScopeRateLimitSource) };
+    const curSource = currentSources[scope];
+    const curUntilMs =
+      typeof currentCooldowns[scope] === "string"
+        ? new Date(currentCooldowns[scope] as string).getTime()
+        : NaN;
+    // An upstream-authoritative `quota_reset` deadline outranks a local snapshot:
+    // the grace window absorbs clock skew against the quota server, so a fast
+    // local clock cannot lift a cooldown upstream still considers active.
+    if (
+      curSource === "quota_reset" &&
+      Number.isFinite(curUntilMs) &&
+      curUntilMs > Date.now() - QUOTA_RESET_CLOCK_SKEW_GRACE_MS
+    ) {
+      return false;
+    }
+
+    delete currentCooldowns[scope];
+    delete currentSources[scope];
+
+    const nextPsd: JsonRecord = { ...currentPsd };
+    const nextCooldowns = omitEmptyRecord(currentCooldowns);
+    const nextSources = omitEmptyRecord(currentSources);
+    if (nextCooldowns) nextPsd.codexScopeRateLimitedUntil = nextCooldowns;
+    else delete nextPsd.codexScopeRateLimitedUntil;
+    if (nextSources) nextPsd.codexScopeRateLimitSource = nextSources;
+    else delete nextPsd.codexScopeRateLimitSource;
+
+    const exhaustedByScope = { ...toRecord(currentPsd.codexExhaustedWindowByScope) };
+    if (exhaustedByScope[scope]) {
+      delete exhaustedByScope[scope];
+      const nextExhausted = omitEmptyRecord(exhaustedByScope);
+      if (nextExhausted) {
+        nextPsd.codexExhaustedWindowByScope = nextExhausted;
+      } else {
+        delete nextPsd.codexExhaustedWindowByScope;
+        delete nextPsd.codexExhaustedWindow;
+      }
+    }
+
+    const now = new Date().toISOString();
+    db.prepare(
+      `UPDATE provider_connections
+       SET provider_specific_data = ?, updated_at = ?
+       WHERE id = ?`
+    ).run(JSON.stringify(nextPsd), now, id);
+    return true;
+  })();
+
+  if (wrote) invalidateDbCache("connections");
+  return wrote;
+}
+
 /** Persist one child cooldown through the shared scoped quota-state transaction. */
 export async function updateCodexScopeCooldown(
   id: string,