fix(autoCombo,sse): drop vendor-retired ids from the auto-combo pool and skip leftover arena_elo (#11625)

changelog.d/fixes/11625-retired-pool.md

@@ -0,0 +1 @@
+- **fix(autoCombo,sse):** vendor-retired catalog ids are dropped from the auto-combo candidate pool and no longer win on leftover `arena_elo` / `user_override` rows; `getModelLifecycleDecision` consults `model-lifecycle.json` (prefix-stripped) so aggregator traffic is not `untracked` for ids the snapshot already knows ([#11625](https://github.com/diegosouzapw/OmniRoute/issues/11625))

open-sse/services/autoCombo/taskFitness.ts

@@ -10,6 +10,9 @@
  * 2b. Layers 1-2 retried against the base model this id inherits quality scores
  *     from, when `resolveScoresAs` resolves one (#11489). Reported as
  *     `<source>:inherited`.
+ * 2c. If the request id or its scoresAs base is vendor-retired (#11625), layers
+ *     1–3 are skipped so a leftover arena row cannot short-circuit the layer-3
+ *     veto, and a dead *codex id cannot keep the coding wildcard boost.
  * 3. Models.dev tier — derived from `model_capabilities` table capability data,
  *     with a vendor-lifecycle veto (#11508: a retired id never earns a tier
  *     score) and the same base-model inheritance as layers 1–2
@@ -43,6 +46,7 @@ import {
 } from "../../../src/lib/db/modelIntelligence.ts";
 import { readFileSync } from "node:fs";
 import { resolveScoresAs } from "./scoresAs.ts";
+import { isVendorRetiredId } from "../modelLifecycle.ts";
 
 // #11508 — vendor lifecycle snapshot (#11507). An id the vendor has retired
 // must never earn a capability-derived tier score: models.dev keeps listing
@@ -463,48 +467,60 @@ export function getTaskFitness(model: string, taskType: string): number {
   return getTaskFitnessWithSource(model, taskType).score;
 }
 
+function isFitnessRetired(modelId: string): boolean {
+  if (isVendorRetiredId(modelId)) return true;
+  const { base, via } = resolveScoresAs(modelId);
+  return via !== null && isVendorRetiredId(base);
+}
+
 export function getTaskFitnessWithSource(
   model: string,
   taskType: string
 ): { score: number; source: string } {
   const normalizedModel = model.toLowerCase();
   const normalizedTask = taskType.toLowerCase();
+  const fitnessRetired = isFitnessRetired(normalizedModel);
 
-  const userOverride = queryModelIntelligence(normalizedModel, normalizedTask, "user_override");
-  if (userOverride !== null) {
-    return { score: userOverride, source: "user_override" };
-  }
+  if (!fitnessRetired) {
+    const userOverride = queryModelIntelligence(normalizedModel, normalizedTask, "user_override");
+    if (userOverride !== null) {
+      return { score: userOverride, source: "user_override" };
+    }
 
-  const arenaElo = queryModelIntelligence(normalizedModel, normalizedTask, "arena_elo");
-  if (arenaElo !== null) {
-    return { score: arenaElo, source: "arena_elo" };
-  }
+    const arenaElo = queryModelIntelligence(normalizedModel, normalizedTask, "arena_elo");
+    if (arenaElo !== null) {
+      return { score: arenaElo, source: "arena_elo" };
+    }
 
-  // Layers 1-2, retried against the base model this id inherits quality from
-  // (#11489). Every DB-backed source publishes scores for BASE models only, so
-  // a variant id — an effort suffix (`gpt-5.6-sol-xhigh`), a vendor alias
-  // (`gpt-5.6`), a `-free` tier marker (`mimo-v2.5-free`, #4517) — misses both
-  // literal lookups and used to fall all the way to the wildcard 0.5, losing
-  // every comparison against a base model that happens to be benchmarked.
-  // The score is inherited VERBATIM: the 12-factor scoring already prices cost
-  // and latency per variant, so there is no basis for inventing an effort
-  // delta. `:inherited` keeps the indirection visible to callers.
-  const inherited = lookupInheritedFitness(normalizedModel, normalizedTask);
-  if (inherited !== null) {
-    return inherited;
-  }
+    // Layers 1-2, retried against the base model this id inherits quality from
+    // (#11489). Every DB-backed source publishes scores for BASE models only, so
+    // a variant id — an effort suffix (`gpt-5.6-sol-xhigh`), a vendor alias
+    // (`gpt-5.6`), a `-free` tier marker (`mimo-v2.5-free`, #4517) — misses both
+    // literal lookups and used to fall all the way to the wildcard 0.5, losing
+    // every comparison against a base model that happens to be benchmarked.
+    // The score is inherited VERBATIM: the 12-factor scoring already prices cost
+    // and latency per variant, so there is no basis for inventing an effort
+    // delta. `:inherited` keeps the indirection visible to callers.
+    const inherited = lookupInheritedFitness(normalizedModel, normalizedTask);
+    if (inherited !== null) {
+      return inherited;
+    }
 
-  const tierHit = getModelsDevTierFitnessWithSource(normalizedModel, normalizedTask);
-  if (tierHit !== null) {
-    return tierHit;
-  }
+    const tierScore = getModelsDevTierFitness(normalizedModel, normalizedTask);
+    if (tierScore !== null) {
+      return { score: tierScore, source: "models_dev_tier" };
+    }
+
+    const staticScore = lookupStaticFitnessTable(normalizedModel, normalizedTask);
+    if (staticScore !== null) {
+      return { score: staticScore, source: "fitness_table" };
+    }
 
-  const staticScore = lookupStaticFitnessTable(normalizedModel, normalizedTask);
-  if (staticScore !== null) {
-    return { score: staticScore, source: "fitness_table" };
+    return { score: lookupWildcardBoosts(normalizedModel, normalizedTask), source: "wildcard_boost" };
   }
 
-  return { score: lookupWildcardBoosts(normalizedModel, normalizedTask), source: "wildcard_boost" };
+  // Retired: 0.5 is "no evidence", never a quality claim and never a *codex boost.
+  return { score: 0.5, source: "wildcard_boost" };
 }
 
 /**
@@ -524,6 +540,7 @@ function lookupInheritedFitness(
   const { base, via } = resolveScoresAs(normalizedModel);
   if (via === null || base === normalizedModel) return null;
   const normalizedBase = base.toLowerCase();
+  if (isVendorRetiredId(normalizedBase)) return null;
 
   for (const source of ["user_override", "arena_elo"] as const) {
     const score = queryModelIntelligence(normalizedBase, normalizedTask, source);

open-sse/services/combo.ts

@@ -70,6 +70,7 @@ import { resolveProviderId } from "../../src/shared/constants/providers.ts";
 import * as semaphore from "./rateLimitSemaphore.ts";
 import { getCircuitBreaker } from "../../src/shared/utils/circuitBreaker";
 import { parseModel } from "./model.ts";
+import { rejectRetiredAutoComboCandidates } from "./modelLifecycle.ts";
 import { createComboContext } from "./combo/context.ts";
 import { phaseComboSetup } from "./combo/comboSetup.ts";
 import { checkCredentialGate, logCredentialSkip } from "./credentialGate.ts";
@@ -627,11 +628,14 @@ export async function buildAutoCandidates(
     })
   );
 
-  // Filter out candidates whose model is hidden by the user in the dashboard
-  return candidates.filter((c) => {
-    const hiddenModels = hiddenModelsMap.get(c.provider);
-    return !hiddenModels?.has(c.model);
-  });
+  // Filter out candidates whose model is hidden by the user in the dashboard,
+  // then drop vendor-retired ids so auto-combo cannot pick them (#11625).
+  return rejectRetiredAutoComboCandidates(
+    candidates.filter((c) => {
+      const hiddenModels = hiddenModelsMap.get(c.provider);
+      return !hiddenModels?.has(c.model);
+    })
+  );
 }
 
 // Context-cache pin health gate — moved to combo/dispatchPrelude.ts alongside the

open-sse/services/modelLifecycle.ts

@@ -4,7 +4,14 @@
  * Replacement model IDs are migration guidance only. This module never rewrites a
  * request: shutdown models are rejected, deprecated models remain callable until
  * their shutdown date, and untracked models pass through unchanged.
+ *
+ * Dated OpenAI rows in MODEL_LIFECYCLE_RECORDS stay provider-scoped (a warn-before-
+ * shutdown date on `openai` must not leak onto an aggregator that still serves the
+ * id). Snapshot `status: "retired"` ids from config/quality/model-lifecycle.json
+ * are id-scoped and prefix-stripped (#11625): `openai/gpt-5.2-codex` on openrouter
+ * is the same retired vendor id as `gpt-5.2-codex`.
  */
+import { readFileSync } from "node:fs";
 
 export const OPENAI_MODEL_DEPRECATIONS_URL = "https://developers.openai.com/api/docs/deprecations";
 
@@ -128,6 +135,67 @@ function shutdownTimestamp(shutdownAt: string): number {
   return Date.parse(`${shutdownAt}T00:00:00.000Z`);
 }
 
+const SNAPSHOT_URL = new URL("../../config/quality/model-lifecycle.json", import.meta.url);
+const SNAPSHOT_SOURCE = "config/quality/model-lifecycle.json";
+
+type VendorRetiredEntry = {
+  vendor?: string;
+  status?: string;
+  retiredOn?: string | null;
+  replacement?: string | null;
+};
+
+let _retiredIds: Set<string> | null = null;
+let _retiredEntries: Map<string, VendorRetiredEntry> | null = null;
+
+function loadVendorRetiredSnapshot(): {
+  ids: Set<string>;
+  entries: Map<string, VendorRetiredEntry>;
+} {
+  if (_retiredIds && _retiredEntries) return { ids: _retiredIds, entries: _retiredEntries };
+  const ids = new Set<string>();
+  const entries = new Map<string, VendorRetiredEntry>();
+  try {
+    const parsed = JSON.parse(readFileSync(SNAPSHOT_URL, "utf8")) as {
+      retired?: Record<string, VendorRetiredEntry>;
+    };
+    for (const [id, entry] of Object.entries(parsed.retired ?? {})) {
+      if (entry?.status !== "retired") continue;
+      const key = id.toLowerCase();
+      ids.add(key);
+      entries.set(key, entry);
+    }
+  } catch {
+    // Snapshot missing → no id-scoped veto. Dated OpenAI rows still apply.
+  }
+  _retiredIds = ids;
+  _retiredEntries = entries;
+  return { ids, entries };
+}
+
+/** True when `modelId` or its last `vendor/` path segment is `status: "retired"` in the snapshot. */
+export function isVendorRetiredId(modelId: string | null | undefined): boolean {
+  if (typeof modelId !== "string" || modelId.length === 0) return false;
+  const lower = modelId.toLowerCase();
+  const { ids } = loadVendorRetiredSnapshot();
+  if (ids.has(lower)) return true;
+  const slash = lower.lastIndexOf("/");
+  return slash !== -1 && ids.has(lower.slice(slash + 1));
+}
+
+function lookupVendorRetiredEntry(modelId: string): VendorRetiredEntry | null {
+  const lower = modelId.toLowerCase();
+  const { entries } = loadVendorRetiredSnapshot();
+  return entries.get(lower) ?? entries.get(lower.slice(lower.lastIndexOf("/") + 1)) ?? null;
+}
+
+/** Drop auto-combo candidates whose model id the vendor has retired (#11625). */
+export function rejectRetiredAutoComboCandidates<T extends { model: string }>(
+  candidates: readonly T[]
+): T[] {
+  return candidates.filter((candidate) => !isVendorRetiredId(candidate.model));
+}
+
 export function getModelLifecycleDecision(
   provider: string | null | undefined,
   model: string | null | undefined,
@@ -138,6 +206,24 @@ export function getModelLifecycleDecision(
   const record = RECORDS_BY_KEY.get(lifecycleKey(normalizedProvider, normalizedModel));
 
   if (!record) {
+    if (isVendorRetiredId(normalizedModel)) {
+      const entry = lookupVendorRetiredEntry(normalizedModel);
+      const replacementId =
+        typeof entry?.replacement === "string" && entry.replacement.length > 0
+          ? entry.replacement
+          : null;
+      return {
+        provider: normalizedProvider,
+        model: normalizedModel,
+        status: "shutdown",
+        action: "reject",
+        shutdownAt: typeof entry?.retiredOn === "string" ? entry.retiredOn : null,
+        replacement: replacementId
+          ? { provider: entry?.vendor ?? "", model: replacementId }
+          : null,
+        source: SNAPSHOT_SOURCE,
+      };
+    }
     return {
       provider: normalizedProvider,
       model: normalizedModel,
@@ -170,7 +256,8 @@ export function formatModelLifecycleMessage(decision: ModelLifecycleDecision): s
     ? ` Use "${decision.replacement.provider}/${decision.replacement.model}" instead.`
     : "";
   if (decision.status === "shutdown") {
-    return `Model "${modelRef}" was shut down on ${decision.shutdownAt} and cannot be routed automatically.${replacement}`;
+    const when = decision.shutdownAt ? ` was shut down on ${decision.shutdownAt}` : " has been retired by its vendor";
+    return `Model "${modelRef}"${when} and cannot be routed automatically.${replacement}`;
   }
   return `Model "${modelRef}" is deprecated and is scheduled to shut down on ${decision.shutdownAt}.${replacement}`;
 }

tests/unit/model-lifecycle.test.ts

@@ -32,11 +32,23 @@ test("upcoming shutdowns warn before the shutdown date", () => {
   assert.equal(decision.shutdownAt, "2026-08-10");
 });
 
-test("lifecycle records are provider-scoped", () => {
+test("dated OpenAI deprecations stay provider-scoped", () => {
+  // gpt-3.5-turbo-0125 shuts down 2026-10-23; CURRENT_DATE is 2026-07-26.
+  // Snapshot marks it retiring (not retired), so an aggregator still allows it.
+  const openai = getModelLifecycleDecision("openai", "gpt-3.5-turbo-0125", CURRENT_DATE);
+  const aggregator = getModelLifecycleDecision("opencode-zen", "gpt-3.5-turbo-0125", CURRENT_DATE);
+
+  assert.equal(openai.status, "deprecated");
+  assert.equal(openai.action, "warn");
+  assert.equal(aggregator.status, "untracked");
+  assert.equal(aggregator.action, "allow");
+});
+
+test("snapshot-retired ids are rejected on every provider (#11625)", () => {
   const decision = getModelLifecycleDecision("opencode-zen", "gpt-5.2-codex", CURRENT_DATE);
 
-  assert.equal(decision.status, "untracked");
-  assert.equal(decision.action, "allow");
+  assert.equal(decision.status, "shutdown");
+  assert.equal(decision.action, "reject");
 });
 
 test("catalog filtering hides deprecated and shutdown models by default", () => {
@@ -52,7 +64,7 @@ test("catalog filtering hides deprecated and shutdown models by default", () =>
   );
   assert.deepEqual(
     filterSelectableModels("opencode-zen", models, { asOf: CURRENT_DATE }).map((model) => model.id),
-    models.map((model) => model.id)
+    ["gpt-5.6-sol"]
   );
 });