fix(auto): subscription fail-closed, neutral unpriced cost, thrifty rung order (#15546)

changelog.d/fixes/15546-subscription-fail-closed.md

@@ -0,0 +1 @@
+- fix(auto): `auto/subscription` now fails closed when its pool is empty instead of widening to every active provider's full catalog (#15546, complements #15813)

changelog.d/fixes/15546-thrifty-rung-order.md

@@ -0,0 +1 @@
+- fix(auto): `auto/thrifty` keeps its subscription-ladder rung order after scoring instead of letting the score re-sort escalate to metered models first (#15546, complements #15813)

changelog.d/fixes/15546-unpriced-neutral-cost.md

@@ -0,0 +1 @@
+- fix(auto): unpriced models no longer default to a flat $1/M in auto-combo scoring; they take the priced-pool median cost (neutral), and namespaced ids retry the bare model-name pricing lookup (#15546, complements #15813)

open-sse/services/autoCombo/failClosedPool.ts

@@ -0,0 +1,16 @@
+import type { AutoComboSpec } from "./virtualFactory";
+
+/**
+ * #15546: `auto/subscription` promises plan-included capacity only, so an empty
+ * pool must stay empty. The flag is read by `expandAutoComboCandidatePool`,
+ * which would otherwise widen an empty auto pool to every active provider's
+ * full catalog (paid models included).
+ */
+export function failClosedAutoConfig(spec?: Pick<AutoComboSpec, "tier">): {
+  failClosedWhenEmpty?: true;
+  preserveRungOrder?: true;
+} {
+  // `auto/thrifty`: the pool is rung-ordered; scoring must not reshuffle the rungs.
+  if (spec?.tier === "thrifty") return { preserveRungOrder: true };
+  return spec?.tier === "subscription" ? { failClosedWhenEmpty: true } : {};
+}

open-sse/services/autoCombo/virtualFactory.ts

@@ -36,6 +36,7 @@ import { getSyncedAvailableModelsByConnection, getCustomModels } from "@/lib/db/
 import { filterPaidOnlyCandidatesWithDiagnosis } from "./paidModelFilter";
 import { filterLockoutCandidates, warnPoolDrop } from "./modelLockoutFilter";
 import { filterModelExposureCandidates } from "./modelExposureFilter";
+import { failClosedAutoConfig } from "./failClosedPool";
 import {
   filterSubscriptionOnlyCandidates,
   orderPoolByRung,
@@ -975,7 +976,6 @@ export async function createVirtualAutoComboFromPrepared(
     candidatePool.length = 0;
     candidatePool.push(...overrideFilteredPool);
   }
-
   if (candidatePool.length === 0) {
     log.warn("AUTO", "No connected providers with valid credentials for virtual auto-combo");
     const emptyPool: string[] = [];
@@ -984,6 +984,7 @@ export async function createVirtualAutoComboFromPrepared(
       weights: { ...DEFAULT_WEIGHTS },
       explorationRate: 0.05,
       routerStrategy: "lkgp",
+      ...failClosedAutoConfig(spec),
     };
     return {
       id: `virtual-auto-${variant || "default"}`,
@@ -1089,7 +1090,6 @@ export async function createVirtualAutoComboFromPrepared(
       );
     }
   }
-
   let weights: ScoringWeights = { ...DEFAULT_WEIGHTS };
   let explorationRate = 0.05; // Default exploration rate
   let routerStrategy = "lkgp"; // All auto variants use LKGP
@@ -1167,8 +1167,8 @@ export async function createVirtualAutoComboFromPrepared(
     weights,
     explorationRate,
     routerStrategy,
+    ...failClosedAutoConfig(spec),
   };
-
   // Chaos mode fans out to the top-N most stable models in parallel. Panel size
   // is capped to keep a single IDE request from fanning out to dozens of providers;
   // operators can override via env var OMNIROUTE_CHAOS_MAX_PANEL (default 5).

open-sse/services/combo.ts

@@ -10,6 +10,7 @@ import { errorResponse, errorResponseWithComboDiagnostics } from "../utils/error
 import { recordComboFailure } from "./combo/failureTracker.ts";
 import { buildRecoveryHint } from "./combo/pinRecovery.ts";
 import { buildTargetTimeoutRunner } from "./combo/targetTimeoutRunner.ts";
+import { costKey, resolvePoolCosts } from "./combo/candidateCost.ts";
 import { getComboMetrics } from "./comboMetrics.ts";
 import { qualityScoreFor } from "./routing/index.ts";
 import {
@@ -215,7 +216,6 @@ const DEFAULT_MODEL_P95_MS: Record<string, number> = {
   "deepseek-chat": 2000,
 };
 const MIN_HISTORY_SAMPLES = 10;
-const OUTPUT_TOKEN_RATIO = 0.4;
 
 function calculateTargetContextAffinity(
   target: ResolvedComboTarget,
@@ -390,6 +390,17 @@ export async function buildAutoCandidates(
     }
   );
 
+  const poolCosts = await resolvePoolCosts(
+    fingerprintExpandedTargets.map((t) => {
+      const parsed = parseModel(t.modelStr);
+      return {
+        provider: t.provider || parsed.provider || parsed.providerAlias || "unknown",
+        model: parsed.model || t.modelStr,
+      };
+    }),
+    getPricingForModel
+  );
+
   const candidates = await Promise.all(
     fingerprintExpandedTargets.map(async (target) => {
       const modelStr = target.modelStr;
@@ -402,22 +413,7 @@ export async function buildAutoCandidates(
       const hasHistoricalSignal =
         Number.isFinite(historicalTotal) && historicalTotal >= MIN_HISTORY_SAMPLES;
 
-      let costPer1MTokens = 1;
-      try {
-        const pricing = await getPricingForModel(provider, model);
-        const inputPrice = Number(pricing?.input);
-        const outputPrice = Number(pricing?.output);
-        if (Number.isFinite(inputPrice) && inputPrice >= 0) {
-          if (Number.isFinite(outputPrice) && outputPrice >= 0) {
-            costPer1MTokens =
-              inputPrice * (1 - OUTPUT_TOKEN_RATIO) + outputPrice * OUTPUT_TOKEN_RATIO;
-          } else {
-            costPer1MTokens = inputPrice;
-          }
-        }
-      } catch {
-        // keep default cost
-      }
+      const costPer1MTokens = poolCosts.get(costKey(provider, model)) ?? 1;
 
       const modelMetric = metrics?.byModel?.[modelStr] || null;
       const avgLatency = Number(modelMetric?.avgLatencyMs);

open-sse/services/combo/autoStrategy.ts

@@ -466,6 +466,10 @@ export async function expandAutoComboCandidatePool(
   if (Array.isArray(localAutoConfig?.candidatePool) && localAutoConfig.candidatePool.length > 0)
     return eligibleTargets;
 
+  // #15546: `auto/subscription` promises plan-included capacity only. An empty
+  // pool must stay empty (fail closed) rather than widen to the paid catalog.
+  if (localAutoConfig?.failClosedWhenEmpty === true) return eligibleTargets;
+
   // #COMBO-REF: if the combo references other combos via kind:"combo-ref" entries,
   // the resolved eligibleTargets already represent the operator's intended pool.
   // Expanding to ALL providers would defeat the purpose of the combo-ref constraint

open-sse/services/combo/candidateCost.ts

@@ -0,0 +1,75 @@
+/**
+ * Auto-combo candidate cost resolution (#15546).
+ *
+ * The scorer min-max normalises `costPer1MTokens` across the pool. A model with
+ * no pricing row used to default to a flat $1/M, which sits near the bottom of a
+ * pool that contains $20-$40/M frontier models, so genuinely UNPRICED models
+ * looked almost free and won cost-weighted routes. We now treat an unpriced
+ * model neutrally: it takes the median blended cost of the priced models in the
+ * same pool (so it neither wins nor loses on cost), and only falls back to 1
+ * when nothing in the pool is priced (every candidate then ties anyway).
+ */
+
+/** Share of output tokens assumed when blending input/output prices. */
+export const OUTPUT_TOKEN_RATIO = 0.4;
+const NO_PRICING_FALLBACK = 1;
+
+export type PricingLookup = (
+  provider: string,
+  model: string
+) => Promise<{ input?: unknown; output?: unknown } | null | undefined>;
+
+export interface CostKey {
+  provider: string;
+  model: string;
+}
+
+export const costKey = (provider: string, model: string): string => `${provider}/${model}`;
+
+function blend(pricing: { input?: unknown; output?: unknown } | null | undefined): number | null {
+  const input = Number(pricing?.input);
+  if (!Number.isFinite(input) || input < 0) return null;
+  const output = Number(pricing?.output);
+  if (!Number.isFinite(output) || output < 0) return input;
+  return input * (1 - OUTPUT_TOKEN_RATIO) + output * OUTPUT_TOKEN_RATIO;
+}
+
+async function lookupBlended(
+  provider: string,
+  model: string,
+  getPricing: PricingLookup
+): Promise<number | null> {
+  try {
+    const exact = blend(await getPricing(provider, model));
+    if (exact !== null) return exact;
+    // Same fallback as usage/costCalculator: namespaced ids ("org/model").
+    const bare = model.includes("/") ? model.split("/").pop() || model : model;
+    return bare !== model ? blend(await getPricing(provider, bare)) : null;
+  } catch {
+    return null;
+  }
+}
+
+function median(values: number[]): number {
+  const sorted = [...values].sort((a, b) => a - b);
+  const mid = Math.floor(sorted.length / 2);
+  return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
+}
+
+/** Blended $/1M per `provider/model`; unpriced entries get the priced-pool median. */
+export async function resolvePoolCosts(
+  items: CostKey[],
+  getPricing: PricingLookup
+): Promise<Map<string, number>> {
+  const unique = new Map<string, CostKey>();
+  for (const item of items) unique.set(costKey(item.provider, item.model), item);
+  const entries = await Promise.all(
+    [...unique.entries()].map(
+      async ([key, item]) =>
+        [key, await lookupBlended(item.provider, item.model, getPricing)] as const
+    )
+  );
+  const priced = entries.filter((e): e is readonly [string, number] => e[1] !== null);
+  const neutral = priced.length > 0 ? median(priced.map((e) => e[1])) : NO_PRICING_FALLBACK;
+  return new Map(entries.map(([key, cost]) => [key, cost ?? neutral]));
+}

open-sse/services/combo/resolveAutoStrategy.ts

@@ -22,6 +22,7 @@ import { supportsToolCalling } from "../modelCapabilities.ts";
 import type { ResilienceSettings } from "../../../src/lib/resilience/settings";
 import { parseAutoConfig } from "./autoConfig.ts";
 import { dedupeTargetsByExecutionKey } from "./comboData.ts";
+import { restoreIncomingOrder } from "./rungOrder.ts";
 import {
   getModelContextLimitForModelString,
   providerSupportsEmulatedToolCalling,
@@ -455,8 +456,14 @@ export async function resolveAutoStrategyOrder(
     // routable ranked ones (and, when the cutoff is OFF, makes this identical to
     // the pre-cutoff behavior), but a quota-blocked target still survives as a
     // final fallback instead of vanishing — the hard cutoff only de-prioritizes.
+    const preserveRungOrder =
+      (combo as { autoConfig?: { preserveRungOrder?: unknown } })?.autoConfig?.preserveRungOrder ===
+      true;
+    const headTargets = preserveRungOrder
+      ? restoreIncomingOrder(rankedTargets, eligibleTargets)
+      : [selectedTarget, ...rankedTargets];
     orderedTargets = dedupeTargetsByExecutionKey(
-      [selectedTarget, ...rankedTargets, ...eligibleTargets].filter(
+      [...headTargets, ...eligibleTargets].filter(
         (entry): entry is ResolvedComboTarget => entry !== undefined && entry !== null
       )
     );

open-sse/services/combo/rungOrder.ts

@@ -0,0 +1,33 @@
+/**
+ * `auto/thrifty` (#15546): the pool is built in subscription-ladder rung order
+ * (plan-included -> free -> metered), but the auto scorer then re-sorts every
+ * target by score, which can put a metered model ahead of a plan-included one.
+ * For ladder routes the rung order is the contract, so after scoring we put the
+ * targets back in the order the virtual pool delivered them.
+ */
+interface OrderedTarget {
+  modelStr: string;
+  connectionId?: string | null;
+}
+
+const keyOf = (t: OrderedTarget): string => `${t.modelStr}|${t.connectionId ?? ""}`;
+
+/** Stable re-sort of `ranked` by position in `incoming`; unknown targets keep score order, last. */
+export function restoreIncomingOrder<T extends OrderedTarget>(
+  ranked: readonly T[],
+  incoming: readonly OrderedTarget[]
+): T[] {
+  const position = new Map<string, number>();
+  incoming.forEach((target, index) => {
+    const key = keyOf(target);
+    if (!position.has(key)) position.set(key, index);
+  });
+  return ranked
+    .map((target, scoreIndex) => ({
+      target,
+      rung: position.get(keyOf(target)) ?? Number.POSITIVE_INFINITY,
+      scoreIndex,
+    }))
+    .sort((a, b) => (a.rung === b.rung ? a.scoreIndex - b.scoreIndex : a.rung - b.rung))
+    .map((entry) => entry.target);
+}

tests/unit/issue-15546-subscription-fail-closed.test.ts

@@ -0,0 +1,53 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+// #15546 follow-up #2: the virtual `auto/subscription` route must fail CLOSED.
+// When the subscription pool is empty, expandAutoComboCandidatePool must NOT
+// widen it to every model of every active provider (paid models included).
+
+const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-15546-sub-"));
+const ORIGINAL_DATA_DIR = process.env.DATA_DIR;
+process.env.DATA_DIR = TEST_DATA_DIR;
+
+const core = await import("../../src/lib/db/core.ts");
+const providersDb = await import("../../src/lib/db/providers.ts");
+const combo = await import("../../open-sse/services/combo.ts");
+const factory = await import("../../open-sse/services/autoCombo/virtualFactory.ts");
+
+test.after(() => {
+  core.resetDbInstance();
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
+  if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR;
+  else process.env.DATA_DIR = ORIGINAL_DATA_DIR;
+});
+
+async function emptyVirtual(tier: "subscription" | "thrifty" | "fast") {
+  const prepared = { regularCandidates: [], familyCandidates: [] };
+  return factory.createVirtualAutoComboFromPrepared(prepared as never, undefined, {
+    category: "chat",
+    tier,
+  } as never);
+}
+
+test("empty auto/subscription pool is not widened to the full catalog", async () => {
+  await providersDb.createProviderConnection({
+    provider: "openai",
+    authType: "apikey",
+    name: "OpenAI",
+    apiKey: "sk-test-openai",
+    defaultModel: "gpt-4o-mini",
+  });
+  const virtual = await emptyVirtual("subscription");
+  assert.equal(virtual.models.length, 0);
+  const expanded = await combo.expandAutoComboCandidatePool([], virtual as never);
+  assert.equal(expanded.length, 0, "subscription route must fail closed (no paid fallback)");
+});
+
+test("empty non-subscription virtual pool keeps legacy expansion", async () => {
+  const virtual = await emptyVirtual("fast");
+  const expanded = await combo.expandAutoComboCandidatePool([], virtual as never);
+  assert.ok(expanded.length > 0);
+});

tests/unit/issue-15546-thrifty-rung-order.test.ts

@@ -0,0 +1,38 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+
+import { restoreIncomingOrder } from "../../open-sse/services/combo/rungOrder.ts";
+import { failClosedAutoConfig } from "../../open-sse/services/autoCombo/failClosedPool.ts";
+
+const t = (modelStr: string, connectionId: string | null = null) => ({
+  kind: "model" as const,
+  modelStr,
+  connectionId,
+});
+
+test("restoreIncomingOrder keeps the ladder rung order over the score order", () => {
+  // incoming = rung order (subscription -> free -> metered); scored = best score first
+  const rung = [t("sub/a", "c1"), t("free/b", "c2"), t("paid/c", "c3")];
+  const scored = [rung[2], rung[0], rung[1]];
+  const out = restoreIncomingOrder(scored, rung);
+  assert.deepEqual(
+    out.map((x) => x.modelStr),
+    ["sub/a", "free/b", "paid/c"]
+  );
+});
+
+test("restoreIncomingOrder keeps unknown targets in relative score order after known ones", () => {
+  const rung = [t("sub/a", "c1")];
+  const scored = [t("x/new1", "n1"), rung[0], t("x/new2", "n2")];
+  const out = restoreIncomingOrder(scored, rung);
+  assert.deepEqual(
+    out.map((x) => x.modelStr),
+    ["sub/a", "x/new1", "x/new2"]
+  );
+});
+
+test("only auto/thrifty marks the virtual combo to preserve rung order", () => {
+  assert.equal(failClosedAutoConfig({ tier: "thrifty" }).preserveRungOrder, true);
+  assert.equal(failClosedAutoConfig({ tier: "subscription" }).preserveRungOrder, undefined);
+  assert.equal(failClosedAutoConfig({ tier: "fast" }).preserveRungOrder, undefined);
+});