feat(rankings): expose provider reliability alongside free provider rankings

changelog.d/features/10909-free-provider-rankings-reliability.md

@@ -0,0 +1 @@
+- **feat(rankings):** free provider rankings now expose a `reliability` field (raw `testStatus`/`rateLimitedUntil` per connection plus a `healthy`/`degraded`/`down` state, reusing the `ProviderHealthState` vocabulary of the provider health matrix) when the configured/available filters are active — derived from already-loaded data, without touching the ranking order ([#10909](https://github.com/diegosouzapw/OmniRoute/pull/10909))

src/lib/freeProviderRankings.ts

@@ -13,12 +13,16 @@ import { REGISTRY } from "@omniroute/open-sse/config/providerRegistry";
 import { listModelIntelligence } from "./db/modelIntelligence";
 import { getProviderConnections } from "./db/providers";
 import { getCustomModels } from "./db/models";
+// Type-only: reuse the health vocabulary instead of forking it.
+import type { ProviderHealthState } from "./monitoring/providerHealthMatrix";
 import type { ProviderAuthType } from "./freeProviderRankingsAuthType";
 
 // Re-exported for backward-compat / same-module ergonomics (#6915) — the
 // actual implementations live in `freeProviderRankingsAuthType.ts` (DB-free,
 // safe to import from "use client" pages; see that file's header comment).
 export type { ProviderAuthType } from "./freeProviderRankingsAuthType";
+// Re-exported for consumers of `reliability`; the definition stays in monitoring.
+export type { ProviderHealthState } from "./monitoring/providerHealthMatrix";
 export {
   filterRankingsByAuthType,
   sortRankingsAuthTypeFirst,
@@ -43,6 +47,8 @@ export interface FreeProviderRanking {
   topModel: ProviderModelScore | null;
   averageScore: number;
   modelCount: number;
+  /** Present only when connection state was loaded (filters active). See `ProviderReliability`. */
+  reliability?: ProviderReliability;
 }
 
 /**
@@ -222,6 +228,28 @@ export interface ConnectionState {
   rateLimitedUntil?: string | null;
 }
 
+/**
+ * Second, additive dimension exposed on each ranking when connection state is
+ * loaded (configured/available filters active). Derived from data the ranking
+ * builder already holds — zero extra query.
+ *
+ * States use `ProviderHealthState` (`src/lib/monitoring/providerHealthMatrix.ts`)
+ * so both surfaces describe a provider the same way. The raw signals stay
+ * verbatim next to the state: `testStatus` is written on failure paths only and
+ * reset to `active` by an explicit connection test or a re-auth, so it can
+ * outlive the actual recovery.
+ */
+export interface ProviderReliability {
+  /** Same triplet `ProviderHealthMatrixAccount` exposes, one per connection. */
+  connections: Array<{
+    testStatus: string | null;
+    rateLimitedUntil: string | null;
+    state: ProviderHealthState;
+  }>;
+  /** Provider aggregate; absent entirely for providers with no loaded connection. */
+  state: ProviderHealthState;
+}
+
 /**
  * Options controlling the additive "configured" / "available" filters.
  * Both default off (undefined/false) → output identical to current behavior.
@@ -233,6 +261,22 @@ export interface FreeProviderRankingFilterOptions {
   availableOnly?: boolean;
 }
 
+/** Group connection states by provider id (shared by filter and reliability attach). */
+function groupConnectionsByProvider(
+  connections: ConnectionState[]
+): Map<string, ConnectionState[]> {
+  const byProvider = new Map<string, ConnectionState[]>();
+  for (const conn of connections) {
+    const list = byProvider.get(conn.provider);
+    if (list) {
+      list.push(conn);
+    } else {
+      byProvider.set(conn.provider, [conn]);
+    }
+  }
+  return byProvider;
+}
+
 // Terminal connection statuses — mirrors `isTerminalConnectionStatus`
 // (`src/sse/services/auth.ts`). A connection in one of these states stays
 // unavailable until credentials/settings change; it never self-recovers.
@@ -250,16 +294,34 @@ const TERMINAL_CONNECTION_STATUSES = new Set(["credits_exhausted", "banned", "ex
  * quota lockout (model lockout, `open-sse/services/accountFallback.ts`) is a
  * deferred Phase 3 and is intentionally NOT consulted here.
  */
-export function isProviderUsable(connections: ConnectionState[], now: number = Date.now()): boolean {
-  return connections.some((conn) => {
-    const status = (conn.testStatus || "").trim().toLowerCase();
-    if (TERMINAL_CONNECTION_STATUSES.has(status)) return false;
-    if (conn.rateLimitedUntil) {
-      const until = new Date(conn.rateLimitedUntil).getTime();
-      if (Number.isFinite(until) && until > now) return false;
-    }
-    return true;
-  });
+export function isProviderUsable(
+  connections: ConnectionState[],
+  now: number = Date.now()
+): boolean {
+  return connections.some((conn) => classifyConnection(conn, now) === "healthy");
+}
+
+/**
+ * One connection, classified as `classifyAccount` does (health matrix): terminal
+ * status ⇒ `down`, live cooldown ⇒ `degraded`, else `healthy`. Model lockouts are
+ * not loaded here, so — as in `isProviderUsable` — they are not consulted.
+ * The filter reuses this, so it cannot drift from the reported state.
+ */
+function classifyConnection(conn: ConnectionState, now: number): ProviderHealthState {
+  const status = (conn.testStatus || "").trim().toLowerCase();
+  if (TERMINAL_CONNECTION_STATUSES.has(status)) return "down";
+  if (conn.rateLimitedUntil) {
+    const until = new Date(conn.rateLimitedUntil).getTime();
+    if (Number.isFinite(until) && until > now) return "degraded";
+  }
+  return "healthy";
+}
+
+/** Mirrors `classifyProvider`, minus its circuit-breaker input (not loaded here). */
+function classifyProviderConnections(states: ProviderHealthState[]): ProviderHealthState {
+  if (states.length > 0 && states.every((state) => state === "down")) return "down";
+  if (states.some((state) => state !== "healthy")) return "degraded";
+  return "healthy";
 }
 
 /**
@@ -281,15 +343,7 @@ export function filterFreeProviderRankings(
   const { configuredOnly, availableOnly } = opts;
   if (!configuredOnly && !availableOnly) return rankings;
 
-  const byProvider = new Map<string, ConnectionState[]>();
-  for (const conn of connections) {
-    const list = byProvider.get(conn.provider);
-    if (list) {
-      list.push(conn);
-    } else {
-      byProvider.set(conn.provider, [conn]);
-    }
-  }
+  const byProvider = groupConnectionsByProvider(connections);
 
   return rankings.filter((ranking) => {
     const conns = byProvider.get(ranking.id);
@@ -299,6 +353,34 @@ export function filterFreeProviderRankings(
   });
 }
 
+/**
+ * Pure enrichment: attach `reliability` to every ranking with a loaded
+ * connection. Rankings without one are returned unchanged, never mutated.
+ */
+export function attachProviderReliability(
+  rankings: FreeProviderRanking[],
+  connections: ConnectionState[],
+  now: number = Date.now()
+): FreeProviderRanking[] {
+  const byProvider = groupConnectionsByProvider(connections);
+  return rankings.map((ranking) => {
+    const conns = byProvider.get(ranking.id);
+    if (!conns || conns.length === 0) return ranking;
+    const states = conns.map((c) => classifyConnection(c, now));
+    return {
+      ...ranking,
+      reliability: {
+        connections: conns.map((c, i) => ({
+          testStatus: c.testStatus ?? null,
+          rateLimitedUntil: c.rateLimitedUntil ?? null,
+          state: states[i],
+        })),
+        state: classifyProviderConnections(states),
+      },
+    };
+  });
+}
+
 /**
  * Compute rankings for free providers based on ELO scores.
  *
@@ -388,6 +470,10 @@ export async function computeFreeProviderRankings(
       isActive: true,
     })) as unknown as ConnectionState[];
     filtered = filterFreeProviderRankings(rankings, connections, opts);
+    // Second dimension, same snapshot: annotation only, sort and scores untouched.
+    // `availableOnly` already drops providers with no healthy connection, so under
+    // it `state` is never `down`; `down` needs `configuredOnly` alone.
+    filtered = attachProviderReliability(filtered, connections);
   }
 
   return filtered.slice(0, limit);

tests/unit/freeProviderRankings-filters.test.ts

@@ -11,6 +11,7 @@ import assert from "node:assert/strict";
 import {
   isProviderUsable,
   filterFreeProviderRankings,
+  attachProviderReliability,
   type ConnectionState,
   type FreeProviderRanking,
 } from "../../src/lib/freeProviderRankings.ts";
@@ -105,12 +106,7 @@ test("filter: configuredOnly keeps only providers with ≥1 connection", () => {
 
 test("filter: availableOnly drops exhausted-only provider, keeps healthy", () => {
   const connections = [conn("glm"), conn("groq", { testStatus: "credits_exhausted" })];
-  const out = filterFreeProviderRankings(
-    RANKINGS,
-    connections,
-    { availableOnly: true },
-    FIXED_NOW
-  );
+  const out = filterFreeProviderRankings(RANKINGS, connections, { availableOnly: true }, FIXED_NOW);
   assert.deepEqual(
     out.map((r) => r.id),
     ["glm"]
@@ -146,12 +142,7 @@ test("filter: availableOnly keeps a provider that has at least one usable connec
     conn("glm", { testStatus: "banned" }),
     conn("glm"), // second connection is healthy
   ];
-  const out = filterFreeProviderRankings(
-    RANKINGS,
-    connections,
-    { availableOnly: true },
-    FIXED_NOW
-  );
+  const out = filterFreeProviderRankings(RANKINGS, connections, { availableOnly: true }, FIXED_NOW);
   assert.deepEqual(
     out.map((r) => r.id),
     ["glm"]
@@ -163,3 +154,115 @@ test("filter: availableOnly implies configured (unconfigured provider excluded)"
   const out = filterFreeProviderRankings(RANKINGS, [], { availableOnly: true }, FIXED_NOW);
   assert.equal(out.length, 0);
 });
+
+// ──────────────── attachProviderReliability ────────────────
+
+test("attachProviderReliability: healthy connections -> state healthy, signals exposed raw", () => {
+  const rankings = [ranking("alpha"), ranking("beta")];
+  const connections = [
+    conn("alpha", { testStatus: "active", rateLimitedUntil: null }),
+    conn("alpha", { testStatus: "active", rateLimitedUntil: past() }),
+  ];
+  const out = attachProviderReliability(rankings, connections, FIXED_NOW);
+
+  assert.equal(out.length, 2);
+  const alpha = out[0];
+  assert.ok(alpha.reliability, "reliability must be attached to alpha");
+  assert.equal(alpha.reliability.state, "healthy");
+  assert.deepEqual(alpha.reliability.connections, [
+    { testStatus: "active", rateLimitedUntil: null, state: "healthy" },
+    { testStatus: "active", rateLimitedUntil: past(), state: "healthy" },
+  ]);
+  assert.equal(out[1].reliability, undefined, "beta has no connection -> no reliability");
+});
+
+test("attachProviderReliability: a terminal status is down, not degraded", () => {
+  const rankings = [ranking("alpha")];
+  const connections = [conn("alpha", { testStatus: "expired", rateLimitedUntil: null })];
+  const out = attachProviderReliability(rankings, connections, FIXED_NOW);
+
+  // Same split as `classifyAccount` in the health matrix: terminal => down.
+  assert.equal(out[0].reliability?.state, "down");
+  assert.equal(out[0].reliability?.connections[0].state, "down");
+  // Raw signal is NOT reinterpreted: "expired" is exposed exactly as stored.
+  assert.equal(out[0].reliability?.connections[0].testStatus, "expired");
+});
+
+test("attachProviderReliability: future rateLimitedUntil degrades; past one does not", () => {
+  const futureLimited = [ranking("alpha")];
+  const f = attachProviderReliability(
+    futureLimited,
+    [conn("alpha", { testStatus: "active", rateLimitedUntil: future() })],
+    FIXED_NOW
+  );
+  assert.equal(f[0].reliability?.state, "degraded");
+  assert.equal(f[0].reliability?.connections[0].rateLimitedUntil, future());
+
+  const pastLimited = [ranking("alpha")];
+  const p = attachProviderReliability(
+    pastLimited,
+    [conn("alpha", { testStatus: "active", rateLimitedUntil: past() })],
+    FIXED_NOW
+  );
+  assert.equal(p[0].reliability?.state, "healthy");
+});
+
+test("attachProviderReliability: one down + one healthy connection -> provider degraded", () => {
+  // Only an all-down set is `down` (as in `classifyProvider`) — and this is the
+  // case that survives `availableOnly`, so the field stays informative under it.
+  const out = attachProviderReliability(
+    [ranking("alpha")],
+    [conn("alpha", { testStatus: "banned" }), conn("alpha", { testStatus: "active" })],
+    FIXED_NOW
+  );
+  assert.equal(out[0].reliability?.state, "degraded");
+  assert.deepEqual(
+    out[0].reliability?.connections.map((c) => c.state),
+    ["down", "healthy"]
+  );
+});
+
+test("attachProviderReliability: every connection down -> provider down", () => {
+  const out = attachProviderReliability(
+    [ranking("alpha")],
+    [conn("alpha", { testStatus: "banned" }), conn("alpha", { testStatus: "credits_exhausted" })],
+    FIXED_NOW
+  );
+  assert.equal(out[0].reliability?.state, "down");
+});
+
+test("attachProviderReliability: raw testStatus stays verbatim, never rewritten by the state", () => {
+  // The state reads `testStatus`, it never replaces it: the stored value comes
+  // back untouched, original casing and padding included.
+  const out = attachProviderReliability(
+    [ranking("alpha")],
+    [conn("alpha", { testStatus: "  EXPIRED ", rateLimitedUntil: null })],
+    FIXED_NOW
+  );
+  assert.equal(out[0].reliability?.connections[0].testStatus, "  EXPIRED ");
+  assert.equal(out[0].reliability?.connections[0].state, "down");
+});
+
+test("attachProviderReliability: provider without connection keeps its ranking unchanged (no field)", () => {
+  const rankings = [ranking("alpha"), ranking("beta")];
+  const out = attachProviderReliability(rankings, [conn("alpha")], FIXED_NOW);
+  assert.equal(out[1].reliability, undefined);
+  assert.deepEqual(
+    out[1],
+    ranking("beta"),
+    "entry without connection must be structurally identical to input"
+  );
+});
+
+test("attachProviderReliability: input rankings are never mutated (pure function)", () => {
+  const rankings = [ranking("alpha")];
+  const before = JSON.stringify(rankings);
+  const out = attachProviderReliability(
+    rankings,
+    [conn("alpha", { testStatus: "expired" })],
+    FIXED_NOW
+  );
+  assert.notEqual(out, rankings, "returns a new array");
+  assert.notEqual(out[0], rankings[0], "returns new objects");
+  assert.equal(JSON.stringify(rankings), before, "input untouched");
+});