cherry-pick(pr-10148): feat(api): add provider quota telemetry, adaptive routing, and status inventory

bin/cli/commands/quota.mjs

@@ -2,7 +2,7 @@ import { apiFetch, isServerUp } from "../api.mjs";
 import { t } from "../i18n.mjs";
 
 export function registerQuota(program) {
-  program
+  const quota = program
     .command("quota")
     .description(t("quota.description"))
     .option("--provider <id>", "Filter by provider")
@@ -12,6 +12,60 @@ export function registerQuota(program) {
       const exitCode = await runQuotaCommand({ ...opts, output: globalOpts.output });
       if (exitCode !== 0) process.exit(exitCode);
     });
+
+  quota
+    .command("status")
+    .description("Show truthful OmniRoute gateway, quota, pool, and circuit state")
+    .action(async (opts, cmd) => runBoundedJson("/api/omniroute/status", cmd.optsWithGlobals()));
+
+  quota
+    .command("preview")
+    .description("Preview allocation enforcement without an upstream request")
+    .requiredOption("--api-key-id <id>", "API key id")
+    .requiredOption("--pool-id <id>", "quota pool id")
+    .option("--tokens <n>", "estimated token usage")
+    .action(async (opts, cmd) => {
+      const params = new URLSearchParams({ apiKeyId: opts.apiKeyId, poolId: opts.poolId });
+      if (opts.tokens != null) params.set("estimatedTokens", opts.tokens);
+      await runBoundedJson(`/api/quota/preview?${params}`, cmd.optsWithGlobals());
+    });
+
+  quota
+    .command("ensure <json>")
+    .description("Idempotently create or update a quota pool from a JSON object")
+    .action(async (json, opts, cmd) => {
+      let body;
+      try {
+        body = JSON.parse(json);
+      } catch {
+        console.error("Invalid pool JSON");
+        process.exit(2);
+      }
+      await runBoundedJson("/api/quota/pools?ensure=true", cmd.optsWithGlobals(), {
+        method: "POST",
+        body,
+      });
+    });
+}
+
+async function runBoundedJson(path, opts, request = {}) {
+  const started = performance.now();
+  const res = await apiFetch(path, {
+    ...request,
+    retry: false,
+    timeout: Math.min(opts.timeout ?? 5000, 5000),
+    acceptNotOk: true,
+  });
+  const elapsed = Math.round(performance.now() - started);
+  if (process.env.OMNIROUTE_DEBUG === "1") {
+    console.error(`[omniroute] ${request.method ?? "GET"} ${path} completed in ${elapsed}ms`);
+  }
+  const payload = await res.json().catch(() => ({ error: `HTTP ${res.status}` }));
+  if (!res.ok) {
+    console.error(JSON.stringify(payload));
+    process.exit(res.exitCode ?? 1);
+  }
+  console.log(JSON.stringify(payload, null, 2));
 }
 
 export async function runQuotaCommand(opts = {}) {

bin/cli/utils/cliToken.mjs

@@ -8,13 +8,11 @@ let _cached = null;
 export async function getCliToken() {
   if (_cached !== null) return _cached;
   try {
-    const { machineIdSync } = await import("node-machine-id");
-    const mid = machineIdSync();
-    _cached = crypto
-      .createHash("sha256")
-      .update(mid + SALT)
-      .digest("hex")
-      .substring(0, 32);
+    const module = await import("node-machine-id");
+    const machineIdSync = module.machineIdSync ?? module.default?.machineIdSync;
+    if (typeof machineIdSync !== "function") throw new Error("machine-id API unavailable");
+    const mid = machineIdSync(true);
+    _cached = crypto.createHmac("sha256", mid).update(SALT).digest("hex");
   } catch {
     _cached = "";
   }

docs/OMNIROUTE_ALLOCATION_HANDOFF.md

@@ -0,0 +1,9 @@
+# OmniRoute Allocation Handoff
+
+Allocation is not provider quota.
+
+Quota pools define which API keys may consume a provider pool and how hard, soft, or burst policies apply. Provider quota is external capacity reported by a provider or an explicitly configured source. Ghostlight internal budgets are governance limits defined by the administrator.
+
+The `ensurePool` operation is idempotent: an identical pool is unchanged, a changed allocation is updated, and a missing pool is created. This is intended for automation and bounded API callers.
+
+The read-only status endpoint is `GET /api/omniroute/status`. The verification command is `npm run omniroute:verify`; it makes no live model request.

docs/OMNIROUTE_PROVIDER_FAILOVER.md

@@ -0,0 +1,9 @@
+# OmniRoute Provider Failover
+
+Failures are classified before retry decisions are made.
+
+Transient failures such as timeouts, network errors, rate limits, and provider 5xx responses may fail over. Authentication errors, permission errors, invalid requests, unavailable models, and unknown failures are not retried blindly.
+
+The default cross-provider policy allows up to three provider attempts, retries rate limits and timeouts, and keeps administrative disablement separate from temporary circuit state.
+
+Circuit states are `closed`, `open`, and `half_open`. A cooldown schedules a bounded probe; a successful probe closes the circuit and a failed probe reopens it.

docs/OMNIROUTE_QUOTA_TELEMETRY.md

@@ -0,0 +1,17 @@
+# OmniRoute Quota Telemetry
+
+OmniRoute separates provider quota telemetry from Ghostlight accounting.
+
+## Truthful states
+
+- `healthy` means a source reported usable remaining capacity.
+- `approaching_limit` means a source reported remaining capacity at or below the configured threshold.
+- `exhausted` is emitted only when a source reports zero capacity or usage at its limit.
+- `unavailable` means a supported source failed to return data.
+- `unknown` means no supported source exists or no provider limit is known.
+
+Unknown is not exhausted and does not disable a provider.
+
+Sources are preferred in this order: official provider API, authenticated usage API, explicitly mapped response headers, administrator configuration, local estimates, unknown. Local estimates are never presented as provider billing data.
+
+Response headers are parsed only through an explicit provider mapping. Generic header names are not assumed globally.

docs/OMNIROUTE_ROUTING_POLICY.md

@@ -0,0 +1,11 @@
+# OmniRoute Routing Policy
+
+Routing preserves the existing capability and combo selection logic, then applies allocation, health, circuit, quota, latency, reliability, model preference, and cost preference factors.
+
+The adaptive score is explainable and returns both the selected candidate and all ranked candidates. Exhausted quota, denied allocation, and open circuits are ineligible. Unknown quota remains eligible with a neutral quota factor.
+
+Route preview is deterministic and performs zero upstream model requests:
+
+`POST /api/omniroute/route/preview`
+
+The response includes candidate scores, factors, reasons, the selected provider, and `liveRequestExecuted: false`.

package.json

@@ -101,6 +101,7 @@
     "build:secure": "OMNIROUTE_BUILD_PROFILE=minimal node scripts/build/build-next-isolated.mjs",
     "build:backend": "cross-env OMNIROUTE_BUILD_BACKEND_ONLY=1 node scripts/build/build-next-isolated.mjs",
     "build:cli": "node --import tsx scripts/build/prepublish.ts",
+    "omniroute:verify": "node scripts/check/omniroute-verify.mjs",
     "build:release": "rm -rf .build dist && OMNIROUTE_BUILD_SHA=$(git rev-parse --short HEAD) npm run build && npm run build:cli && node scripts/build/write-build-sha.mjs",
     "build:native:tproxy": "cd src/mitm/tproxy/native && npx --yes node-gyp rebuild",
     "start": "node scripts/dev/run-next.mjs start",

scripts/check/omniroute-verify.mjs

@@ -0,0 +1,69 @@
+#!/usr/bin/env node
+
+import { CLI_TOKEN_HEADER, getCliToken } from "../../bin/cli/utils/cliToken.mjs";
+
+const baseUrl = (process.env.OMNIROUTE_BASE_URL || "http://127.0.0.1:20128").replace(/\/$/, "");
+const apiKey = process.env.OMNIROUTE_API_KEY || "";
+const timeoutMs = 5000;
+
+async function get(path) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  let hardTimer;
+  const hardTimeout = new Promise((_, reject) => {
+    hardTimer = setTimeout(
+      () => reject(new Error(`request timeout after ${timeoutMs}ms`)),
+      timeoutMs + 100
+    );
+  });
+  try {
+    const response = await Promise.race([
+      fetch(`${baseUrl}${path}`, {
+        headers: {
+          ...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}),
+          [CLI_TOKEN_HEADER]: await getCliToken(),
+        },
+        signal: controller.signal,
+      }),
+      hardTimeout,
+    ]);
+    const body = await response.json().catch(() => null);
+    return { ok: response.ok, status: response.status, body };
+  } finally {
+    clearTimeout(timer);
+    clearTimeout(hardTimer);
+  }
+}
+
+function check(label, passed, detail = "") {
+  console.log(`${label}: ${passed ? "PASS" : "FAIL"}${detail ? ` (${detail})` : ""}`);
+  return passed;
+}
+
+console.log("OmniRoute Verification");
+console.log(`Gateway: ${baseUrl}`);
+const results = [];
+
+try {
+  const models = await get("/v1/models");
+  results.push(check("Gateway", models.ok, `HTTP ${models.status}`));
+  const modelCount = Array.isArray(models.body?.data) ? models.body.data.length : 0;
+  results.push(check("Catalog", modelCount > 0, `${modelCount} models`));
+
+  const pools = await get("/api/quota/pools");
+  const poolRows = Array.isArray(pools.body?.pools) ? pools.body.pools : [];
+  const allocations = poolRows.reduce((sum, pool) => sum + (pool.allocations?.length || 0), 0);
+  results.push(check("Pools", pools.ok, `${poolRows.length}`));
+  results.push(check("Allocations", pools.ok && allocations >= poolRows.length, `${allocations}`));
+
+  const status = await get("/api/omniroute/status");
+  results.push(check("Status API", status.ok, `HTTP ${status.status}`));
+  results.push(check("No live request", status.body?.liveRequestExecuted === false));
+} catch (error) {
+  results.push(
+    check("Verification", false, error instanceof Error ? error.message : String(error))
+  );
+}
+
+console.log(`Live upstream requests: 0`);
+if (results.some((passed) => !passed)) process.exitCode = 1;

src/app/api/omniroute/route/preview/route.ts

@@ -0,0 +1,36 @@
+import { NextResponse } from "next/server";
+import { z } from "zod";
+import { requireManagementAuth } from "@/lib/api/requireManagementAuth";
+import { rankCandidates } from "@/lib/routing/adaptiveRouting";
+
+const candidateSchema = z.object({
+  providerId: z.string().min(1),
+  modelId: z.string().min(1),
+  capabilityScore: z.number().min(0).max(1),
+  allocation: z.enum(["allow", "warn", "deny"]),
+  healthScore: z.number().min(0).max(1),
+  circuit: z.enum(["closed", "open", "half_open"]),
+  quota: z.enum(["healthy", "approaching_limit", "exhausted", "unavailable", "unknown"]),
+  latencyMs: z.number().nonnegative().optional(),
+  errorRate: z.number().min(0).max(1).optional(),
+  modelPreference: z.number().min(0).max(1).optional(),
+  costPreference: z.number().min(0).max(1).optional(),
+});
+
+const requestSchema = z.object({ candidates: z.array(candidateSchema).min(1).max(100) });
+
+/** Deterministic routing preview. It never calls an upstream provider. */
+export async function POST(request: Request): Promise<Response> {
+  const authError = await requireManagementAuth(request);
+  if (authError) return authError;
+  const parsed = requestSchema.safeParse(await request.json().catch(() => null));
+  if (!parsed.success) return NextResponse.json({ error: parsed.error.message }, { status: 400 });
+
+  const result = rankCandidates(parsed.data);
+  return NextResponse.json({
+    request: { candidateCount: parsed.data.candidates.length },
+    ...result,
+    selected: result.selected?.providerId ?? null,
+    liveRequestExecuted: false,
+  });
+}

src/app/api/omniroute/status/route.ts

@@ -0,0 +1,21 @@
+import { NextResponse } from "next/server";
+import { requireManagementAuth } from "@/lib/api/requireManagementAuth";
+import { buildOmniRouteStatus } from "@/lib/omnirouteStatus";
+
+export const dynamic = "force-dynamic";
+
+/** Read-only operational status; never performs an upstream model request. */
+export async function GET(request: Request): Promise<Response> {
+  const authError = await requireManagementAuth(request);
+  if (authError) return authError;
+
+  try {
+    return NextResponse.json({
+      generatedAt: new Date().toISOString(),
+      liveRequestExecuted: false,
+      ...(await buildOmniRouteStatus()),
+    });
+  } catch {
+    return NextResponse.json({ error: "Failed to build OmniRoute status" }, { status: 500 });
+  }
+}

src/app/api/quota/pools/route.ts

@@ -16,7 +16,7 @@ import { NextResponse } from "next/server";
 import { buildErrorBody } from "@omniroute/open-sse/utils/error";
 import { requireManagementAuth } from "@/lib/api/requireManagementAuth";
 import { PoolCreateSchema } from "@/shared/schemas/quota";
-import { listPools, createPool } from "@/lib/localDb";
+import { listPools, createPool, ensurePool } from "@/lib/localDb";
 import { logAuditEvent, getAuditRequestContext } from "@/lib/compliance/index";
 
 export const dynamic = "force-dynamic";
@@ -48,17 +48,28 @@ export async function POST(request: Request): Promise<Response> {
       return NextResponse.json(buildErrorBody(400, parsed.error.message), { status: 400 });
     }
 
-    const pool = createPool(parsed.data);
+    const ensure = new URL(request.url).searchParams.get("ensure") === "true";
+    const ensured = ensure ? ensurePool(parsed.data) : null;
+    const pool = ensured?.pool ?? createPool(parsed.data);
     const ctx = getAuditRequestContext(request);
     logAuditEvent({
-      action: "quota.pool.created",
+      action: ensured?.updated ? "quota.pool.updated" : "quota.pool.created",
       target: pool.id,
-      metadata: { connectionId: pool.connectionId, name: pool.name },
+      metadata: {
+        connectionId: pool.connectionId,
+        name: pool.name,
+        ensure,
+        created: ensured?.created ?? true,
+        updated: ensured?.updated ?? false,
+      },
       ipAddress: ctx.ipAddress ?? undefined,
       requestId: ctx.requestId,
     });
 
-    return NextResponse.json({ pool }, { status: 201 });
+    return NextResponse.json(
+      { pool, ...(ensured ? { created: ensured.created, updated: ensured.updated } : {}) },
+      { status: ensured?.created === false ? 200 : 201 }
+    );
   } catch (err) {
     const message = err instanceof Error ? err.message : "Failed to create pool";
     return NextResponse.json(buildErrorBody(500, message), { status: 500 });