fix(health): opt-in deep check sampling completions surface

changelog.d/fixes/14236-health-deep-check.md

@@ -0,0 +1 @@
+- **fix(health):** the system health endpoint offers an opt-in deep check (`?deep=1`, off by default, authenticated callers only) that samples the completions surface with a minimal non-streaming request and caches the verdict briefly; only `502`/`503` answers raise the failover signal, every other failure keeps serving the existing payload unchanged. Wiring only: the probe target is not configured yet, so `?deep=1` stays inert until a dedicated settings change supplies it. ([#14236](https://github.com/diegosouzapw/OmniRoute/pull/14236)) — thanks @maxmad64bis

src/app/api/monitoring/health/route.ts

@@ -30,8 +30,25 @@ export function __test_resetMonitoringHealthPayloadCache(): void {
   healthPayloadCache = null;
   healthPayloadRefreshInFlight = false;
   healthPayloadCacheGeneration += 1;
+  deepHealthVerdictCache = null;
+  deepHealthRefreshInFlight = false;
 }
 
+/** Test-only: seed the deep-health verdict cache (proves TTL + status passthrough). */
+export function __test_seedDeepHealthVerdict(verdict: unknown, ttlMs = 30_000): void {
+  deepHealthVerdictCache = { verdict, expiresAt: Date.now() + ttlMs };
+}
+
+// Opt-in deep check (off unless DEEP_HEALTH_CHECK_ENABLED=1): an authenticated
+// caller may append ?deep=1 to sample the completions surface once per TTL.
+// Anonymous callers never trigger a probe; the flag gates the rest.
+function isDeepHealthOptedIn(request: Request): boolean {
+  if ((process.env.DEEP_HEALTH_CHECK_ENABLED ?? "").trim() !== "1") return false;
+  return new URL(request.url).searchParams.get("deep") === "1";
+}
+let deepHealthVerdictCache: { verdict: unknown; expiresAt: number } | null = null;
+let deepHealthRefreshInFlight = false;
+
 // GHSA-mvf8-qc78-5mxm: the full health payload fingerprints the host (version,
 // node version, pid, memory, provider config). An anonymous caller — the common
 // case on a keyless install, and what a liveness/load-balancer probe needs — gets
@@ -65,19 +82,58 @@ function scheduleHealthPayloadRefresh(): void {
   });
 }
 
+// Deep verdict helpers: the verdict never alters `status` — it rides along as
+// an additive `deepHealth` key on the full view only, served from its own
+// 30s cache and refreshed off the request path (same SWR motif as above).
+function withDeepHealth(payload: unknown, wantDeep: boolean): unknown {
+  if (!wantDeep || !deepHealthVerdictCache) return payload;
+  return { ...(payload as Record<string, unknown>), deepHealth: deepHealthVerdictCache.verdict };
+}
+
+function refreshDeepHealthVerdict(): void {
+  if (deepHealthRefreshInFlight) return;
+  if (deepHealthVerdictCache && Date.now() <= deepHealthVerdictCache.expiresAt) return;
+  deepHealthRefreshInFlight = true;
+  setImmediate(() => {
+    void (async () => {
+      try {
+        const { probeDeepHealth, DEEP_HEALTH_VERDICT_TTL_MS, DEEP_HEALTH_PROBE_TIMEOUT_MS } =
+          await import("@/lib/monitoring/observability");
+        const { getCachedSettings } = await import("@/lib/db/readCache");
+        const settings = (await getCachedSettings()) as Record<string, unknown>;
+        const deepHealthUrl = typeof settings.deepHealthUrl === "string" ? settings.deepHealthUrl : "";
+        if (!deepHealthUrl) return;
+        const deepHealthToken =
+          typeof settings.deepHealthToken === "string" ? settings.deepHealthToken : undefined;
+        const verdict = await probeDeepHealth(deepHealthUrl, {
+          timeoutMs: DEEP_HEALTH_PROBE_TIMEOUT_MS,
+          token: deepHealthToken,
+        });
+        deepHealthVerdictCache = { verdict, expiresAt: Date.now() + DEEP_HEALTH_VERDICT_TTL_MS };
+      } catch {
+        deepHealthVerdictCache = null;
+      } finally {
+        deepHealthRefreshInFlight = false;
+      }
+    })();
+  });
+}
+
 export async function GET(request: Request) {
   const fullView = (await requireManagementAuth(request, { alwaysRequireAuth: true })) === null;
+  const wantDeep = fullView && isDeepHealthOptedIn(request);
+  if (wantDeep) refreshDeepHealthVerdict();
   const cachedNow = Date.now();
   if (healthPayloadCache) {
     if (cachedNow > healthPayloadCache.expiresAt) {
       scheduleHealthPayloadRefresh();
     }
-    return serveHealthPayload(fullView, healthPayloadCache.payload);
+    return serveHealthPayload(fullView, withDeepHealth(healthPayloadCache.payload, wantDeep));
   }
 
   try {
     const payload = await rebuildHealthPayload();
-    return serveHealthPayload(fullView, payload);
+    return serveHealthPayload(fullView, withDeepHealth(payload, wantDeep));
   } catch (error) {
     console.error("[API] GET /api/monitoring/health error:", error);
     return NextResponse.json({

src/lib/monitoring/observability.ts

@@ -606,3 +606,54 @@ export function buildHealthPayload({
     setupComplete: settings?.setupComplete || false,
   };
 }
+
+/** Short cache window for the opt-in deep-health verdict (distinct from the 1s payload cache). */
+export const DEEP_HEALTH_VERDICT_TTL_MS = 30_000;
+/** Bounded probe budget: the deep check never holds a monitoring request hostage. */
+export const DEEP_HEALTH_PROBE_TIMEOUT_MS = 3_000;
+
+export interface DeepHealthVerdict {
+  ok: boolean;
+  /** Failover signal: true ONLY on 502/503. 4xx/timeout/network never trip it (fail-open). */
+  failover: boolean;
+  status: number | null;
+  latencyMs: number;
+  at: string;
+}
+
+/**
+ * Minimal opt-in liveness probe against the completions surface: 1 token,
+ * non-streaming, bounded timeout. Never throws — every failure mode returns
+ * a verdict with failover:false except 502/503. Never logs bodies.
+ */
+export async function probeDeepHealth(
+  url: string,
+  opts: { timeoutMs?: number; token?: string; fetcher?: typeof fetch } = {}
+): Promise<DeepHealthVerdict> {
+  const timeoutMs = opts.timeoutMs ?? DEEP_HEALTH_PROBE_TIMEOUT_MS;
+  const fetcher = opts.fetcher ?? fetch;
+  const started = Date.now();
+  const verdict = (ok: boolean, failover: boolean, status: number | null): DeepHealthVerdict => ({
+    ok,
+    failover,
+    status,
+    latencyMs: Date.now() - started,
+    at: new Date().toISOString(),
+  });
+  try {
+    const res = await fetcher(url, {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        ...(opts.token ? { authorization: `Bearer ${opts.token}` } : {}),
+      },
+      body: JSON.stringify({ max_tokens: 1, stream: false }),
+      signal: AbortSignal.timeout(timeoutMs),
+    });
+    const status = res.status;
+    if (status >= 200 && status < 300) return verdict(true, false, status);
+    return verdict(false, status === 502 || status === 503, status);
+  } catch {
+    return verdict(false, false, null);
+  }
+}

tests/unit/monitoring-deep-health.test.ts

@@ -0,0 +1,149 @@
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { probeDeepHealth, DEEP_HEALTH_VERDICT_TTL_MS } from "@/lib/monitoring/observability.js";
+
+function stubFetcher(status: number, delayMs = 0): () => Promise<Response> {
+  return async () =>
+    new Promise((resolve) =>
+      setTimeout(() => resolve(new Response("{}", { status })), delayMs)
+    );
+}
+
+describe("probeDeepHealth — fail-open verdict, failover only on 502/503", () => {
+  it("502 and 503 set failover:true", async () => {
+    for (const status of [502, 503]) {
+      const v = await probeDeepHealth("http://localhost:1/v1/chat/completions", {
+        timeoutMs: 3000,
+        fetcher: stubFetcher(status),
+      });
+      assert.equal(v.ok, false);
+      assert.equal(v.failover, true);
+      assert.equal(v.status, status);
+    }
+  });
+
+  it("4xx sets failover:false without throwing", async () => {
+    for (const status of [400, 401, 404, 429]) {
+      const v = await probeDeepHealth("http://localhost:1/v1/chat/completions", {
+        timeoutMs: 3000,
+        fetcher: stubFetcher(status),
+      });
+      assert.equal(v.ok, false);
+      assert.equal(v.failover, false, `status ${status} must never trip failover`);
+    }
+  });
+
+  it("2xx is ok without failover", async () => {
+    const v = await probeDeepHealth("http://localhost:1/v1/chat/completions", {
+      timeoutMs: 3000,
+      fetcher: stubFetcher(200),
+    });
+    assert.equal(v.ok, true);
+    assert.equal(v.failover, false);
+  });
+
+  it("timeout and network errors return a verdict, never throw", async () => {
+    const timeout = await probeDeepHealth("http://localhost:1/v1/chat/completions", {
+      timeoutMs: 50,
+      fetcher: (async () => {
+        await new Promise((_, reject) => setTimeout(() => reject(new Error("aborted")), 5000));
+        return new Response("{}", { status: 200 });
+      }) as typeof fetch,
+    });
+    assert.equal(timeout.ok, false);
+    assert.equal(timeout.failover, false);
+    const refused = await probeDeepHealth("http://localhost:1/v1/chat/completions", {
+      timeoutMs: 3000,
+      fetcher: async () => {
+        throw new Error("connection refused");
+      },
+    });
+    assert.equal(refused.ok, false);
+    assert.equal(refused.failover, false);
+  });
+
+  it("posts a minimal 1-token non-streaming completion", async () => {
+    let seenUrl = "";
+    let seenInit: RequestInit | null = null;
+    await probeDeepHealth("http://gw.example.com/v1/chat/completions", {
+      timeoutMs: 3000,
+      fetcher: (async (url: string, init: RequestInit) => {
+        seenUrl = url;
+        seenInit = init;
+        return new Response("{}", { status: 200 });
+      }) as typeof fetch,
+    });
+    assert.equal(seenUrl, "http://gw.example.com/v1/chat/completions");
+    assert.ok(seenInit, "fetcher must be called");
+    const body = JSON.parse(String(seenInit.body)) as Record<string, unknown>;
+    assert.equal(body["stream"], false);
+    assert.equal(body["max_tokens"], 1);
+  });
+
+  it("verdict TTL is a short cache window", () => {
+    assert.equal(DEEP_HEALTH_VERDICT_TTL_MS, 30_000);
+  });
+});
+
+describe("route gating — the probe never fires unless opted-in and authenticated", () => {
+  it("anonymous callers never trigger a probe even with ?deep=1", async () => {
+    const { GET, __test_resetMonitoringHealthPayloadCache } = await import(
+      "@/app/api/monitoring/health/route.js"
+    );
+    __test_resetMonitoringHealthPayloadCache();
+    const res = (await GET(
+      new Request("http://localhost/api/monitoring/health?deep=1")
+    )) as Response;
+    const body = (await res.json()) as Record<string, unknown>;
+    assert.ok(!("deepHealth" in body), "anonymous view must never carry deepHealth");
+  });
+
+  it("opt-in off + ?deep=1 as management caller → no probe, no deepHealth", async () => {
+    delete process.env.DEEP_HEALTH_CHECK_ENABLED;
+    const { makeManagementSessionRequest } = await import("../helpers/managementSession.ts");
+    const { GET, __test_resetMonitoringHealthPayloadCache } = await import(
+      "@/app/api/monitoring/health/route.js"
+    );
+    __test_resetMonitoringHealthPayloadCache();
+    const req = await makeManagementSessionRequest(
+      "http://localhost/api/monitoring/health?deep=1"
+    );
+    const res = (await GET(req as never)) as Response;
+    const body = (await res.json()) as Record<string, unknown>;
+    assert.ok(!("deepHealth" in body), "flag off must stay inert even when authenticated");
+  });
+
+  it("deepHealth rides along without altering status on success paths", async () => {
+    const { makeManagementSessionRequest } = await import("../helpers/managementSession.ts");
+    const route = await import("@/app/api/monitoring/health/route.js");
+    route.__test_resetMonitoringHealthPayloadCache();
+    const req = await makeManagementSessionRequest("http://localhost/api/monitoring/health");
+    const plain = (await (await route.GET(req as never)).json()) as Record<string, unknown>;
+    assert.ok("status" in plain, "baseline payload must carry status");
+    assert.ok(!("deepHealth" in plain), "no verdict cached yet → no deepHealth key");
+  });
+
+  it("a cached verdict is served on the 2nd call with status intact (TTL)", async () => {
+    process.env.DEEP_HEALTH_CHECK_ENABLED = "1";
+    try {
+      const { makeManagementSessionRequest } = await import("../helpers/managementSession.ts");
+      const route = await import("@/app/api/monitoring/health/route.js");
+      route.__test_resetMonitoringHealthPayloadCache();
+      const seed = { ok: false, failover: true, status: 503, latencyMs: 12, at: "t" };
+      route.__test_seedDeepHealthVerdict(seed);
+      const url = "http://localhost/api/monitoring/health?deep=1";
+      const first = (await (
+        await route.GET(((await makeManagementSessionRequest(url)) as unknown) as never)
+      ).json()) as Record<string, unknown>;
+      const second = (await (
+        await route.GET(((await makeManagementSessionRequest(url)) as unknown) as never)
+      ).json()) as Record<string, unknown>;
+      assert.deepEqual(first["deepHealth"], seed, "1st call serves the cached verdict");
+      assert.deepEqual(second["deepHealth"], seed, "2nd call serves the same cached verdict");
+      assert.equal(second["status"], first["status"], "deepHealth must never alter status");
+      assert.ok(!("error" in second) || second["status"] !== undefined);
+    } finally {
+      delete process.env.DEEP_HEALTH_CHECK_ENABLED;
+    }
+  });
+});