diff --git a/changelog.d/fixes/14236-health-deep-check.md b/changelog.d/fixes/14236-health-deep-check.md new file mode 100644 index 000000000000..32a635de5407 --- /dev/null +++ b/changelog.d/fixes/14236-health-deep-check.md @@ -0,0 +1 @@ +- **fix(health):** the system health endpoint offers an opt-in deep check (`?deep=1`, off by default, authenticated callers only) that samples the completions surface with a minimal non-streaming request and caches the verdict briefly; only `502`/`503` answers raise the failover signal, every other failure keeps serving the existing payload unchanged. Wiring only: the probe target is not configured yet, so `?deep=1` stays inert until a dedicated settings change supplies it. ([#14236](https://github.com/diegosouzapw/OmniRoute/pull/14236)) — thanks @maxmad64bis diff --git a/src/app/api/monitoring/health/route.ts b/src/app/api/monitoring/health/route.ts index 08f3cf04bd92..2759c4d85412 100644 --- a/src/app/api/monitoring/health/route.ts +++ b/src/app/api/monitoring/health/route.ts @@ -30,8 +30,25 @@ export function __test_resetMonitoringHealthPayloadCache(): void { healthPayloadCache = null; healthPayloadRefreshInFlight = false; healthPayloadCacheGeneration += 1; + deepHealthVerdictCache = null; + deepHealthRefreshInFlight = false; } +/** Test-only: seed the deep-health verdict cache (proves TTL + status passthrough). */ +export function __test_seedDeepHealthVerdict(verdict: unknown, ttlMs = 30_000): void { + deepHealthVerdictCache = { verdict, expiresAt: Date.now() + ttlMs }; +} + +// Opt-in deep check (off unless DEEP_HEALTH_CHECK_ENABLED=1): an authenticated +// caller may append ?deep=1 to sample the completions surface once per TTL. +// Anonymous callers never trigger a probe; the flag gates the rest. +function isDeepHealthOptedIn(request: Request): boolean { + if ((process.env.DEEP_HEALTH_CHECK_ENABLED ?? "").trim() !== "1") return false; + return new URL(request.url).searchParams.get("deep") === "1"; +} +let deepHealthVerdictCache: { verdict: unknown; expiresAt: number } | null = null; +let deepHealthRefreshInFlight = false; + // GHSA-mvf8-qc78-5mxm: the full health payload fingerprints the host (version, // node version, pid, memory, provider config). An anonymous caller — the common // case on a keyless install, and what a liveness/load-balancer probe needs — gets @@ -65,19 +82,58 @@ function scheduleHealthPayloadRefresh(): void { }); } +// Deep verdict helpers: the verdict never alters `status` — it rides along as +// an additive `deepHealth` key on the full view only, served from its own +// 30s cache and refreshed off the request path (same SWR motif as above). +function withDeepHealth(payload: unknown, wantDeep: boolean): unknown { + if (!wantDeep || !deepHealthVerdictCache) return payload; + return { ...(payload as Record), deepHealth: deepHealthVerdictCache.verdict }; +} + +function refreshDeepHealthVerdict(): void { + if (deepHealthRefreshInFlight) return; + if (deepHealthVerdictCache && Date.now() <= deepHealthVerdictCache.expiresAt) return; + deepHealthRefreshInFlight = true; + setImmediate(() => { + void (async () => { + try { + const { probeDeepHealth, DEEP_HEALTH_VERDICT_TTL_MS, DEEP_HEALTH_PROBE_TIMEOUT_MS } = + await import("@/lib/monitoring/observability"); + const { getCachedSettings } = await import("@/lib/db/readCache"); + const settings = (await getCachedSettings()) as Record; + const deepHealthUrl = typeof settings.deepHealthUrl === "string" ? settings.deepHealthUrl : ""; + if (!deepHealthUrl) return; + const deepHealthToken = + typeof settings.deepHealthToken === "string" ? settings.deepHealthToken : undefined; + const verdict = await probeDeepHealth(deepHealthUrl, { + timeoutMs: DEEP_HEALTH_PROBE_TIMEOUT_MS, + token: deepHealthToken, + }); + deepHealthVerdictCache = { verdict, expiresAt: Date.now() + DEEP_HEALTH_VERDICT_TTL_MS }; + } catch { + deepHealthVerdictCache = null; + } finally { + deepHealthRefreshInFlight = false; + } + })(); + }); +} + export async function GET(request: Request) { const fullView = (await requireManagementAuth(request, { alwaysRequireAuth: true })) === null; + const wantDeep = fullView && isDeepHealthOptedIn(request); + if (wantDeep) refreshDeepHealthVerdict(); const cachedNow = Date.now(); if (healthPayloadCache) { if (cachedNow > healthPayloadCache.expiresAt) { scheduleHealthPayloadRefresh(); } - return serveHealthPayload(fullView, healthPayloadCache.payload); + return serveHealthPayload(fullView, withDeepHealth(healthPayloadCache.payload, wantDeep)); } try { const payload = await rebuildHealthPayload(); - return serveHealthPayload(fullView, payload); + return serveHealthPayload(fullView, withDeepHealth(payload, wantDeep)); } catch (error) { console.error("[API] GET /api/monitoring/health error:", error); return NextResponse.json({ diff --git a/src/lib/monitoring/observability.ts b/src/lib/monitoring/observability.ts index 6ccac5138888..2a4582d1cd9a 100644 --- a/src/lib/monitoring/observability.ts +++ b/src/lib/monitoring/observability.ts @@ -606,3 +606,54 @@ export function buildHealthPayload({ setupComplete: settings?.setupComplete || false, }; } + +/** Short cache window for the opt-in deep-health verdict (distinct from the 1s payload cache). */ +export const DEEP_HEALTH_VERDICT_TTL_MS = 30_000; +/** Bounded probe budget: the deep check never holds a monitoring request hostage. */ +export const DEEP_HEALTH_PROBE_TIMEOUT_MS = 3_000; + +export interface DeepHealthVerdict { + ok: boolean; + /** Failover signal: true ONLY on 502/503. 4xx/timeout/network never trip it (fail-open). */ + failover: boolean; + status: number | null; + latencyMs: number; + at: string; +} + +/** + * Minimal opt-in liveness probe against the completions surface: 1 token, + * non-streaming, bounded timeout. Never throws — every failure mode returns + * a verdict with failover:false except 502/503. Never logs bodies. + */ +export async function probeDeepHealth( + url: string, + opts: { timeoutMs?: number; token?: string; fetcher?: typeof fetch } = {} +): Promise { + const timeoutMs = opts.timeoutMs ?? DEEP_HEALTH_PROBE_TIMEOUT_MS; + const fetcher = opts.fetcher ?? fetch; + const started = Date.now(); + const verdict = (ok: boolean, failover: boolean, status: number | null): DeepHealthVerdict => ({ + ok, + failover, + status, + latencyMs: Date.now() - started, + at: new Date().toISOString(), + }); + try { + const res = await fetcher(url, { + method: "POST", + headers: { + "content-type": "application/json", + ...(opts.token ? { authorization: `Bearer ${opts.token}` } : {}), + }, + body: JSON.stringify({ max_tokens: 1, stream: false }), + signal: AbortSignal.timeout(timeoutMs), + }); + const status = res.status; + if (status >= 200 && status < 300) return verdict(true, false, status); + return verdict(false, status === 502 || status === 503, status); + } catch { + return verdict(false, false, null); + } +} diff --git a/tests/unit/monitoring-deep-health.test.ts b/tests/unit/monitoring-deep-health.test.ts new file mode 100644 index 000000000000..30bd425894ed --- /dev/null +++ b/tests/unit/monitoring-deep-health.test.ts @@ -0,0 +1,149 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { probeDeepHealth, DEEP_HEALTH_VERDICT_TTL_MS } from "@/lib/monitoring/observability.js"; + +function stubFetcher(status: number, delayMs = 0): () => Promise { + return async () => + new Promise((resolve) => + setTimeout(() => resolve(new Response("{}", { status })), delayMs) + ); +} + +describe("probeDeepHealth — fail-open verdict, failover only on 502/503", () => { + it("502 and 503 set failover:true", async () => { + for (const status of [502, 503]) { + const v = await probeDeepHealth("http://localhost:1/v1/chat/completions", { + timeoutMs: 3000, + fetcher: stubFetcher(status), + }); + assert.equal(v.ok, false); + assert.equal(v.failover, true); + assert.equal(v.status, status); + } + }); + + it("4xx sets failover:false without throwing", async () => { + for (const status of [400, 401, 404, 429]) { + const v = await probeDeepHealth("http://localhost:1/v1/chat/completions", { + timeoutMs: 3000, + fetcher: stubFetcher(status), + }); + assert.equal(v.ok, false); + assert.equal(v.failover, false, `status ${status} must never trip failover`); + } + }); + + it("2xx is ok without failover", async () => { + const v = await probeDeepHealth("http://localhost:1/v1/chat/completions", { + timeoutMs: 3000, + fetcher: stubFetcher(200), + }); + assert.equal(v.ok, true); + assert.equal(v.failover, false); + }); + + it("timeout and network errors return a verdict, never throw", async () => { + const timeout = await probeDeepHealth("http://localhost:1/v1/chat/completions", { + timeoutMs: 50, + fetcher: (async () => { + await new Promise((_, reject) => setTimeout(() => reject(new Error("aborted")), 5000)); + return new Response("{}", { status: 200 }); + }) as typeof fetch, + }); + assert.equal(timeout.ok, false); + assert.equal(timeout.failover, false); + const refused = await probeDeepHealth("http://localhost:1/v1/chat/completions", { + timeoutMs: 3000, + fetcher: async () => { + throw new Error("connection refused"); + }, + }); + assert.equal(refused.ok, false); + assert.equal(refused.failover, false); + }); + + it("posts a minimal 1-token non-streaming completion", async () => { + let seenUrl = ""; + let seenInit: RequestInit | null = null; + await probeDeepHealth("http://gw.example.com/v1/chat/completions", { + timeoutMs: 3000, + fetcher: (async (url: string, init: RequestInit) => { + seenUrl = url; + seenInit = init; + return new Response("{}", { status: 200 }); + }) as typeof fetch, + }); + assert.equal(seenUrl, "http://gw.example.com/v1/chat/completions"); + assert.ok(seenInit, "fetcher must be called"); + const body = JSON.parse(String(seenInit.body)) as Record; + assert.equal(body["stream"], false); + assert.equal(body["max_tokens"], 1); + }); + + it("verdict TTL is a short cache window", () => { + assert.equal(DEEP_HEALTH_VERDICT_TTL_MS, 30_000); + }); +}); + +describe("route gating — the probe never fires unless opted-in and authenticated", () => { + it("anonymous callers never trigger a probe even with ?deep=1", async () => { + const { GET, __test_resetMonitoringHealthPayloadCache } = await import( + "@/app/api/monitoring/health/route.js" + ); + __test_resetMonitoringHealthPayloadCache(); + const res = (await GET( + new Request("http://localhost/api/monitoring/health?deep=1") + )) as Response; + const body = (await res.json()) as Record; + assert.ok(!("deepHealth" in body), "anonymous view must never carry deepHealth"); + }); + + it("opt-in off + ?deep=1 as management caller → no probe, no deepHealth", async () => { + delete process.env.DEEP_HEALTH_CHECK_ENABLED; + const { makeManagementSessionRequest } = await import("../helpers/managementSession.ts"); + const { GET, __test_resetMonitoringHealthPayloadCache } = await import( + "@/app/api/monitoring/health/route.js" + ); + __test_resetMonitoringHealthPayloadCache(); + const req = await makeManagementSessionRequest( + "http://localhost/api/monitoring/health?deep=1" + ); + const res = (await GET(req as never)) as Response; + const body = (await res.json()) as Record; + assert.ok(!("deepHealth" in body), "flag off must stay inert even when authenticated"); + }); + + it("deepHealth rides along without altering status on success paths", async () => { + const { makeManagementSessionRequest } = await import("../helpers/managementSession.ts"); + const route = await import("@/app/api/monitoring/health/route.js"); + route.__test_resetMonitoringHealthPayloadCache(); + const req = await makeManagementSessionRequest("http://localhost/api/monitoring/health"); + const plain = (await (await route.GET(req as never)).json()) as Record; + assert.ok("status" in plain, "baseline payload must carry status"); + assert.ok(!("deepHealth" in plain), "no verdict cached yet → no deepHealth key"); + }); + + it("a cached verdict is served on the 2nd call with status intact (TTL)", async () => { + process.env.DEEP_HEALTH_CHECK_ENABLED = "1"; + try { + const { makeManagementSessionRequest } = await import("../helpers/managementSession.ts"); + const route = await import("@/app/api/monitoring/health/route.js"); + route.__test_resetMonitoringHealthPayloadCache(); + const seed = { ok: false, failover: true, status: 503, latencyMs: 12, at: "t" }; + route.__test_seedDeepHealthVerdict(seed); + const url = "http://localhost/api/monitoring/health?deep=1"; + const first = (await ( + await route.GET(((await makeManagementSessionRequest(url)) as unknown) as never) + ).json()) as Record; + const second = (await ( + await route.GET(((await makeManagementSessionRequest(url)) as unknown) as never) + ).json()) as Record; + assert.deepEqual(first["deepHealth"], seed, "1st call serves the cached verdict"); + assert.deepEqual(second["deepHealth"], seed, "2nd call serves the same cached verdict"); + assert.equal(second["status"], first["status"], "deepHealth must never alter status"); + assert.ok(!("error" in second) || second["status"] !== undefined); + } finally { + delete process.env.DEEP_HEALTH_CHECK_ENABLED; + } + }); +});