From 26a9cfe97f67d03c9e52ed16e221c5daa9817ecf Mon Sep 17 00:00:00 2001 From: "R.D." Date: Tue, 9 Jun 2026 17:02:44 -0400 Subject: [PATCH 1/2] Fix runtime breaker state rendering --- .../dashboard/runtime/RuntimePageClient.tsx | 106 ++++++++-- tests/unit/ui/runtime-page-client.test.tsx | 190 ++++++++++++++++++ 2 files changed, 274 insertions(+), 22 deletions(-) create mode 100644 tests/unit/ui/runtime-page-client.test.tsx diff --git a/src/app/(dashboard)/dashboard/runtime/RuntimePageClient.tsx b/src/app/(dashboard)/dashboard/runtime/RuntimePageClient.tsx index ec429c9b517..bff773f6ab4 100644 --- a/src/app/(dashboard)/dashboard/runtime/RuntimePageClient.tsx +++ b/src/app/(dashboard)/dashboard/runtime/RuntimePageClient.tsx @@ -10,7 +10,8 @@ import ProviderIcon from "@/shared/components/ProviderIcon"; // Types // ───────────────────────────────────────────────────────────────────────────── -type BreakerState = "CLOSED" | "OPEN" | "HALF_OPEN"; +type KnownBreakerState = "CLOSED" | "OPEN" | "HALF_OPEN" | "DEGRADED"; +type BreakerState = KnownBreakerState | (string & {}); type ProviderBreaker = { provider: string; @@ -110,11 +111,11 @@ type FeedFilter = "all" | "circuits" | "cooldowns" | "lockouts" | "sessions" | " const REFRESH_INTERVAL_MS = 5000; const FEED_MAX_EVENTS = 50; +const EMPTY_PROVIDER_BREAKERS: ProviderBreaker[] = []; -const BREAKER_TONE: Record< - BreakerState, - { dot: string; bg: string; ring: string; label: string; icon: string } -> = { +type BreakerTone = { dot: string; bg: string; ring: string; label: string; icon: string }; + +const BREAKER_TONE: Record = { CLOSED: { dot: "#22c55e", bg: "rgba(34,197,94,0.10)", @@ -129,6 +130,13 @@ const BREAKER_TONE: Record< label: "RECOV", icon: "sync", }, + DEGRADED: { + dot: "#f97316", + bg: "rgba(249,115,22,0.10)", + ring: "rgba(249,115,22,0.30)", + label: "DEG", + icon: "warning", + }, OPEN: { dot: "#ef4444", bg: "rgba(239,68,68,0.10)", @@ -138,6 +146,14 @@ const BREAKER_TONE: Record< }, }; +const FALLBACK_BREAKER_TONE: BreakerTone = { + dot: "#64748b", + bg: "rgba(100,116,139,0.10)", + ring: "rgba(100,116,139,0.30)", + label: "UNK", + icon: "help", +}; + const FEED_KIND_META: Record = { "circuit-opened": { icon: "block", color: "#ef4444", group: "circuits" }, "circuit-recovered": { icon: "sync", color: "#eab308", group: "circuits" }, @@ -188,6 +204,16 @@ function untilMs(value: number | string | null | undefined): number { return 0; } +function normalizeBreakerState(state: string | null | undefined): string { + return String(state || "") + .trim() + .toUpperCase(); +} + +function getBreakerTone(state: string | null | undefined): BreakerTone { + return BREAKER_TONE[normalizeBreakerState(state)] || FALLBACK_BREAKER_TONE; +} + function pushFeed(prev: FeedEvent[], events: FeedEvent[]): FeedEvent[] { if (events.length === 0) return prev; const merged = [...events, ...prev]; @@ -210,8 +236,10 @@ function diffSnapshots( for (const [provider, nextB] of nextBreakers) { const prevB = prevBreakers.get(provider); if (!prevB) continue; - if (prevB.state === nextB.state) continue; - if (nextB.state === "OPEN") { + const prevState = normalizeBreakerState(prevB.state); + const nextState = normalizeBreakerState(nextB.state); + if (prevState === nextState) continue; + if (nextState === "OPEN") { out.push({ id: `cb-open-${provider}-${nowTs}`, ts: nowTs, @@ -219,7 +247,7 @@ function diffSnapshots( title: `${provider} circuit OPEN`, detail: `threshold hit · retry in ${fmtMs(nextB.retryAfterMs)}`, }); - } else if (nextB.state === "HALF_OPEN") { + } else if (nextState === "HALF_OPEN") { out.push({ id: `cb-half-${provider}-${nowTs}`, ts: nowTs, @@ -227,7 +255,15 @@ function diffSnapshots( title: `${provider} HALF_OPEN`, detail: `probing recovery`, }); - } else if (nextB.state === "CLOSED" && prevB.state !== "CLOSED") { + } else if (nextState === "DEGRADED") { + out.push({ + id: `cb-deg-${provider}-${nowTs}`, + ts: nowTs, + kind: "circuit-recovered", + title: `${provider} DEGRADED`, + detail: `${nextB.failureCount} failures · degraded but serving`, + }); + } else if (nextState === "CLOSED" && prevState !== "CLOSED") { out.push({ id: `cb-close-${provider}-${nowTs}`, ts: nowTs, @@ -423,16 +459,26 @@ export default function RuntimePageClient() { return connections.filter((c) => c.rateLimitedUntil && untilMs(c.rateLimitedUntil) > 0); }, [connections]); - const breakers = health?.providerBreakers ?? []; + const breakers = health?.providerBreakers ?? EMPTY_PROVIDER_BREAKERS; const lockoutEntries = useMemo>( () => Object.entries(health?.lockouts ?? {}), [health] ); const counts = useMemo(() => { - const openCircuits = breakers.filter((b) => b.state === "OPEN").length; - const halfCircuits = breakers.filter((b) => b.state === "HALF_OPEN").length; + const knownBreakerStates = new Set(["CLOSED", "OPEN", "HALF_OPEN", "DEGRADED"]); + const openCircuits = breakers.filter((b) => normalizeBreakerState(b.state) === "OPEN").length; + const halfCircuits = breakers.filter( + (b) => normalizeBreakerState(b.state) === "HALF_OPEN" + ).length; + const degradedCircuits = breakers.filter( + (b) => normalizeBreakerState(b.state) === "DEGRADED" + ).length; + const unknownCircuits = breakers.filter( + (b) => !knownBreakerStates.has(normalizeBreakerState(b.state)) + ).length; const totalBreakers = breakers.length; + const affectedCircuits = openCircuits + halfCircuits + degradedCircuits + unknownCircuits; const sessions = health?.sessions?.activeCount ?? 0; const lockouts = lockoutEntries.length; const quota = health?.quotaMonitor; @@ -443,6 +489,9 @@ export default function RuntimePageClient() { stickyBound: health?.sessions?.stickyBoundCount ?? 0, openCircuits, halfCircuits, + degradedCircuits, + unknownCircuits, + affectedCircuits, totalBreakers, cooldowns: cooldowns.length, lockouts, @@ -456,7 +505,7 @@ export default function RuntimePageClient() { return feed.filter((ev) => FEED_KIND_META[ev.kind].group === feedFilter); }, [feed, feedFilter]); - const overallHealthy = counts.totalBreakers - counts.openCircuits - counts.halfCircuits; + const overallHealthy = counts.totalBreakers - counts.affectedCircuits; const overallPercent = counts.totalBreakers > 0 ? Math.round((overallHealthy / counts.totalBreakers) * 100) : 100; @@ -519,14 +568,20 @@ export default function RuntimePageClient() { label={t("kpiCircuits")} value={`${counts.openCircuits} / ${counts.totalBreakers}`} hint={ - counts.halfCircuits > 0 - ? t("hintRecovering", { count: counts.halfCircuits }) + counts.halfCircuits + counts.degradedCircuits + counts.unknownCircuits > 0 + ? t("hintRecovering", { + count: counts.halfCircuits + counts.degradedCircuits + counts.unknownCircuits, + }) : counts.openCircuits === 0 ? t("hintAllHealthy") : t("hintOpen") } tone={ - counts.openCircuits > 0 ? "#ef4444" : counts.halfCircuits > 0 ? "#eab308" : "#22c55e" + counts.openCircuits > 0 + ? "#ef4444" + : counts.halfCircuits + counts.degradedCircuits + counts.unknownCircuits > 0 + ? "#eab308" + : "#22c55e" } onClick={() => setFeedFilter("circuits")} active={feedFilter === "circuits"} @@ -561,7 +616,9 @@ export default function RuntimePageClient() { trailing={
✓ {overallHealthy} - ⚠ {counts.halfCircuits} + + ⚠ {counts.halfCircuits + counts.degradedCircuits + counts.unknownCircuits} + ⛔ {counts.openCircuits}
} @@ -585,11 +642,15 @@ export default function RuntimePageClient() { title={t("layer1Title")} description={t("layer1Desc")} badge={t("badgeAffectedOf", { - affected: counts.openCircuits + counts.halfCircuits, + affected: counts.affectedCircuits, total: counts.totalBreakers, })} badgeTone={ - counts.openCircuits > 0 ? "red" : counts.halfCircuits > 0 ? "amber" : "green" + counts.openCircuits > 0 + ? "red" + : counts.halfCircuits + counts.degradedCircuits + counts.unknownCircuits > 0 + ? "amber" + : "green" } > {breakers.length === 0 ? ( @@ -597,13 +658,14 @@ export default function RuntimePageClient() { ) : (
{breakers.map((b) => { - const tone = BREAKER_TONE[b.state]; + const state = normalizeBreakerState(b.state); + const tone = getBreakerTone(state); return (
@@ -613,7 +675,7 @@ export default function RuntimePageClient() {
- {b.state === "OPEN" + {state === "OPEN" ? `retry ${fmtMs(b.retryAfterMs)}` : `${b.failureCount} failures`}
diff --git a/tests/unit/ui/runtime-page-client.test.tsx b/tests/unit/ui/runtime-page-client.test.tsx new file mode 100644 index 00000000000..1c20d4d3ad6 --- /dev/null +++ b/tests/unit/ui/runtime-page-client.test.tsx @@ -0,0 +1,190 @@ +// @vitest-environment jsdom +import React from "react"; +import { act } from "react"; +import { createRoot } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import RuntimePageClient from "../../../src/app/(dashboard)/dashboard/runtime/RuntimePageClient"; + +function jsonResponse(data: unknown) { + return { + ok: true, + json: async () => data, + } as Response; +} + +function getRequestPath(input: RequestInfo | URL) { + return typeof input === "string" ? input : input instanceof URL ? input.pathname : input.url; +} + +const cleanupCallbacks: Array<() => void> = []; + +async function waitForText(text: string) { + const startedAt = Date.now(); + while (!document.body.textContent?.includes(text)) { + if (Date.now() - startedAt > 1000) { + throw new Error(`Timed out waiting for text: ${text}`); + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } +} + +async function renderRuntimePage() { + const container = document.createElement("div"); + document.body.appendChild(container); + const root = createRoot(container); + + await act(async () => { + root.render(); + }); + + cleanupCallbacks.push(() => { + act(() => { + root.unmount(); + }); + container.remove(); + }); +} + +vi.mock("next/link", () => ({ + default: ({ href, children, ...props }: React.AnchorHTMLAttributes) => ( + + {children} + + ), +})); + +vi.mock("@/shared/components/Card", () => ({ + default: ({ children }: { children: React.ReactNode }) =>
{children}
, +})); + +vi.mock("@/shared/components/ProviderIcon", () => ({ + default: ({ providerId }: { providerId: string }) => ( + + ), +})); + +vi.mock("next-intl", () => ({ + useTranslations: () => { + const messages: Record = { + title: "Runtime", + description: "Realtime observability", + pause: "Pause", + resume: "Resume", + refreshNow: "Refresh now", + kpiSessions: "Sessions", + kpiCircuits: "Circuits", + kpiCooldowns: "Cooldowns", + kpiLockouts: "Lockouts", + hintStickyBound: "{count} sticky-bound", + hintRecovering: "{count} recovering", + hintAllHealthy: "all healthy", + hintOpen: "open", + hintConnsCooling: "connections cooling", + hintModelsBlocked: "models blocked", + resilienceTitle: "3-Layer Resilience", + resilienceSubtitle: "Mirrors the documented resilience model", + providersHealthy: "{percent}% providers healthy", + layer: "Layer {n}", + layer1Title: "Provider Circuit Breakers", + layer1Desc: "Stop traffic to providers failing at the upstream level", + layer2Title: "Connection Cooldowns", + layer2Desc: "Skip one bad account/key", + layer3Title: "Model Lockouts", + layer3Desc: "Per-model rate-limit locks", + badgeAffectedOf: "{affected} of {total} affected", + badgeCooling: "{count} cooling", + badgeLocked: "{count} locked", + emptyCircuits: "No circuit breakers active yet", + emptyCooldowns: "No connection cooldowns active", + emptyLockouts: "No model lockouts", + feedTitle: "Live Feed", + feedSubtitle: "Last {count} events", + feedFilterAll: "All", + feedFilterCircuits: "Circuits", + feedFilterCooldowns: "Cooldowns", + feedFilterLockouts: "Lockouts", + feedFilterSessions: "Sessions", + feedFilterQuotas: "Quotas", + feedClear: "Clear", + feedEmptyWaiting: "Waiting for events...", + feedEmptyFiltered: "No events match this filter", + sessionsTitle: "Active Sessions", + sessionsSubtitle: "Sticky-bound request fingerprints", + sessionsActive: "{count} active", + sessionsEmptyTitle: "No active sessions", + sessionsEmptyHint: "Sessions appear as requests flow through the proxy", + tblSession: "Session", + tblAge: "Age", + tblIdle: "Idle", + tblReqs: "Reqs", + tblBoundTo: "Bound to", + topApiKeys: "Top API keys", + quotaMonitorsTitle: "Quota Monitors", + quotaMonitorsSubtitle: "Live quota state per account window", + openQuota: "Open Quota", + allQuotasHealthy: "All quotas healthy", + moreSuffix: "+{count} more", + }; + + return (key: string, values?: Record) => { + let message = messages[key] ?? key; + if (values) { + for (const [name, value] of Object.entries(values)) { + message = message.replace(`{${name}}`, String(value)); + } + } + return message; + }; + }, +})); + +describe("RuntimePageClient", () => { + const fetchMock = vi.fn(); + + beforeEach(() => { + ( + globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT?: boolean } + ).IS_REACT_ACT_ENVIRONMENT = true; + fetchMock.mockReset(); + vi.stubGlobal("fetch", fetchMock); + }); + + afterEach(() => { + while (cleanupCallbacks.length > 0) { + cleanupCallbacks.pop()?.(); + } + document.body.innerHTML = ""; + vi.unstubAllGlobals(); + }); + + it("renders degraded and unknown provider breaker states without crashing", async () => { + fetchMock.mockImplementation((input: RequestInfo | URL) => { + const path = getRequestPath(input); + if (path === "/api/monitoring/health") { + return Promise.resolve( + jsonResponse({ + providerBreakers: [ + { provider: "free3", state: "DEGRADED", failureCount: 7, retryAfterMs: 0 }, + { provider: "future-provider", state: "SUSPENDED", failureCount: 1, retryAfterMs: 0 }, + ], + lockouts: {}, + sessions: { activeCount: 0, stickyBoundCount: 0, byApiKey: {}, top: [] }, + quotaMonitor: { active: 0, alerting: 0, exhausted: 0, errors: 0, monitors: [] }, + }) + ); + } + if (path === "/api/providers/client") { + return Promise.resolve(jsonResponse({ connections: [] })); + } + return Promise.resolve(jsonResponse({})); + }); + + await renderRuntimePage(); + + await waitForText("free3"); + await waitForText("future-provider"); + await waitForText("DEG"); + await waitForText("UNK"); + expect(document.body.textContent).not.toContain("Internal Server Error"); + }); +}); From 2c633b7aa2338d51331784ed5c6291b6c8ed4f9f Mon Sep 17 00:00:00 2001 From: "R.D." Date: Tue, 9 Jun 2026 17:13:47 -0400 Subject: [PATCH 2/2] Address runtime review feedback --- .../dashboard/runtime/RuntimePageClient.tsx | 37 ++++++++++++------- tests/unit/ui/runtime-page-client.test.tsx | 8 ++-- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/app/(dashboard)/dashboard/runtime/RuntimePageClient.tsx b/src/app/(dashboard)/dashboard/runtime/RuntimePageClient.tsx index bff773f6ab4..7a7176f6aa5 100644 --- a/src/app/(dashboard)/dashboard/runtime/RuntimePageClient.tsx +++ b/src/app/(dashboard)/dashboard/runtime/RuntimePageClient.tsx @@ -84,6 +84,7 @@ type Connection = { type FeedEventKind = | "circuit-opened" + | "circuit-degraded" | "circuit-recovered" | "circuit-closed" | "cooldown-added" @@ -156,6 +157,7 @@ const FALLBACK_BREAKER_TONE: BreakerTone = { const FEED_KIND_META: Record = { "circuit-opened": { icon: "block", color: "#ef4444", group: "circuits" }, + "circuit-degraded": { icon: "warning", color: "#f97316", group: "circuits" }, "circuit-recovered": { icon: "sync", color: "#eab308", group: "circuits" }, "circuit-closed": { icon: "check_circle", color: "#22c55e", group: "circuits" }, "cooldown-added": { icon: "ac_unit", color: "#3b82f6", group: "cooldowns" }, @@ -210,8 +212,8 @@ function normalizeBreakerState(state: string | null | undefined): string { .toUpperCase(); } -function getBreakerTone(state: string | null | undefined): BreakerTone { - return BREAKER_TONE[normalizeBreakerState(state)] || FALLBACK_BREAKER_TONE; +function getBreakerTone(normalizedState: string): BreakerTone { + return BREAKER_TONE[normalizedState] || FALLBACK_BREAKER_TONE; } function pushFeed(prev: FeedEvent[], events: FeedEvent[]): FeedEvent[] { @@ -259,7 +261,7 @@ function diffSnapshots( out.push({ id: `cb-deg-${provider}-${nowTs}`, ts: nowTs, - kind: "circuit-recovered", + kind: "circuit-degraded", title: `${provider} DEGRADED`, detail: `${nextB.failureCount} failures · degraded but serving`, }); @@ -466,17 +468,24 @@ export default function RuntimePageClient() { ); const counts = useMemo(() => { - const knownBreakerStates = new Set(["CLOSED", "OPEN", "HALF_OPEN", "DEGRADED"]); - const openCircuits = breakers.filter((b) => normalizeBreakerState(b.state) === "OPEN").length; - const halfCircuits = breakers.filter( - (b) => normalizeBreakerState(b.state) === "HALF_OPEN" - ).length; - const degradedCircuits = breakers.filter( - (b) => normalizeBreakerState(b.state) === "DEGRADED" - ).length; - const unknownCircuits = breakers.filter( - (b) => !knownBreakerStates.has(normalizeBreakerState(b.state)) - ).length; + let openCircuits = 0; + let halfCircuits = 0; + let degradedCircuits = 0; + let unknownCircuits = 0; + + for (const breaker of breakers) { + const state = normalizeBreakerState(breaker.state); + if (state === "OPEN") { + openCircuits++; + } else if (state === "HALF_OPEN") { + halfCircuits++; + } else if (state === "DEGRADED") { + degradedCircuits++; + } else if (state !== "CLOSED") { + unknownCircuits++; + } + } + const totalBreakers = breakers.length; const affectedCircuits = openCircuits + halfCircuits + degradedCircuits + unknownCircuits; const sessions = health?.sessions?.activeCount ?? 0; diff --git a/tests/unit/ui/runtime-page-client.test.tsx b/tests/unit/ui/runtime-page-client.test.tsx index 1c20d4d3ad6..e3c068a44b4 100644 --- a/tests/unit/ui/runtime-page-client.test.tsx +++ b/tests/unit/ui/runtime-page-client.test.tsx @@ -18,13 +18,15 @@ function getRequestPath(input: RequestInfo | URL) { const cleanupCallbacks: Array<() => void> = []; -async function waitForText(text: string) { +async function waitForText(text: string, timeoutMs = 3000) { const startedAt = Date.now(); while (!document.body.textContent?.includes(text)) { - if (Date.now() - startedAt > 1000) { + if (Date.now() - startedAt > timeoutMs) { throw new Error(`Timed out waiting for text: ${text}`); } - await new Promise((resolve) => setTimeout(resolve, 10)); + await act(async () => { + await new Promise((resolve) => setTimeout(resolve, 20)); + }); } }