diff --git a/scripts/check-kpi.mjs b/scripts/check-kpi.mjs index 6e50368ab..c9b480879 100755 --- a/scripts/check-kpi.mjs +++ b/scripts/check-kpi.mjs @@ -4,11 +4,14 @@ import { hasDuplicateJsonObjectKeys } from "./normalize-commercial-readiness-evi const fs = await import("node:fs/promises"); const { existsSync } = await import("node:fs"); +const calendarDatePrefixPattern = /^(\d{4}-\d{2}-\d{2})(?:$|[T ])/; const inputPath = process.argv[2] ?? "exchange-30d.ndjson"; const failureThreshold = Number(process.argv[3] ?? "0.02"); const p95Threshold = Number(process.argv[4] ?? "300"); -const requireWindowDays = Number(process.env.NOEMA_KPI_REQUIRE_WINDOW_DAYS); +const requireWindowDaysRaw = process.env.NOEMA_KPI_REQUIRE_WINDOW_DAYS; +const hasWindowRequirement = requireWindowDaysRaw !== undefined; +const requireWindowDays = hasWindowRequirement ? Number(requireWindowDaysRaw) : Number.NaN; if (!inputPath) { console.error("Usage: node scripts/check-kpi.mjs [wrangler-tail-ndjson] [failureThreshold] [p95ThresholdMs]"); @@ -20,13 +23,18 @@ if (!existsSync(inputPath)) { process.exit(1); } -if (!Number.isFinite(failureThreshold) || !Number.isFinite(p95Threshold)) { - console.error("Invalid threshold values."); +if (!Number.isFinite(failureThreshold) || failureThreshold < 0 || failureThreshold > 1) { + console.error("KPI failure threshold must be between 0 and 1."); process.exit(1); } -if (Number.isFinite(requireWindowDays) && requireWindowDays <= 0) { - console.error("NOEMA_KPI_REQUIRE_WINDOW_DAYS must be a positive number."); +if (!Number.isFinite(p95Threshold) || p95Threshold < 0) { + console.error("KPI p95 threshold must be non-negative and finite."); + process.exit(1); +} + +if (hasWindowRequirement && (!Number.isFinite(requireWindowDays) || requireWindowDays <= 0)) { + console.error("NOEMA_KPI_REQUIRE_WINDOW_DAYS must be a positive finite number."); process.exit(1); } @@ -71,19 +79,47 @@ for (const line of lines) { } const route = resolveRoute(record); - const event = record.event || "http_request"; - if (route !== "/exchange" || event !== "http_request") continue; + if (route !== "/exchange") continue; + if (typeof record.event !== "string" || record.event.trim().length === 0) { + console.error("KPI exchange record is missing canonical http_request event identity."); + process.exit(1); + } + if (record.event !== "http_request") continue; exchanges += 1; - const status = Number(record.status_code || record.status || record.response?.status); - if (Number.isNaN(status) || status >= 400) failures += 1; + const status = record.status_code ?? record.status ?? record.response?.status; + if (typeof status !== "number" || !Number.isInteger(status) || status < 100 || status > 599) { + console.error("Invalid exchange HTTP status in KPI log; expected an integer from 100 through 599."); + process.exit(1); + } + if (status >= 400) failures += 1; - const latency = Number(record.latency_ms || record.latencyMs || record.duration_ms); - if (!Number.isNaN(latency)) latencies.push(latency); + const latency = record.latency_ms ?? record.latencyMs ?? record.duration_ms; + if (latency === undefined || latency === null) { + console.error("KPI exchange latency is required for every canonical http_request event."); + process.exit(1); + } + if (typeof latency !== "number" || !Number.isFinite(latency) || latency < 0) { + console.error("Invalid exchange latency in KPI log; expected a finite non-negative number."); + process.exit(1); + } + latencies.push(latency); const ts = resolveTimestampMs(record); + if (Number.isNaN(ts)) { + console.error("Invalid exchange timestamp in KPI log; refusing normalized calendar evidence."); + process.exit(1); + } if (ts != null) { + if (!Number.isFinite(ts) || ts < 0) { + console.error("Invalid exchange timestamp in KPI log; expected a finite non-negative timestamp."); + process.exit(1); + } + if (ts > Date.now()) { + console.error("Invalid exchange timestamp in KPI log; timestamp cannot be in the future."); + process.exit(1); + } exchangesWithTimestamp += 1; minTimestampMs = Math.min(minTimestampMs, ts); maxTimestampMs = Math.max(maxTimestampMs, ts); @@ -99,7 +135,6 @@ latencies.sort((a, b) => a - b); const p95Index = Math.max(0, Math.ceil((0.95 * latencies.length) - 1)); const p95 = latencies.length ? latencies[p95Index] : null; const failureRate = failures / exchanges; -const hasWindowRequirement = Number.isFinite(requireWindowDays); const requiredWindowMs = hasWindowRequirement ? requireWindowDays * 24 * 60 * 60 * 1000 : null; const exchangeWindowMs = Number.isFinite(minTimestampMs) && Number.isFinite(maxTimestampMs) ? maxTimestampMs - minTimestampMs @@ -172,6 +207,14 @@ function resolveTimestampMs(record) { return candidate; } if (typeof candidate === "string") { + const calendarMatch = calendarDatePrefixPattern.exec(candidate); + if (calendarMatch) { + const datePart = calendarMatch[1]; + const calendarDate = new Date(`${datePart}T00:00:00.000Z`); + if (Number.isNaN(calendarDate.getTime()) || calendarDate.toISOString().slice(0, 10) !== datePart) { + return Number.NaN; + } + } const parsed = Date.parse(candidate); if (!Number.isNaN(parsed)) return parsed; const numeric = Number(candidate); diff --git a/test/check-kpi-event-authority.test.ts b/test/check-kpi-event-authority.test.ts new file mode 100644 index 000000000..260128a9b --- /dev/null +++ b/test/check-kpi-event-authority.test.ts @@ -0,0 +1,93 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; +import { describe, expect, it } from "vitest"; + +function runCheckKpi(logPath: string) { + const env = { ...process.env }; + delete env.NOEMA_KPI_REQUIRE_WINDOW_DAYS; + return spawnSync(process.execPath, ["scripts/check-kpi.mjs", logPath, "0.02", "300"], { + cwd: process.cwd(), + encoding: "utf8", + env, + }); +} + +function expectInvalidEventIdentity(record: Record) { + const dir = mkdtempSync(join(tmpdir(), "noema-check-kpi-event-")); + try { + const logPath = join(dir, "exchange-30d.ndjson"); + writeFileSync(logPath, `${JSON.stringify(record)}\n`); + + const result = runCheckKpi(logPath); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("exchange record is missing canonical http_request event identity"); + expect(result.stdout).toBe(""); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +describe("KPI event authority", () => { + it("rejects an exchange-shaped record that omits the canonical http_request event identity", () => { + expectInvalidEventIdentity({ + route: "/exchange", + status_code: 200, + latency_ms: 10, + }); + }); + + it("rejects an empty event identity instead of treating it as a canonical http_request", () => { + expectInvalidEventIdentity({ + event: "", + route: "/exchange", + status_code: 200, + latency_ms: 10, + }); + }); + + it("rejects a whitespace-only event identity instead of excluding an exchange observation", () => { + expectInvalidEventIdentity({ + event: " \t", + route: "/exchange", + status_code: 500, + latency_ms: 10, + }); + }); + + it("continues to ignore explicitly non-http_request events on the exchange route", () => { + const dir = mkdtempSync(join(tmpdir(), "noema-check-kpi-event-ignore-")); + try { + const logPath = join(dir, "exchange-30d.ndjson"); + writeFileSync(logPath, [ + JSON.stringify({ + event: "workflow_trust", + route: "/exchange", + status_code: 403, + latency_ms: 1, + }), + JSON.stringify({ + event: "http_request", + route: "/exchange", + status_code: 200, + latency_ms: 10, + }), + "", + ].join("\n")); + + const result = runCheckKpi(logPath); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(""); + expect(JSON.parse(result.stdout)).toMatchObject({ + exchange_requests: 1, + exchange_failures: 0, + pass: true, + }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/test/check-kpi-input-integrity.test.ts b/test/check-kpi-input-integrity.test.ts index a57e4a0bc..facb6c2d9 100644 --- a/test/check-kpi-input-integrity.test.ts +++ b/test/check-kpi-input-integrity.test.ts @@ -4,9 +4,13 @@ import { join } from "node:path"; import { spawnSync } from "node:child_process"; import { describe, expect, it } from "vitest"; -function runCheckKpi(logPath: string) { +function runCheckKpi(logPath: string, requireWindowDays?: string) { const env = { ...process.env }; - delete env.NOEMA_KPI_REQUIRE_WINDOW_DAYS; + if (requireWindowDays === undefined) { + delete env.NOEMA_KPI_REQUIRE_WINDOW_DAYS; + } else { + env.NOEMA_KPI_REQUIRE_WINDOW_DAYS = requireWindowDays; + } return spawnSync(process.execPath, ["scripts/check-kpi.mjs", logPath, "0.02", "300"], { cwd: process.cwd(), encoding: "utf8", @@ -55,6 +59,106 @@ describe("KPI threshold input integrity", () => { } }); + it("rejects an impossible calendar timestamp instead of normalizing it into window evidence", () => { + const dir = mkdtempSync(join(tmpdir(), "noema-check-kpi-time-")); + try { + const logPath = join(dir, "exchange-30d.ndjson"); + writeFileSync(logPath, [ + JSON.stringify({ + event: "http_request", + route: "/exchange", + status_code: 200, + latency_ms: 120, + timestamp: "2026-02-01T00:00:00.000Z", + }), + JSON.stringify({ + event: "http_request", + route: "/exchange", + status_code: 200, + latency_ms: 140, + timestamp: "2026-02-30", + }), + "", + ].join("\n")); + + const result = runCheckKpi(logPath, "28"); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("Invalid exchange timestamp in KPI log"); + expect(result.stdout).toBe(""); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects negative latency instead of letting corrupt evidence lower p95", () => { + const dir = mkdtempSync(join(tmpdir(), "noema-check-kpi-latency-")); + try { + const logPath = join(dir, "exchange-30d.ndjson"); + writeFileSync(logPath, `${JSON.stringify({ + event: "http_request", + route: "/exchange", + status_code: 200, + latency_ms: -1, + })}\n`); + + const result = runCheckKpi(logPath); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("Invalid exchange latency in KPI log"); + expect(result.stdout).toBe(""); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects string latency instead of coercing untyped evidence into p95", () => { + const dir = mkdtempSync(join(tmpdir(), "noema-check-kpi-latency-type-")); + try { + const logPath = join(dir, "exchange-30d.ndjson"); + writeFileSync(logPath, `${JSON.stringify({ + event: "http_request", + route: "/exchange", + status_code: 200, + latency_ms: "0", + })}\n`); + + const result = runCheckKpi(logPath); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("Invalid exchange latency in KPI log"); + expect(result.stdout).toBe(""); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("preserves a zero-millisecond latency as a real KPI sample", () => { + const dir = mkdtempSync(join(tmpdir(), "noema-check-kpi-zero-latency-")); + try { + const logPath = join(dir, "exchange-30d.ndjson"); + writeFileSync(logPath, `${JSON.stringify({ + event: "http_request", + route: "/exchange", + status_code: 200, + latency_ms: 0, + })}\n`); + + const result = runCheckKpi(logPath); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(""); + expect(JSON.parse(result.stdout)).toMatchObject({ + exchange_requests: 1, + exchange_failures: 0, + exchange_p95_latency_ms: 0, + pass: true, + }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it("preserves valid threshold semantics and ignores non-JSON lines", () => { const dir = mkdtempSync(join(tmpdir(), "noema-check-kpi-")); try { diff --git a/test/check-kpi-latency-completeness.test.ts b/test/check-kpi-latency-completeness.test.ts new file mode 100644 index 000000000..2772f24c8 --- /dev/null +++ b/test/check-kpi-latency-completeness.test.ts @@ -0,0 +1,46 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; +import { describe, expect, it } from "vitest"; + +function runCheckKpi(logPath: string) { + const env = { ...process.env }; + delete env.NOEMA_KPI_REQUIRE_WINDOW_DAYS; + return spawnSync(process.execPath, ["scripts/check-kpi.mjs", logPath, "0.02", "300"], { + cwd: process.cwd(), + encoding: "utf8", + env, + }); +} + +describe("KPI latency evidence completeness", () => { + it("rejects a canonical exchange event that omits latency instead of shrinking the p95 sample", () => { + const dir = mkdtempSync(join(tmpdir(), "noema-check-kpi-latency-")); + try { + const logPath = join(dir, "exchange-30d.ndjson"); + writeFileSync(logPath, [ + JSON.stringify({ + event: "http_request", + route: "/exchange", + status_code: 200, + latency_ms: 10, + }), + JSON.stringify({ + event: "http_request", + route: "/exchange", + status_code: 200, + }), + "", + ].join("\n")); + + const result = runCheckKpi(logPath); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("exchange latency is required"); + expect(result.stdout).toBe(""); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/test/check-kpi-status-integrity.test.ts b/test/check-kpi-status-integrity.test.ts new file mode 100644 index 000000000..beaa812d8 --- /dev/null +++ b/test/check-kpi-status-integrity.test.ts @@ -0,0 +1,86 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; +import { describe, expect, it } from "vitest"; + +function runCheckKpi(logPath: string) { + const env = { ...process.env }; + delete env.NOEMA_KPI_REQUIRE_WINDOW_DAYS; + return spawnSync(process.execPath, ["scripts/check-kpi.mjs", logPath, "0.02", "300"], { + cwd: process.cwd(), + encoding: "utf8", + env, + }); +} + +describe("KPI HTTP status evidence integrity", () => { + for (const statusCode of [99, 200.5, -1, 600]) { + it(`rejects impossible HTTP status ${statusCode} instead of treating it as KPI evidence`, () => { + const dir = mkdtempSync(join(tmpdir(), "noema-check-kpi-status-")); + try { + const logPath = join(dir, "exchange-30d.ndjson"); + writeFileSync(logPath, `${JSON.stringify({ + event: "http_request", + route: "/exchange", + status_code: statusCode, + latency_ms: 10, + })}\n`); + + const result = runCheckKpi(logPath); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("Invalid exchange HTTP status in KPI log"); + expect(result.stdout).toBe(""); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + } + + it("rejects string HTTP status instead of coercing untyped evidence into success", () => { + const dir = mkdtempSync(join(tmpdir(), "noema-check-kpi-status-type-")); + try { + const logPath = join(dir, "exchange-30d.ndjson"); + writeFileSync(logPath, `${JSON.stringify({ + event: "http_request", + route: "/exchange", + status_code: "200", + latency_ms: 10, + })}\n`); + + const result = runCheckKpi(logPath); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("Invalid exchange HTTP status in KPI log"); + expect(result.stdout).toBe(""); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("accepts a canonical successful HTTP status", () => { + const dir = mkdtempSync(join(tmpdir(), "noema-check-kpi-status-")); + try { + const logPath = join(dir, "exchange-30d.ndjson"); + writeFileSync(logPath, `${JSON.stringify({ + event: "http_request", + route: "/exchange", + status_code: 204, + latency_ms: 10, + })}\n`); + + const result = runCheckKpi(logPath); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(""); + expect(JSON.parse(result.stdout)).toMatchObject({ + exchange_requests: 1, + exchange_failures: 0, + pass: true, + }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/test/check-kpi-threshold-domain.test.ts b/test/check-kpi-threshold-domain.test.ts new file mode 100644 index 000000000..71c756d5c --- /dev/null +++ b/test/check-kpi-threshold-domain.test.ts @@ -0,0 +1,63 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; +import { describe, expect, it } from "vitest"; + +function runCheckKpi(logPath: string, failureThreshold: string, p95Threshold: string) { + const env = { ...process.env }; + delete env.NOEMA_KPI_REQUIRE_WINDOW_DAYS; + return spawnSync(process.execPath, ["scripts/check-kpi.mjs", logPath, failureThreshold, p95Threshold], { + cwd: process.cwd(), + encoding: "utf8", + env, + }); +} + +function withHealthyExchangeLog(run: (logPath: string) => void) { + const dir = mkdtempSync(join(tmpdir(), "noema-check-kpi-threshold-")); + try { + const logPath = join(dir, "exchange-30d.ndjson"); + writeFileSync(logPath, `${JSON.stringify({ + event: "http_request", + route: "/exchange", + status_code: 200, + latency_ms: 10, + })}\n`); + run(logPath); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +describe("KPI threshold authority domain", () => { + it("rejects a failure-rate threshold above the mathematical maximum instead of manufacturing a pass", () => { + withHealthyExchangeLog((logPath) => { + const result = runCheckKpi(logPath, "1.01", "300"); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("failure threshold must be between 0 and 1"); + expect(result.stdout).toBe(""); + }); + }); + + it("rejects a negative failure-rate threshold as impossible threshold authority", () => { + withHealthyExchangeLog((logPath) => { + const result = runCheckKpi(logPath, "-0.01", "300"); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("failure threshold must be between 0 and 1"); + expect(result.stdout).toBe(""); + }); + }); + + it("rejects a negative latency threshold instead of accepting impossible threshold authority", () => { + withHealthyExchangeLog((logPath) => { + const result = runCheckKpi(logPath, "0.02", "-1"); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("p95 threshold must be non-negative"); + expect(result.stdout).toBe(""); + }); + }); +}); diff --git a/test/check-kpi-window-authority.test.ts b/test/check-kpi-window-authority.test.ts new file mode 100644 index 000000000..778b2f781 --- /dev/null +++ b/test/check-kpi-window-authority.test.ts @@ -0,0 +1,77 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; +import { describe, expect, it } from "vitest"; + +function runCheckKpi(logPath: string, requireWindowDays?: string) { + const env = { ...process.env }; + if (requireWindowDays === undefined) { + delete env.NOEMA_KPI_REQUIRE_WINDOW_DAYS; + } else { + env.NOEMA_KPI_REQUIRE_WINDOW_DAYS = requireWindowDays; + } + return spawnSync(process.execPath, ["scripts/check-kpi.mjs", logPath, "0.02", "300"], { + cwd: process.cwd(), + encoding: "utf8", + env, + }); +} + +describe("KPI window authority", () => { + it("rejects future exchange timestamps instead of letting them inflate the observed window", () => { + const dir = mkdtempSync(join(tmpdir(), "noema-check-kpi-future-")); + try { + const logPath = join(dir, "exchange-30d.ndjson"); + const now = Date.now(); + writeFileSync(logPath, [ + JSON.stringify({ + event: "http_request", + route: "/exchange", + status_code: 200, + latency_ms: 100, + timestamp: new Date(now - (31 * 86400000)).toISOString(), + }), + JSON.stringify({ + event: "http_request", + route: "/exchange", + status_code: 200, + latency_ms: 110, + timestamp: new Date(now + 3600000).toISOString(), + }), + "", + ].join("\n")); + + const result = runCheckKpi(logPath, "30"); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("timestamp cannot be in the future"); + expect(result.stdout).toBe(""); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + for (const configuredWindow of ["Infinity", "NaN"]) { + it(`rejects explicit non-finite window requirement ${configuredWindow}`, () => { + const dir = mkdtempSync(join(tmpdir(), "noema-check-kpi-window-config-")); + try { + const logPath = join(dir, "exchange-30d.ndjson"); + writeFileSync(logPath, `${JSON.stringify({ + event: "http_request", + route: "/exchange", + status_code: 200, + latency_ms: 100, + })}\n`); + + const result = runCheckKpi(logPath, configuredWindow); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("NOEMA_KPI_REQUIRE_WINDOW_DAYS must be a positive finite number"); + expect(result.stdout).toBe(""); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + } +});