From a9039741c8633e2121f95f36040b5f37a82f2399 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 21:33:06 -0700 Subject: [PATCH 1/4] test(kpi): reject mismatched provenance log path --- test/kpi-provenance-log-path-identity.test.ts | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 test/kpi-provenance-log-path-identity.test.ts diff --git a/test/kpi-provenance-log-path-identity.test.ts b/test/kpi-provenance-log-path-identity.test.ts new file mode 100644 index 000000000..c57778ecf --- /dev/null +++ b/test/kpi-provenance-log-path-identity.test.ts @@ -0,0 +1,77 @@ +import { createHash } from "node:crypto"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; + +import { describe, expect, it } from "vitest"; + +function writeThirtyDayLog(path: string) { + const records = [ + { + event: "http_request", + route: "/exchange", + status_code: 200, + latency_ms: 120, + timestamp: "2026-06-01T00:00:00.000Z", + }, + { + event: "http_request", + route: "/exchange", + status_code: 200, + latency_ms: 150, + timestamp: "2026-07-01T03:00:00.000Z", + }, + ]; + const bytes = Buffer.from(`${records.map((record) => JSON.stringify(record)).join("\n")}\n`); + writeFileSync(path, bytes); + return { + logSha256: createHash("sha256").update(bytes).digest("hex"), + logBytes: bytes.byteLength, + }; +} + +describe("strict KPI provenance log path identity", () => { + it("rejects provenance that claims a different logPath from the bytes being verified", () => { + const directory = mkdtempSync(join(tmpdir(), "noema-kpi-log-path-identity-")); + try { + const logPath = join(directory, "exchange-30d.ndjson"); + const provenancePath = `${logPath}.provenance.json`; + const evidencePath = join(directory, "evidence.json"); + const identity = writeThirtyDayLog(logPath); + writeFileSync( + provenancePath, + `${JSON.stringify({ + sourceKind: "production", + sourceId: "cloudflare-logpush:noema-production", + sourceMethod: "log-url", + logPath: join(directory, "different-production-log.ndjson"), + records: 2, + collectedAt: "2026-07-02T00:00:00.000Z", + ...identity, + }, null, 2)}\n`, + ); + + const result = spawnSync( + process.execPath, + ["scripts/kpi-gate.mjs", "--strict", "--require-window-days", "30", logPath], + { + cwd: process.cwd(), + encoding: "utf8", + env: { + ...process.env, + NOEMA_KPI_PROVENANCE_PATH: provenancePath, + NOEMA_KPI_EVIDENCE_PATH: evidencePath, + }, + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toContain( + "KPI provenance logPath must exactly identify the production log being verified", + ); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); +}); From 67866cc4301e5c1c6593bd581decfe89013812c6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 21:34:51 -0700 Subject: [PATCH 2/4] fix(kpi): bind provenance log path identity --- scripts/kpi-gate.mjs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/kpi-gate.mjs b/scripts/kpi-gate.mjs index 983858e96..2c2127a13 100644 --- a/scripts/kpi-gate.mjs +++ b/scripts/kpi-gate.mjs @@ -340,6 +340,7 @@ async function loadProductionProvenance(path, expectedLogPath) { const sourceKind = String(parsed.sourceKind ?? ""); const sourceId = typeof parsed.sourceId === "string" ? parsed.sourceId : ""; const sourceMethod = typeof parsed.sourceMethod === "string" ? parsed.sourceMethod : ""; + const provenanceLogPath = typeof parsed.logPath === "string" ? parsed.logPath : ""; const collectedAt = typeof parsed.collectedAt === "string" ? parsed.collectedAt : ""; const records = parsed.records; const logSha256 = typeof parsed.logSha256 === "string" ? parsed.logSha256 : ""; @@ -375,6 +376,12 @@ async function loadProductionProvenance(path, expectedLogPath) { reason: "KPI provenance sourceMethod must be one of the reviewed collection methods: log-url or tail-command.", }; } + if (provenanceLogPath !== expectedLogPath) { + return { + pass: false, + reason: "KPI provenance logPath must exactly identify the production log being verified.", + }; + } const collectedAtMs = Date.parse(collectedAt); if ( !collectedAt @@ -426,7 +433,7 @@ async function loadProductionProvenance(path, expectedLogPath) { sourceId, collectedAt, records, - logPath: parsed.logPath ?? null, + logPath: provenanceLogPath, sourceMethod, logSha256, logBytes, From b7dfe2df2f75acd41c8813b16f1babe100404050 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 21:38:32 -0700 Subject: [PATCH 3/4] fix(kpi): preserve optional logPath while binding present identity --- scripts/kpi-gate.mjs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/kpi-gate.mjs b/scripts/kpi-gate.mjs index 2c2127a13..90a82779b 100644 --- a/scripts/kpi-gate.mjs +++ b/scripts/kpi-gate.mjs @@ -340,7 +340,7 @@ async function loadProductionProvenance(path, expectedLogPath) { const sourceKind = String(parsed.sourceKind ?? ""); const sourceId = typeof parsed.sourceId === "string" ? parsed.sourceId : ""; const sourceMethod = typeof parsed.sourceMethod === "string" ? parsed.sourceMethod : ""; - const provenanceLogPath = typeof parsed.logPath === "string" ? parsed.logPath : ""; + const provenanceLogPath = parsed.logPath ?? null; const collectedAt = typeof parsed.collectedAt === "string" ? parsed.collectedAt : ""; const records = parsed.records; const logSha256 = typeof parsed.logSha256 === "string" ? parsed.logSha256 : ""; @@ -376,10 +376,10 @@ async function loadProductionProvenance(path, expectedLogPath) { reason: "KPI provenance sourceMethod must be one of the reviewed collection methods: log-url or tail-command.", }; } - if (provenanceLogPath !== expectedLogPath) { + if (provenanceLogPath !== null && provenanceLogPath !== expectedLogPath) { return { pass: false, - reason: "KPI provenance logPath must exactly identify the production log being verified.", + reason: "KPI provenance logPath must exactly identify the production log being verified when present.", }; } const collectedAtMs = Date.parse(collectedAt); From c484a13fa33f83ee2d32ada6d721bf594a02a579 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 21:40:44 -0700 Subject: [PATCH 4/4] test(kpi): cover provenance log path rejection in-process --- test/kpi-provenance-log-path-identity.test.ts | 88 +++++++++++++++---- 1 file changed, 69 insertions(+), 19 deletions(-) diff --git a/test/kpi-provenance-log-path-identity.test.ts b/test/kpi-provenance-log-path-identity.test.ts index c57778ecf..d3f0252d5 100644 --- a/test/kpi-provenance-log-path-identity.test.ts +++ b/test/kpi-provenance-log-path-identity.test.ts @@ -1,10 +1,26 @@ import { createHash } from "node:crypto"; import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { spawnSync } from "node:child_process"; +import { join, resolve } from "node:path"; -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +const originalEnvironment = { ...process.env }; +const originalArgv = [...process.argv]; + +class ExitSignal extends Error { + constructor(readonly code: number) { + super(`EXIT:${code}`); + } +} + +function restoreProcessState() { + for (const key of Object.keys(process.env)) { + if (!(key in originalEnvironment)) delete process.env[key]; + } + Object.assign(process.env, originalEnvironment); + process.argv.splice(0, process.argv.length, ...originalArgv); +} function writeThirtyDayLog(path: string) { const records = [ @@ -31,8 +47,54 @@ function writeThirtyDayLog(path: string) { }; } +async function runStrictGate(logPath: string, provenancePath: string, evidencePath: string) { + restoreProcessState(); + Object.assign(process.env, { + NOEMA_KPI_PROVENANCE_PATH: provenancePath, + NOEMA_KPI_EVIDENCE_PATH: evidencePath, + NOEMA_KPI_STRICT: "1", + NOEMA_KPI_REQUIRE_WINDOW_DAYS: "30", + }); + process.argv.splice( + 0, + process.argv.length, + originalArgv[0] ?? process.execPath, + resolve(process.cwd(), "scripts/kpi-gate.mjs"), + logPath, + ); + + vi.resetModules(); + const logs: string[] = []; + const logSpy = vi.spyOn(console, "log").mockImplementation((...values: unknown[]) => { + logs.push(values.map(String).join(" ")); + }); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new ExitSignal(code ?? 0); + }) as never); + + let exitCode: number | null = null; + try { + await import("../scripts/kpi-gate.mjs"); + } catch (error) { + if (error instanceof ExitSignal) exitCode = error.code; + else throw error; + } finally { + exitSpy.mockRestore(); + logSpy.mockRestore(); + errorSpy.mockRestore(); + restoreProcessState(); + } + return { exitCode, logs }; +} + +afterEach(() => { + vi.restoreAllMocks(); + restoreProcessState(); +}); + describe("strict KPI provenance log path identity", () => { - it("rejects provenance that claims a different logPath from the bytes being verified", () => { + it("rejects provenance that claims a different logPath from the bytes being verified", async () => { const directory = mkdtempSync(join(tmpdir(), "noema-kpi-log-path-identity-")); try { const logPath = join(directory, "exchange-30d.ndjson"); @@ -52,22 +114,10 @@ describe("strict KPI provenance log path identity", () => { }, null, 2)}\n`, ); - const result = spawnSync( - process.execPath, - ["scripts/kpi-gate.mjs", "--strict", "--require-window-days", "30", logPath], - { - cwd: process.cwd(), - encoding: "utf8", - env: { - ...process.env, - NOEMA_KPI_PROVENANCE_PATH: provenancePath, - NOEMA_KPI_EVIDENCE_PATH: evidencePath, - }, - }, - ); + const result = await runStrictGate(logPath, provenancePath, evidencePath); - expect(result.status).toBe(1); - expect(result.stdout).toContain( + expect(result.exitCode).toBe(1); + expect(result.logs.join("\n")).toContain( "KPI provenance logPath must exactly identify the production log being verified", ); } finally {