From a05ae76ada81db78278309aedbc2bdf09101fa41 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 15 Aug 2026 06:38:04 +0900 Subject: [PATCH 1/4] fix(kpi): restack provenance JSON byte integrity on 26de1c main --- scripts/kpi-gate.mjs | 22 ++++- test/kpi-provenance-json-integrity.test.ts | 109 +++++++++++++++++++++ 2 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 test/kpi-provenance-json-integrity.test.ts diff --git a/scripts/kpi-gate.mjs b/scripts/kpi-gate.mjs index 12363af16..2ce57ef36 100644 --- a/scripts/kpi-gate.mjs +++ b/scripts/kpi-gate.mjs @@ -7,7 +7,9 @@ import { join } from "node:path"; import { spawnSync } from "node:child_process"; import { hasUnsafeSourceId } from "./lib/source-id.mjs"; import { createKpiChildEnvironment } from "./lib/kpi-child-environment.mjs"; +import { hasDuplicateJsonObjectKeys } from "./normalize-commercial-readiness-evidence.mjs"; +const fatalUtf8Decoder = new TextDecoder("utf-8", { fatal: true }); const parsedArgs = parseArgs(process.argv.slice(2)); const logPath = parsedArgs.positionals[0] ?? process.env.NOEMA_KPI_LOG_PATH ?? "exchange-30d.ndjson"; const failThreshold = parsedArgs.positionals[1] ?? process.env.NOEMA_KPI_FAILURE_THRESHOLD ?? "0.02"; @@ -233,9 +235,25 @@ async function loadProductionProvenance(path, expectedLogPath) { }; } + let provenanceText; + try { + provenanceText = fatalUtf8Decoder.decode(await readFile(path)); + } catch { + return { + pass: false, + reason: `KPI provenance file is not valid UTF-8: ${path}.`, + }; + } + let parsed; try { - parsed = JSON.parse(await readFile(path, "utf8")); + if (hasDuplicateJsonObjectKeys(provenanceText)) { + return { + pass: false, + reason: `KPI provenance file contains duplicate decoded JSON object keys: ${path}.`, + }; + } + parsed = JSON.parse(provenanceText); } catch { return { pass: false, @@ -461,4 +479,4 @@ function parseArgs(args) { result.positionals.push(arg); } return result; -} +} \ No newline at end of file diff --git a/test/kpi-provenance-json-integrity.test.ts b/test/kpi-provenance-json-integrity.test.ts new file mode 100644 index 000000000..c8a072efd --- /dev/null +++ b/test/kpi-provenance-json-integrity.test.ts @@ -0,0 +1,109 @@ +import { createHash } from "node:crypto"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; +import { describe, expect, it } from "vitest"; + +function writeThirtyDayExchangeLog(path: string) { + const records = [ + { + event: "http_request", + route: "/exchange", + status_code: 200, + latency_ms: 120, + timestamp: "2026-06-01T00:00:00.000Z", + }, + { + event: "http_request", + route: "/exchange", + status_code: 200, + latency_ms: 150, + timestamp: "2026-07-01T03:00:00.000Z", + }, + ]; + writeFileSync(path, `${records.map((record) => JSON.stringify(record)).join("\n")}\n`); +} + +function logIdentity(path: string) { + const bytes = readFileSync(path); + return { + logSha256: createHash("sha256").update(bytes).digest("hex"), + logBytes: bytes.byteLength, + }; +} + +function runStrictGate(logPath: string, provenancePath: string, evidencePath: string) { + return spawnSync(process.execPath, ["scripts/kpi-gate.mjs", logPath], { + cwd: process.cwd(), + encoding: "utf8", + env: { + ...process.env, + NOEMA_KPI_STRICT: "1", + NOEMA_KPI_REQUIRE_WINDOW_DAYS: "30", + NOEMA_KPI_PROVENANCE_PATH: provenancePath, + NOEMA_KPI_EVIDENCE_PATH: evidencePath, + }, + }); +} + +function validProvenanceJson(logPath: string) { + const identity = logIdentity(logPath); + return JSON.stringify({ + sourceKind: "production", + sourceId: "cloudflare-logpush:noema-production", + sourceMethod: "log-url", + logPath, + records: 2, + collectedAt: "2026-07-02T00:00:00.000Z", + ...identity, + }); +} + +describe("strict KPI provenance JSON integrity", () => { + it("rejects duplicate decoded provenance keys before last-key-wins parsing", () => { + const dir = mkdtempSync(join(tmpdir(), "noema-kpi-provenance-integrity-")); + try { + const logPath = join(dir, "exchange-30d.ndjson"); + const provenancePath = join(dir, "exchange-30d.ndjson.provenance.json"); + const evidencePath = join(dir, "evidence.json"); + writeThirtyDayExchangeLog(logPath); + const { logSha256, logBytes } = logIdentity(logPath); + writeFileSync( + provenancePath, + `{"sourceKind":"staging","sourceK\\u0069nd":"production","sourceId":"cloudflare-logpush:noema-production","sourceMethod":"log-url","logPath":${JSON.stringify(logPath)},"records":2,"collectedAt":"2026-07-02T00:00:00.000Z","logSha256":"${logSha256}","logBytes":${logBytes}}`, + ); + + const result = runStrictGate(logPath, provenancePath, evidencePath); + + expect(result.status).toBe(1); + expect(result.stdout).toContain("KPI provenance file"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects malformed UTF-8 provenance bytes instead of replacement-decoding them", () => { + const dir = mkdtempSync(join(tmpdir(), "noema-kpi-provenance-integrity-")); + try { + const logPath = join(dir, "exchange-30d.ndjson"); + const provenancePath = join(dir, "exchange-30d.ndjson.provenance.json"); + const evidencePath = join(dir, "evidence.json"); + writeThirtyDayExchangeLog(logPath); + const text = validProvenanceJson(logPath); + const bytes = Buffer.from(text, "utf8"); + const marker = Buffer.from("log-url", "utf8"); + const markerOffset = bytes.indexOf(marker); + expect(markerOffset).toBeGreaterThanOrEqual(0); + bytes[markerOffset] = 0x80; + writeFileSync(provenancePath, bytes); + + const result = runStrictGate(logPath, provenancePath, evidencePath); + + expect(result.status).toBe(1); + expect(result.stdout).toContain("KPI provenance file"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); From abcd18a1d279667af423d9a0a782794a46e6fa26 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 15 Aug 2026 07:08:23 +0900 Subject: [PATCH 2/4] test(coverage): count KPI gate production source --- vitest.config.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vitest.config.ts b/vitest.config.ts index fef24afa4..13cfd6342 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -6,6 +6,7 @@ export default defineConfig({ reporter: ["json-summary", "text"], include: [ "src/**/*.ts", + "scripts/kpi-gate.mjs", "scripts/lockfile-change-control.mjs", "scripts/normalize-commercial-readiness-evidence.mjs", "scripts/prepare-agent-pr-message.mjs", @@ -21,4 +22,4 @@ export default defineConfig({ }, }, }, -}); \ No newline at end of file +}); From a238ec2b348820c676d718499fb9e4270e349eb1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 15 Aug 2026 07:20:59 +0900 Subject: [PATCH 3/4] test(kpi): exercise production gate in-process --- test/kpi-gate-production-coverage.test.ts | 329 ++++++++++++++++++++++ 1 file changed, 329 insertions(+) create mode 100644 test/kpi-gate-production-coverage.test.ts diff --git a/test/kpi-gate-production-coverage.test.ts b/test/kpi-gate-production-coverage.test.ts new file mode 100644 index 000000000..28808c4fe --- /dev/null +++ b/test/kpi-gate-production-coverage.test.ts @@ -0,0 +1,329 @@ +import { createHash } from "node:crypto"; +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { afterAll, describe, expect, it, vi } from "vitest"; + +const originalEnvironment = { ...process.env }; +const originalArgv = [...process.argv]; +const originalExitCode = process.exitCode; +const directories: string[] = []; + +class ExitSignal extends Error { + constructor(readonly code: number) { + super(`EXIT:${code}`); + } +} + +function createFixture() { + const directory = mkdtempSync(join(tmpdir(), "noema-kpi-production-")); + directories.push(directory); + return { + directory, + logPath: join(directory, "exchange-30d.ndjson"), + provenancePath: join(directory, "exchange-30d.ndjson.provenance.json"), + evidencePath: join(directory, "evidence.json"), + }; +} + +function writeThirtyDayExchangeLog(path: string, statusCode = 200) { + const records = [ + { + event: "http_request", + route: "/exchange", + status_code: statusCode, + latency_ms: 120, + timestamp: "2026-06-01T00:00:00.000Z", + }, + { + event: "http_request", + route: "/exchange", + status_code: 200, + latency_ms: 157, + timestamp: "2026-07-01T03:00:00.000Z", + }, + ]; + writeFileSync(path, `${records.map((record) => JSON.stringify(record)).join("\n")}\n`, "utf8"); +} + +function identity(path: string) { + const bytes = readFileSync(path); + return { + logSha256: createHash("sha256").update(bytes).digest("hex"), + logBytes: bytes.byteLength, + }; +} + +function validProvenance(logPath: string) { + return { + sourceKind: "production", + sourceId: "cloudflare-logpush:noema-production", + sourceMethod: "log-url", + logPath, + records: 2, + collectedAt: "2026-07-02T00:00:00.000Z", + ...identity(logPath), + }; +} + +function restoreProcessState() { + for (const key of Object.keys(process.env)) { + if (!(key in originalEnvironment)) delete process.env[key]; + } + Object.assign(process.env, originalEnvironment); + process.argv.splice(0, process.argv.length, ...originalArgv); + process.exitCode = originalExitCode; +} + +async function runGate({ + fixture, + args = [], + env = {}, + evidence = true, +}: { + fixture: ReturnType; + args?: string[]; + env?: NodeJS.ProcessEnv; + evidence?: boolean; +}) { + restoreProcessState(); + Object.assign(process.env, { + NOEMA_KPI_LOG_PATH: fixture.logPath, + NOEMA_KPI_PROVENANCE_PATH: fixture.provenancePath, + NOEMA_KPI_REQUIRE_WINDOW_DAYS: "30", + ...env, + }); + if (evidence) process.env.NOEMA_KPI_EVIDENCE_PATH = fixture.evidencePath; + else delete process.env.NOEMA_KPI_EVIDENCE_PATH; + process.argv.splice( + 0, + process.argv.length, + originalArgv[0] ?? process.execPath, + resolve("scripts/kpi-gate.mjs"), + ...args, + ); + + vi.resetModules(); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new ExitSignal(code ?? 0); + }) as never); + let exitCode: number | null = null; + let error: unknown = null; + try { + await import("../scripts/kpi-gate.mjs"); + } catch (caught) { + if (caught instanceof ExitSignal) exitCode = caught.code; + else error = caught; + } finally { + exitSpy.mockRestore(); + logSpy.mockRestore(); + errorSpy.mockRestore(); + restoreProcessState(); + } + if (error) throw error; + return { exitCode }; +} + +function writeProvenance(fixture: ReturnType, overrides = {}) { + writeFileSync( + fixture.provenancePath, + `${JSON.stringify({ ...validProvenance(fixture.logPath), ...overrides }, null, 2)}\n`, + "utf8", + ); +} + +afterAll(() => { + restoreProcessState(); + for (const directory of directories) rmSync(directory, { recursive: true, force: true }); +}); + +describe("KPI gate production entrypoint coverage", () => { + it("persists a non-strict missing-log SKIP without manufacturing production evidence", async () => { + const fixture = createFixture(); + const result = await runGate({ + fixture, + env: { + NOEMA_KPI_STRICT: "0", + NOEMA_KPI_FAILURE_THRESHOLD: "0.02", + NOEMA_KPI_P95_THRESHOLD_MS: "300", + }, + }); + + expect(result.exitCode).toBe(0); + expect(JSON.parse(readFileSync(fixture.evidencePath, "utf8"))).toMatchObject({ + status: "SKIP", + strict: false, + }); + }); + + it("takes the no-evidence-path branch for a non-strict missing log", async () => { + const fixture = createFixture(); + const result = await runGate({ fixture, evidence: false, env: { NOEMA_KPI_STRICT: "0" } }); + expect(result.exitCode).toBe(0); + }); + + it("rejects non-numeric thresholds before reading the log", async () => { + const fixture = createFixture(); + const result = await runGate({ fixture, args: [fixture.logPath, "nan", "300"] }); + expect(result.exitCode).toBe(1); + }); + + it("rejects a non-positive strict window", async () => { + const fixture = createFixture(); + const result = await runGate({ + fixture, + args: ["--strict", "--require-window-days", "0", fixture.logPath], + env: { NOEMA_KPI_STRICT: "0" }, + }); + expect(result.exitCode).toBe(1); + }); + + it("fails strict mode when the production log is missing", async () => { + const fixture = createFixture(); + const result = await runGate({ fixture, env: { NOEMA_KPI_STRICT: "1" } }); + expect(result.exitCode).toBe(1); + expect(JSON.parse(readFileSync(fixture.evidencePath, "utf8")).status).toBe("FAIL"); + }); + + it("fails strict mode when provenance is missing", async () => { + const fixture = createFixture(); + writeThirtyDayExchangeLog(fixture.logPath); + const result = await runGate({ fixture, env: { NOEMA_KPI_STRICT: "1" } }); + expect(result.exitCode).toBe(1); + expect(readFileSync(fixture.evidencePath, "utf8")).toContain("Missing KPI provenance file"); + }); + + it("rejects malformed provenance UTF-8", async () => { + const fixture = createFixture(); + writeThirtyDayExchangeLog(fixture.logPath); + writeFileSync(fixture.provenancePath, Buffer.from([0x7b, 0x22, 0x61, 0x22, 0x3a, 0xc3, 0x28, 0x7d])); + const result = await runGate({ fixture, env: { NOEMA_KPI_STRICT: "1" } }); + expect(result.exitCode).toBe(1); + expect(readFileSync(fixture.evidencePath, "utf8")).toContain("not valid UTF-8"); + }); + + it("rejects duplicate decoded provenance keys before JSON.parse", async () => { + const fixture = createFixture(); + writeThirtyDayExchangeLog(fixture.logPath); + const valid = validProvenance(fixture.logPath); + writeFileSync( + fixture.provenancePath, + `{"sourceKind":"production","source\\u004bind":"staging","sourceId":"${valid.sourceId}","records":2,"collectedAt":"${valid.collectedAt}","logSha256":"${valid.logSha256}","logBytes":${valid.logBytes}}`, + "utf8", + ); + const result = await runGate({ fixture, env: { NOEMA_KPI_STRICT: "1" } }); + expect(result.exitCode).toBe(1); + expect(readFileSync(fixture.evidencePath, "utf8")).toContain("duplicate decoded JSON object keys"); + }); + + it("rejects malformed provenance JSON", async () => { + const fixture = createFixture(); + writeThirtyDayExchangeLog(fixture.logPath); + writeFileSync(fixture.provenancePath, "{", "utf8"); + const result = await runGate({ fixture, env: { NOEMA_KPI_STRICT: "1" } }); + expect(result.exitCode).toBe(1); + expect(readFileSync(fixture.evidencePath, "utf8")).toContain("not valid JSON"); + }); + + it.each([ + [{ sourceKind: "staging" }, "sourceKind"], + [{ sourceId: "" }, "sourceId is required"], + [{ sourceId: "https://logs.example/?token=secret" }, "stable non-secret label"], + [{ collectedAt: "not-a-date" }, "collectedAt"], + [{ records: 0 }, "records must be a positive number"], + [{ logSha256: "BAD" }, "logSha256"], + [{ logBytes: 0 }, "logBytes"], + ])("rejects invalid strict provenance %#", async (overrides, reason) => { + const fixture = createFixture(); + writeThirtyDayExchangeLog(fixture.logPath); + writeProvenance(fixture, overrides); + const result = await runGate({ fixture, env: { NOEMA_KPI_STRICT: "1" } }); + expect(result.exitCode).toBe(1); + expect(readFileSync(fixture.evidencePath, "utf8")).toContain(reason); + }); + + it("rejects provenance whose authenticated bytes no longer match the log", async () => { + const fixture = createFixture(); + writeThirtyDayExchangeLog(fixture.logPath); + writeProvenance(fixture); + writeFileSync(fixture.logPath, `${readFileSync(fixture.logPath, "utf8")}{}\n`, "utf8"); + const result = await runGate({ fixture, env: { NOEMA_KPI_STRICT: "1" } }); + expect(result.exitCode).toBe(1); + expect(readFileSync(fixture.evidencePath, "utf8")).toContain("identity does not match production provenance"); + }); + + it("fails closed when a verified snapshot cannot be created", async () => { + const fixture = createFixture(); + writeThirtyDayExchangeLog(fixture.logPath); + writeProvenance(fixture); + const missingTmp = join(fixture.directory, "missing-tmp"); + const result = await runGate({ + fixture, + env: { NOEMA_KPI_STRICT: "1", TMPDIR: missingTmp, TMP: missingTmp, TEMP: missingTmp }, + }); + expect(result.exitCode).toBe(1); + expect(readFileSync(fixture.evidencePath, "utf8")).toContain("permission-restricted verified snapshot"); + }); + + it("passes a real strict 30-day production gate and persists exact provenance", async () => { + const fixture = createFixture(); + writeThirtyDayExchangeLog(fixture.logPath); + writeProvenance(fixture); + const result = await runGate({ + fixture, + args: ["--strict", "--require-window-days", "30", fixture.logPath, "0.02", "300"], + env: { NOEMA_KPI_STRICT: "0" }, + }); + + expect(result.exitCode).toBeNull(); + const evidence = JSON.parse(readFileSync(fixture.evidencePath, "utf8")); + expect(evidence.status).toBe("PASS"); + expect(evidence.provenance).toMatchObject(identity(fixture.logPath)); + expect(evidence.steps).toContainEqual({ + name: "kpi-log-identity-final", + status: "PASS", + exitCode: 0, + }); + }); + + it("records a realistic KPI child failure instead of converting it to PASS", async () => { + const fixture = createFixture(); + writeThirtyDayExchangeLog(fixture.logPath, 500); + writeProvenance(fixture); + const result = await runGate({ fixture, env: { NOEMA_KPI_STRICT: "1" } }); + expect(result.exitCode).toBe(1); + const evidence = JSON.parse(readFileSync(fixture.evidencePath, "utf8")); + expect(evidence.status).toBe("FAIL"); + expect(evidence.steps.some((step: { status: string }) => step.status === "FAIL")).toBe(true); + }); + + it("fails strict mode when the final evidence path cannot be opened safely", async () => { + const fixture = createFixture(); + writeThirtyDayExchangeLog(fixture.logPath); + writeProvenance(fixture); + mkdirSync(fixture.evidencePath); + const result = await runGate({ fixture, env: { NOEMA_KPI_STRICT: "1" } }); + expect(result.exitCode).toBe(1); + }); + + it("refuses a symlink evidence target through O_NOFOLLOW", async () => { + const fixture = createFixture(); + writeThirtyDayExchangeLog(fixture.logPath); + writeProvenance(fixture); + const target = join(fixture.directory, "target.json"); + writeFileSync(target, "sentinel", "utf8"); + symlinkSync(target, fixture.evidencePath); + const result = await runGate({ fixture, env: { NOEMA_KPI_STRICT: "1" } }); + expect(result.exitCode).toBe(1); + expect(readFileSync(target, "utf8")).toBe("sentinel"); + }); +}); From 3073c08765c7b5ce295a0d1c927efe1d827d25aa Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 15 Aug 2026 07:27:12 +0900 Subject: [PATCH 4/4] test(kpi): exercise escaped child JSON parsing --- test/kpi-gate-production-coverage.test.ts | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/test/kpi-gate-production-coverage.test.ts b/test/kpi-gate-production-coverage.test.ts index 28808c4fe..525487178 100644 --- a/test/kpi-gate-production-coverage.test.ts +++ b/test/kpi-gate-production-coverage.test.ts @@ -33,7 +33,7 @@ function createFixture() { }; } -function writeThirtyDayExchangeLog(path: string, statusCode = 200) { +function writeThirtyDayExchangeLog(path: string, statusCode = 200, errorCode = "") { const records = [ { event: "http_request", @@ -41,6 +41,7 @@ function writeThirtyDayExchangeLog(path: string, statusCode = 200) { status_code: statusCode, latency_ms: 120, timestamp: "2026-06-01T00:00:00.000Z", + ...(errorCode ? { error_code: errorCode } : {}), }, { event: "http_request", @@ -295,6 +296,20 @@ describe("KPI gate production entrypoint coverage", () => { }); }); + it("parses escaped child JSON output without losing string-boundary state", async () => { + const fixture = createFixture(); + writeThirtyDayExchangeLog(fixture.logPath, 200, "ERR_CUSTOM_\\_BOUNDARY"); + writeProvenance(fixture); + const result = await runGate({ fixture, env: { NOEMA_KPI_STRICT: "1" } }); + + expect(result.exitCode).toBeNull(); + const evidence = JSON.parse(readFileSync(fixture.evidencePath, "utf8")); + expect(evidence.parsed.alert.alerts.errorCodeTop10).toContainEqual({ + error_code: "ERR_CUSTOM_\\_BOUNDARY", + count: 1, + }); + }); + it("records a realistic KPI child failure instead of converting it to PASS", async () => { const fixture = createFixture(); writeThirtyDayExchangeLog(fixture.logPath, 500);