diff --git a/data/discovery-evidence.json b/data/discovery-evidence.json index fe51488c..9edb0dc6 100644 --- a/data/discovery-evidence.json +++ b/data/discovery-evidence.json @@ -1 +1,4 @@ -[] +{ + "schemaVersion": 1, + "entries": [] +} diff --git a/src/agents/contracts/evidence.ts b/src/agents/contracts/evidence.ts index b291bdaa..b97625fb 100644 --- a/src/agents/contracts/evidence.ts +++ b/src/agents/contracts/evidence.ts @@ -25,6 +25,8 @@ export type CrossRunEvidence = Record; // --- Discovery evidence --- +export const DISCOVERY_EVIDENCE_SCHEMA_VERSION = 1; + export const DiscoveryEvidenceEntrySchema = z.object({ description: z.string(), category: z.string(), @@ -35,3 +37,10 @@ export const DiscoveryEvidenceEntrySchema = z.object({ }); export type DiscoveryEvidenceEntry = z.infer; + +export const DiscoveryEvidenceFileSchema = z.object({ + schemaVersion: z.literal(DISCOVERY_EVIDENCE_SCHEMA_VERSION), + entries: z.array(z.unknown()), +}); + +export type DiscoveryEvidenceFile = z.infer; diff --git a/src/agents/evidence-collector.test.ts b/src/agents/evidence-collector.test.ts index 91e19313..b3fe2974 100644 --- a/src/agents/evidence-collector.test.ts +++ b/src/agents/evidence-collector.test.ts @@ -8,6 +8,7 @@ import { loadDiscoveryEvidence, appendDiscoveryEvidence, pruneDiscoveryEvidence, + DISCOVERY_EVIDENCE_SCHEMA_VERSION, } from "./evidence-collector.js"; import type { CalibrationEvidenceEntry, @@ -187,7 +188,21 @@ describe("evidence-collector", () => { expect(result).toEqual([]); }); - it("returns entries from file", () => { + it("loads entries from versioned format", () => { + const file = { + schemaVersion: DISCOVERY_EVIDENCE_SCHEMA_VERSION, + entries: [ + { description: "gap1", category: "layout", impact: "hard", fixture: "fx1", timestamp: "t1", source: "evaluation" }, + ], + }; + writeFileSync(disPath, JSON.stringify(file), "utf-8"); + + const result = loadDiscoveryEvidence(disPath); + expect(result).toHaveLength(1); + expect(result[0]!.category).toBe("layout"); + }); + + it("loads entries from legacy plain-array format (v0 fallback)", () => { const entries: DiscoveryEvidenceEntry[] = [ { description: "gap1", category: "layout", impact: "hard", fixture: "fx1", timestamp: "t1", source: "evaluation" }, ]; @@ -197,59 +212,243 @@ describe("evidence-collector", () => { expect(result).toHaveLength(1); expect(result[0]!.category).toBe("layout"); }); + + it("handles malformed JSON gracefully", () => { + writeFileSync(disPath, "not json", "utf-8"); + const result = loadDiscoveryEvidence(disPath); + expect(result).toEqual([]); + }); + + it("skips invalid entries in legacy array", () => { + writeFileSync(disPath, JSON.stringify([ + { description: "gap1", category: "layout", impact: "hard", fixture: "fx1", timestamp: "t1", source: "evaluation" }, + { bad: "entry" }, + ]), "utf-8"); + + const result = loadDiscoveryEvidence(disPath); + expect(result).toHaveLength(1); + }); + + it("skips invalid entries in versioned format (partial corruption)", () => { + const file = { + schemaVersion: DISCOVERY_EVIDENCE_SCHEMA_VERSION, + entries: [ + { description: "good", category: "layout", impact: "hard", fixture: "fx1", timestamp: "t1", source: "evaluation" }, + { bad: "entry" }, + { description: "also good", category: "color", impact: "easy", fixture: "fx2", timestamp: "t2", source: "gap-analysis" }, + ], + }; + writeFileSync(disPath, JSON.stringify(file), "utf-8"); + + const result = loadDiscoveryEvidence(disPath); + expect(result).toHaveLength(2); + expect(result[0]!.description).toBe("good"); + expect(result[1]!.description).toBe("also good"); + }); + + it("throws on unsupported schemaVersion to prevent silent overwrite", () => { + const file = { + schemaVersion: 999, + entries: [ + { description: "gap1", category: "layout", impact: "hard", fixture: "fx1", timestamp: "t1", source: "evaluation" }, + ], + }; + writeFileSync(disPath, JSON.stringify(file), "utf-8"); + + expect(() => loadDiscoveryEvidence(disPath)).toThrow(/Unsupported discovery-evidence schemaVersion: 999/); + }); }); describe("appendDiscoveryEvidence", () => { - it("creates file and appends entries", () => { + it("creates file in versioned format", () => { appendDiscoveryEvidence([ { description: "gap1", category: "layout", impact: "hard", fixture: "fx1", timestamp: "t1", source: "gap-analysis" }, ], disPath); - const raw = JSON.parse(readFileSync(disPath, "utf-8")) as DiscoveryEvidenceEntry[]; - expect(raw).toHaveLength(1); - expect(raw[0]!.source).toBe("gap-analysis"); + const raw = JSON.parse(readFileSync(disPath, "utf-8")) as { schemaVersion: number; entries: DiscoveryEvidenceEntry[] }; + expect(raw.schemaVersion).toBe(DISCOVERY_EVIDENCE_SCHEMA_VERSION); + expect(raw.entries).toHaveLength(1); + expect(raw.entries[0]!.source).toBe("gap-analysis"); }); - it("appends to existing entries", () => { - writeFileSync(disPath, JSON.stringify([ + it("appends to existing entries (different keys)", () => { + appendDiscoveryEvidence([ { description: "gap1", category: "layout", impact: "hard", fixture: "fx1", timestamp: "t1", source: "evaluation" }, - ]), "utf-8"); + ], disPath); appendDiscoveryEvidence([ { description: "gap2", category: "color", impact: "moderate", fixture: "fx2", timestamp: "t2", source: "gap-analysis" }, ], disPath); - const raw = JSON.parse(readFileSync(disPath, "utf-8")) as DiscoveryEvidenceEntry[]; - expect(raw).toHaveLength(2); + const raw = JSON.parse(readFileSync(disPath, "utf-8")) as { entries: DiscoveryEvidenceEntry[] }; + expect(raw.entries).toHaveLength(2); + }); + + it("deduplicates by (category + description + fixture), last-write-wins", () => { + appendDiscoveryEvidence([ + { description: "gap1", category: "layout", impact: "hard", fixture: "fx1", timestamp: "t1", source: "evaluation" }, + ], disPath); + + // Same category+description+fixture, different impact/timestamp → replaces + appendDiscoveryEvidence([ + { description: "gap1", category: "layout", impact: "moderate", fixture: "fx1", timestamp: "t2", source: "evaluation" }, + ], disPath); + + const raw = JSON.parse(readFileSync(disPath, "utf-8")) as { entries: DiscoveryEvidenceEntry[] }; + expect(raw.entries).toHaveLength(1); + expect(raw.entries[0]!.impact).toBe("moderate"); + expect(raw.entries[0]!.timestamp).toBe("t2"); + }); + + it("dedupe is case-insensitive for category and description", () => { + appendDiscoveryEvidence([ + { description: "Gap One", category: "Layout", impact: "hard", fixture: "fx1", timestamp: "t1", source: "evaluation" }, + ], disPath); + + appendDiscoveryEvidence([ + { description: "gap one", category: "layout", impact: "easy", fixture: "fx1", timestamp: "t2", source: "evaluation" }, + ], disPath); + + const raw = JSON.parse(readFileSync(disPath, "utf-8")) as { entries: DiscoveryEvidenceEntry[] }; + expect(raw.entries).toHaveLength(1); + expect(raw.entries[0]!.impact).toBe("easy"); + }); + + it("dedupe is case-insensitive for fixture", () => { + appendDiscoveryEvidence([ + { description: "gap1", category: "layout", impact: "hard", fixture: "FX1", timestamp: "t1", source: "evaluation" }, + ], disPath); + + appendDiscoveryEvidence([ + { description: "gap1", category: "layout", impact: "easy", fixture: "fx1", timestamp: "t2", source: "evaluation" }, + ], disPath); + + const raw = JSON.parse(readFileSync(disPath, "utf-8")) as { entries: DiscoveryEvidenceEntry[] }; + expect(raw.entries).toHaveLength(1); + expect(raw.entries[0]!.impact).toBe("easy"); + }); + + it("dedupes within a single append call (last row wins)", () => { + appendDiscoveryEvidence([ + { description: "gap1", category: "layout", impact: "hard", fixture: "fx1", timestamp: "t1", source: "evaluation" }, + { description: "gap1", category: "layout", impact: "easy", fixture: "fx1", timestamp: "t2", source: "evaluation" }, + ], disPath); + + const raw = JSON.parse(readFileSync(disPath, "utf-8")) as { entries: DiscoveryEvidenceEntry[] }; + expect(raw.entries).toHaveLength(1); + expect(raw.entries[0]!.impact).toBe("easy"); + }); + + it("same description different fixture → kept as separate entries", () => { + appendDiscoveryEvidence([ + { description: "gap1", category: "layout", impact: "hard", fixture: "fx1", timestamp: "t1", source: "evaluation" }, + { description: "gap1", category: "layout", impact: "hard", fixture: "fx2", timestamp: "t1", source: "evaluation" }, + ], disPath); + + const raw = JSON.parse(readFileSync(disPath, "utf-8")) as { entries: DiscoveryEvidenceEntry[] }; + expect(raw.entries).toHaveLength(2); + }); + + it("migrates legacy array to versioned format on append", () => { + // Write legacy format + writeFileSync(disPath, JSON.stringify([ + { description: "old", category: "layout", impact: "hard", fixture: "fx1", timestamp: "t0", source: "evaluation" }, + ]), "utf-8"); + + appendDiscoveryEvidence([ + { description: "new", category: "color", impact: "easy", fixture: "fx2", timestamp: "t1", source: "gap-analysis" }, + ], disPath); + + const raw = JSON.parse(readFileSync(disPath, "utf-8")) as { schemaVersion: number; entries: DiscoveryEvidenceEntry[] }; + expect(raw.schemaVersion).toBe(DISCOVERY_EVIDENCE_SCHEMA_VERSION); + expect(raw.entries).toHaveLength(2); + }); + + it("does nothing for empty entries", () => { + appendDiscoveryEvidence([ + { description: "gap1", category: "layout", impact: "hard", fixture: "fx1", timestamp: "t1", source: "evaluation" }, + ], disPath); + const before = readFileSync(disPath, "utf-8"); + + appendDiscoveryEvidence([], disPath); + + const after = readFileSync(disPath, "utf-8"); + expect(after).toBe(before); + }); + + it("throws when file has unsupported schemaVersion", () => { + const file = { schemaVersion: 999, entries: [] }; + writeFileSync(disPath, JSON.stringify(file), "utf-8"); + const before = readFileSync(disPath, "utf-8"); + + expect(() => appendDiscoveryEvidence([ + { description: "new", category: "layout", impact: "hard", fixture: "fx1", timestamp: "t1", source: "evaluation" }, + ], disPath)).toThrow(/Unsupported discovery-evidence schemaVersion/); + + // File must not be overwritten + expect(readFileSync(disPath, "utf-8")).toBe(before); }); }); describe("pruneDiscoveryEvidence", () => { it("removes entries for specified categories (case-insensitive)", () => { - const entries: DiscoveryEvidenceEntry[] = [ + appendDiscoveryEvidence([ { description: "gap1", category: "Layout", impact: "hard", fixture: "fx1", timestamp: "t1", source: "evaluation" }, { description: "gap2", category: "layout", impact: "hard", fixture: "fx2", timestamp: "t2", source: "gap-analysis" }, { description: "gap3", category: "color", impact: "moderate", fixture: "fx1", timestamp: "t1", source: "evaluation" }, - ]; - writeFileSync(disPath, JSON.stringify(entries), "utf-8"); + ], disPath); pruneDiscoveryEvidence(["layout"], disPath); - const raw = JSON.parse(readFileSync(disPath, "utf-8")) as DiscoveryEvidenceEntry[]; - expect(raw).toHaveLength(1); - expect(raw[0]!.category).toBe("color"); + const raw = JSON.parse(readFileSync(disPath, "utf-8")) as { entries: DiscoveryEvidenceEntry[] }; + expect(raw.entries).toHaveLength(1); + expect(raw.entries[0]!.category).toBe("color"); + }); + + it("writes versioned format after prune", () => { + appendDiscoveryEvidence([ + { description: "gap1", category: "layout", impact: "hard", fixture: "fx1", timestamp: "t1", source: "evaluation" }, + ], disPath); + + pruneDiscoveryEvidence(["layout"], disPath); + + const raw = JSON.parse(readFileSync(disPath, "utf-8")) as { schemaVersion: number; entries: DiscoveryEvidenceEntry[] }; + expect(raw.schemaVersion).toBe(DISCOVERY_EVIDENCE_SCHEMA_VERSION); + expect(raw.entries).toHaveLength(0); }); it("does nothing for empty categories", () => { - const entries: DiscoveryEvidenceEntry[] = [ + appendDiscoveryEvidence([ { description: "gap1", category: "layout", impact: "hard", fixture: "fx1", timestamp: "t1", source: "evaluation" }, - ]; - writeFileSync(disPath, JSON.stringify(entries), "utf-8"); + ], disPath); pruneDiscoveryEvidence([], disPath); - const raw = JSON.parse(readFileSync(disPath, "utf-8")) as DiscoveryEvidenceEntry[]; - expect(raw).toHaveLength(1); + const raw = JSON.parse(readFileSync(disPath, "utf-8")) as { entries: DiscoveryEvidenceEntry[] }; + expect(raw.entries).toHaveLength(1); + }); + + it("trims categories when matching", () => { + appendDiscoveryEvidence([ + { description: "gap1", category: "layout", impact: "hard", fixture: "fx1", timestamp: "t1", source: "evaluation" }, + ], disPath); + + pruneDiscoveryEvidence([" layout "], disPath); + + const raw = JSON.parse(readFileSync(disPath, "utf-8")) as { entries: DiscoveryEvidenceEntry[] }; + expect(raw.entries).toHaveLength(0); + }); + + it("throws when file has unsupported schemaVersion", () => { + const file = { schemaVersion: 999, entries: [ + { description: "gap1", category: "layout", impact: "hard", fixture: "fx1", timestamp: "t1", source: "evaluation" }, + ]}; + writeFileSync(disPath, JSON.stringify(file), "utf-8"); + const before = readFileSync(disPath, "utf-8"); + + expect(() => pruneDiscoveryEvidence(["layout"], disPath)).toThrow(/Unsupported discovery-evidence schemaVersion/); + + expect(readFileSync(disPath, "utf-8")).toBe(before); }); }); }); diff --git a/src/agents/evidence-collector.ts b/src/agents/evidence-collector.ts index 6ea7497f..83391ed9 100644 --- a/src/agents/evidence-collector.ts +++ b/src/agents/evidence-collector.ts @@ -3,6 +3,8 @@ import { dirname, resolve } from "node:path"; import { CalibrationEvidenceEntrySchema, DiscoveryEvidenceEntrySchema, + DiscoveryEvidenceFileSchema, + DISCOVERY_EVIDENCE_SCHEMA_VERSION, } from "./contracts/evidence.js"; import type { CalibrationEvidenceEntry, @@ -11,6 +13,7 @@ import type { } from "./contracts/evidence.js"; export type { CalibrationEvidenceEntry, CrossRunEvidence, DiscoveryEvidenceEntry }; +export { DISCOVERY_EVIDENCE_SCHEMA_VERSION }; const DEFAULT_CALIBRATION_PATH = resolve("data/calibration-evidence.json"); @@ -120,26 +123,126 @@ export function pruneCalibrationEvidence( const DEFAULT_DISCOVERY_PATH = resolve("data/discovery-evidence.json"); +/** + * Build a dedupe key for a discovery evidence entry. + * Key: category (lowered) + normalized description + fixture (trimmed, lowered). + */ +function discoveryDedupeKey(e: DiscoveryEvidenceEntry): string { + const cat = e.category.toLowerCase().trim(); + const desc = e.description.toLowerCase().trim().replace(/\s+/g, " "); + const fix = e.fixture.toLowerCase().trim(); + return `${cat}\0${desc}\0${fix}`; +} + +/** + * Read discovery evidence from file, supporting both legacy (plain array) + * and versioned ({ schemaVersion, entries }) formats. + * Throws if the file contains a versioned object with an unsupported schemaVersion + * to prevent silent data loss on subsequent writes. + */ +function readDiscoveryEvidence(filePath: string): DiscoveryEvidenceEntry[] { + if (!existsSync(filePath)) return []; + try { + const raw = JSON.parse(readFileSync(filePath, "utf-8")) as unknown; + + // Versioned format: { schemaVersion, entries } + const versionedParse = DiscoveryEvidenceFileSchema.safeParse(raw); + if (versionedParse.success) { + // Validate entries individually so one bad row doesn't discard all + const result: DiscoveryEvidenceEntry[] = []; + for (const item of versionedParse.data.entries) { + const parsed = DiscoveryEvidenceEntrySchema.safeParse(item); + if (parsed.success && parsed.data !== undefined) { + result.push(parsed.data); + } + } + return result; + } + + // Detect unsupported versioned format — refuse to load to prevent silent overwrite + if ( + typeof raw === "object" && + raw !== null && + !Array.isArray(raw) && + "schemaVersion" in raw + ) { + const version = (raw as { schemaVersion: unknown }).schemaVersion; + throw new Error( + `Unsupported discovery-evidence schemaVersion: ${String(version)} (expected ${DISCOVERY_EVIDENCE_SCHEMA_VERSION}). ` + + `Upgrade canicode to read this file, or delete it to start fresh.` + ); + } + + // Legacy format: plain array (v0, before schemaVersion was introduced) + if (Array.isArray(raw)) { + const result: DiscoveryEvidenceEntry[] = []; + for (const item of raw) { + const parsed = DiscoveryEvidenceEntrySchema.safeParse(item); + if (parsed.success && parsed.data !== undefined) { + result.push(parsed.data); + } + } + return result; + } + + return []; + } catch (err) { + // Re-throw unsupported version errors; swallow everything else (malformed JSON, etc.) + if (err instanceof Error && err.message.startsWith("Unsupported discovery-evidence")) { + throw err; + } + return []; + } +} + +/** + * Write discovery evidence in the versioned format. + */ +function writeDiscoveryEvidence(filePath: string, entries: DiscoveryEvidenceEntry[]): void { + const dir = dirname(filePath); + if (!existsSync(dir)) { + mkdirSync(dir, { recursive: true }); + } + const data = { + schemaVersion: DISCOVERY_EVIDENCE_SCHEMA_VERSION, + entries, + }; + writeFileSync(filePath, JSON.stringify(data, null, 2) + "\n", "utf-8"); +} + /** * Load all discovery evidence entries. */ export function loadDiscoveryEvidence( evidencePath: string = DEFAULT_DISCOVERY_PATH ): DiscoveryEvidenceEntry[] { - return readValidatedArray(evidencePath, DiscoveryEvidenceEntrySchema); + return readDiscoveryEvidence(evidencePath); } /** - * Append new discovery evidence entries (missing-rule + gap analysis). + * Append new discovery evidence entries with deduplication. + * Dedupe key: (category + normalized description + fixture). + * Last-write-wins for duplicate keys. */ export function appendDiscoveryEvidence( entries: DiscoveryEvidenceEntry[], evidencePath: string = DEFAULT_DISCOVERY_PATH ): void { if (entries.length === 0) return; - const existing = readValidatedArray(evidencePath, DiscoveryEvidenceEntrySchema); - existing.push(...entries); - writeJsonArray(evidencePath, existing); + const existing = readDiscoveryEvidence(evidencePath); + + // Build map of existing entries keyed by dedupe key + const byKey = new Map(); + for (const e of existing) { + byKey.set(discoveryDedupeKey(e), e); + } + + // Incoming entries override existing duplicates (last-write-wins) + for (const e of entries) { + byKey.set(discoveryDedupeKey(e), e); + } + + writeDiscoveryEvidence(evidencePath, [...byKey.values()]); } /** @@ -150,8 +253,10 @@ export function pruneDiscoveryEvidence( evidencePath: string = DEFAULT_DISCOVERY_PATH ): void { if (categories.length === 0) return; - const catSet = new Set(categories.map((c) => c.toLowerCase())); - const existing = readValidatedArray(evidencePath, DiscoveryEvidenceEntrySchema); - const pruned = existing.filter((e) => !catSet.has(e.category.toLowerCase())); - writeJsonArray(evidencePath, pruned); + const catSet = new Set( + categories.map((c) => c.toLowerCase().trim()).filter((c) => c.length > 0), + ); + const existing = readDiscoveryEvidence(evidencePath); + const pruned = existing.filter((e) => !catSet.has(e.category.toLowerCase().trim())); + writeDiscoveryEvidence(evidencePath, pruned); } diff --git a/src/agents/gap-rule-report.ts b/src/agents/gap-rule-report.ts index 16279e32..76c23e60 100644 --- a/src/agents/gap-rule-report.ts +++ b/src/agents/gap-rule-report.ts @@ -5,6 +5,7 @@ import type { RuleId } from "../core/contracts/rule.js"; import { runCalibrationEvaluate } from "./orchestrator.js"; import { GapAnalyzerOutputSchema } from "./contracts/gap-analyzer.js"; import { loadCalibrationEvidence, loadDiscoveryEvidence } from "./evidence-collector.js"; +import type { DiscoveryEvidenceEntry } from "./evidence-collector.js"; type CalibrationAnalysisJson = Parameters[0] & { ruleScores: Record; @@ -508,7 +509,12 @@ export function generateGapRuleReport(options: GapRuleReportOptions): GapRuleRep } lines.push(""); - const discoveryEvidence = loadDiscoveryEvidence(); + let discoveryEvidence: DiscoveryEvidenceEntry[] = []; + try { + discoveryEvidence = loadDiscoveryEvidence(); + } catch (err) { + console.warn("[evidence] Failed to load discovery evidence (non-fatal):", err); + } lines.push("## Cross-run discovery evidence (git-tracked)"); lines.push(""); if (discoveryEvidence.length === 0) { diff --git a/src/cli/index.ts b/src/cli/index.ts index 09485b1d..9a6a46b4 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -700,8 +700,14 @@ cli ) .action((category: string | string[]) => { const categories = Array.isArray(category) ? category : [category]; - pruneDiscoveryEvidence(categories); - console.log(`Pruned discovery evidence for categories: ${categories.join(", ")}`); + try { + pruneDiscoveryEvidence(categories); + console.log(`Pruned discovery evidence for categories: ${categories.join(", ")}`); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + console.error(`[evidence] Failed to prune discovery evidence: ${msg}`); + process.exitCode = 1; + } }); interface CalibrateRunOptions {