Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ pnpm lint # Type check

- Use ESM modules (`import`/`export`)
- Use `.js` extension for relative imports
- Use `@/*` path alias to reference `src/`
- Use relative paths for imports (not `@/*` alias)

### TypeScript

Expand Down
4 changes: 2 additions & 2 deletions src/agents/analysis-agent.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { runAnalysisAgent, extractRuleScores } from "./analysis-agent.js";
import type { AnalysisResult, AnalysisIssue } from "@/core/engine/rule-engine.js";
import type { AnalysisFile } from "@/core/contracts/figma-node.js";
import type { AnalysisResult, AnalysisIssue } from "../core/engine/rule-engine.js";
import type { AnalysisFile } from "../core/contracts/figma-node.js";

const mockFile = {
fileKey: "test",
Expand Down
8 changes: 4 additions & 4 deletions src/agents/analysis-agent.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import type { AnalysisResult } from "@/core/engine/rule-engine.js";
import { calculateScores } from "@/core/engine/scoring.js";
import type { RuleId } from "@/core/contracts/rule.js";
import type { Severity } from "@/core/contracts/severity.js";
import type { AnalysisResult } from "../core/engine/rule-engine.js";
import { calculateScores } from "../core/engine/scoring.js";
import type { RuleId } from "../core/contracts/rule.js";
import type { Severity } from "../core/contracts/severity.js";
import type {
AnalysisAgentInput,
AnalysisAgentOutput,
Expand Down
2 changes: 1 addition & 1 deletion src/agents/code-renderer.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { writeFileSync, readFileSync, mkdirSync } from "node:fs";
import { resolve } from "node:path";
import { renderCodeScreenshot } from "@/core/engine/visual-compare.js";
import { renderCodeScreenshot } from "../core/engine/visual-compare.js";

/**
* Render generated HTML/CSS/React code to a PNG screenshot.
Expand Down
8 changes: 4 additions & 4 deletions src/agents/contracts/analysis-agent.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import { z } from "zod";
import type { AnalysisResult } from "@/core/engine/rule-engine.js";
import type { ScoreReport } from "@/core/engine/scoring.js";
import type { RuleId } from "@/core/contracts/rule.js";
import type { Severity } from "@/core/contracts/severity.js";
import type { AnalysisResult } from "../../core/engine/rule-engine.js";
import type { ScoreReport } from "../../core/engine/scoring.js";
import type { RuleId } from "../../core/contracts/rule.js";
import type { Severity } from "../../core/contracts/severity.js";

export const NodeIssueSummarySchema = z.object({
nodeId: z.string(),
Expand Down
2 changes: 1 addition & 1 deletion src/agents/contracts/evaluation-agent.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { z } from "zod";
import { DifficultySchema } from "./conversion-agent.js";
import { SeveritySchema } from "@/core/contracts/severity.js";
import { SeveritySchema } from "../../core/contracts/severity.js";

export const MismatchTypeSchema = z.enum([
"overscored",
Expand Down
2 changes: 1 addition & 1 deletion src/agents/contracts/tuning-agent.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { z } from "zod";
import { SeveritySchema } from "@/core/contracts/severity.js";
import { SeveritySchema } from "../../core/contracts/severity.js";
import type { CrossRunEvidence } from "./evidence.js";

export const ConfidenceSchema = z.enum(["high", "medium", "low"]);
Expand Down
2 changes: 1 addition & 1 deletion src/agents/evaluation-agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import type {
MismatchType,
} from "./contracts/evaluation-agent.js";
import type { Difficulty } from "./contracts/conversion-agent.js";
import type { Severity } from "@/core/contracts/severity.js";
import type { Severity } from "../core/contracts/severity.js";

/**
* Difficulty-to-score range mapping.
Expand Down
4 changes: 2 additions & 2 deletions src/agents/orchestrator.test.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import type { AnalysisFile } from "@/core/contracts/figma-node.js";
import type { AnalysisFile } from "../core/contracts/figma-node.js";
import { runCalibrationEvaluate, ENVIRONMENT_NOISE_PATTERNS } from "./orchestrator.js";

// Register rules so RULE_CONFIGS is populated
import "@/core/rules/index.js";
import "../core/rules/index.js";

vi.mock("node:fs/promises", async (importOriginal) => {
const original = await importOriginal<typeof import("node:fs/promises")>();
Expand Down
12 changes: 6 additions & 6 deletions src/agents/orchestrator.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
import type { AnalysisFile, AnalysisNode, AnalysisNodeType } from "@/core/contracts/figma-node.js";
import { analyzeFile } from "@/core/engine/rule-engine.js";
import { RULE_CONFIGS } from "@/core/rules/rule-config.js";
import type { AnalysisFile, AnalysisNode, AnalysisNodeType } from "../core/contracts/figma-node.js";
import { analyzeFile } from "../core/engine/rule-engine.js";
import { RULE_CONFIGS } from "../core/rules/rule-config.js";

import type { CalibrationConfig } from "./contracts/calibration.js";
import { CalibrationConfigSchema } from "./contracts/calibration.js";
import type { NodeIssueSummary } from "./contracts/analysis-agent.js";
import type { ScoreReport } from "@/core/engine/scoring.js";
import type { ScoreReport } from "../core/engine/scoring.js";

import { runAnalysisAgent, extractRuleScores } from "./analysis-agent.js";
import { runEvaluationAgent } from "./evaluation-agent.js";
Expand Down Expand Up @@ -89,7 +89,7 @@ const ELIGIBLE_NODE_TYPES: Set<AnalysisNodeType> = new Set([
"INSTANCE",
]);

import { isExcludedName } from "@/core/rules/excluded-names.js";
import { isExcludedName } from "../core/rules/excluded-names.js";

/**
* Filter node summaries to meaningful conversion candidates.
Expand Down Expand Up @@ -147,7 +147,7 @@ export function filterConversionCandidates(
}

// Reuse loader from core engine
import { loadFile as coreLoadFile } from "@/core/engine/loader.js";
import { loadFile as coreLoadFile } from "../core/engine/loader.js";

async function loadFile(
input: string,
Expand Down
4 changes: 2 additions & 2 deletions src/agents/report-generator.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import type { ScoreReport, CategoryScoreResult } from "@/core/engine/scoring.js";
import type { Category } from "@/core/contracts/category.js";
import type { ScoreReport, CategoryScoreResult } from "../core/engine/scoring.js";
import type { Category } from "../core/contracts/category.js";
import type { MismatchCase } from "./contracts/evaluation-agent.js";
import type { ScoreAdjustment, NewRuleProposal } from "./contracts/tuning-agent.js";
import {
Expand Down
2 changes: 1 addition & 1 deletion src/agents/report-generator.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type { ScoreReport } from "@/core/engine/scoring.js";
import type { ScoreReport } from "../core/engine/scoring.js";
import type { MismatchCase } from "./contracts/evaluation-agent.js";
import type { ScoreAdjustment, NewRuleProposal } from "./contracts/tuning-agent.js";

Expand Down
133 changes: 133 additions & 0 deletions src/agents/run-directory.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ import {
listCalibrationRuns,
extractAppliedRuleIds,
isConverged,
resolveLatestRunDir,
checkConvergence,
} from "./run-directory.js";

describe("extractFixtureName", () => {
Expand Down Expand Up @@ -202,6 +204,137 @@ describe("isConverged", () => {
});
});

describe("resolveLatestRunDir", () => {
const origCwd = process.cwd();
let tempDir: string;

beforeEach(() => {
tempDir = mkdtempSync(join(tmpdir(), "run-dir-test-"));
process.chdir(tempDir);
});

afterEach(async () => {
process.chdir(origCwd);
await rm(tempDir, { recursive: true, force: true });
});

it("returns latest run directory for a fixture", () => {
createCalibrationRunDir("my-fixture");
const dir2 = createCalibrationRunDir("my-fixture");
createCalibrationRunDir("other-fixture");

const latest = resolveLatestRunDir("my-fixture");
expect(latest).toBe(dir2);
});

it("returns null when no matching runs exist", () => {
createCalibrationRunDir("other-fixture");
expect(resolveLatestRunDir("nonexistent")).toBeNull();
});

it("returns null when no runs at all", () => {
expect(resolveLatestRunDir("anything")).toBeNull();
});
});

describe("checkConvergence", () => {
let tempDir: string;

beforeEach(() => {
tempDir = mkdtempSync(join(tmpdir(), "converge-test-"));
});

afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});

it("returns detailed summary with decision counts", () => {
writeFileSync(
join(tempDir, "debate.json"),
JSON.stringify({
arbitrator: {
summary: "test",
decisions: [
{ ruleId: "a", decision: "applied" },
{ ruleId: "b", decision: "revised" },
{ ruleId: "c", decision: "rejected" },
{ ruleId: "d", decision: "kept" },
],
},
}),
);
const summary = checkConvergence(tempDir);
expect(summary.converged).toBe(false);
expect(summary.mode).toBe("strict");
expect(summary.applied).toBe(1);
expect(summary.revised).toBe(1);
expect(summary.rejected).toBe(1);
expect(summary.kept).toBe(1);
expect(summary.total).toBe(4);
expect(summary.reason).toContain("not converged");
expect(summary.reason).toContain("1 applied");
expect(summary.reason).toContain("1 revised");
});

it("strict: not converged with rejections only", () => {
writeFileSync(
join(tempDir, "debate.json"),
JSON.stringify({
arbitrator: {
summary: "test",
decisions: [{ ruleId: "x", decision: "rejected" }],
},
}),
);
const summary = checkConvergence(tempDir);
expect(summary.converged).toBe(false);
expect(summary.mode).toBe("strict");
});

it("lenient: converged with rejections only", () => {
writeFileSync(
join(tempDir, "debate.json"),
JSON.stringify({
arbitrator: {
summary: "test",
decisions: [{ ruleId: "x", decision: "rejected" }],
},
}),
);
const summary = checkConvergence(tempDir, { lenient: true });
expect(summary.converged).toBe(true);
expect(summary.mode).toBe("lenient");
expect(summary.reason).toContain("converged");
expect(summary.reason).toContain("lenient");
});

it("converged when skipped", () => {
writeFileSync(
join(tempDir, "debate.json"),
JSON.stringify({ skipped: "zero proposals" }),
);
const summary = checkConvergence(tempDir);
expect(summary.converged).toBe(true);
expect(summary.reason).toBe("zero proposals");
});

it("not converged when no debate.json", () => {
const summary = checkConvergence(tempDir);
expect(summary.converged).toBe(false);
expect(summary.reason).toBe("no debate.json found");
});

it("not converged when no arbitrator", () => {
writeFileSync(
join(tempDir, "debate.json"),
JSON.stringify({ critic: null }),
);
const summary = checkConvergence(tempDir);
expect(summary.converged).toBe(false);
expect(summary.reason).toBe("no arbitrator result");
});
});

describe("listCalibrationRuns", () => {
const origCwd = process.cwd();
let tempDir: string;
Expand Down
67 changes: 67 additions & 0 deletions src/agents/run-directory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,73 @@ export function extractAppliedRuleIds(debate: DebateResult): string[] {
.filter((id) => id.length > 0);
}

/**
* Resolve the latest calibration run directory for a given fixture name.
* Searches `logs/calibration/<name>--*` and returns the most recent (last sorted).
* Returns null if no matching run exists.
*/
export function resolveLatestRunDir(fixtureName: string): string | null {
const runs = listCalibrationRuns();
const matching = runs.filter((runPath) => {
const dirName = basename(runPath);
const parsed = parseRunDirName(dirName);
return parsed.name === fixtureName;
});
return matching.length > 0 ? matching[matching.length - 1]! : null;
}

/** Convergence summary with decision counts. */
export interface ConvergenceSummary {
converged: boolean;
mode: "strict" | "lenient";
applied: number;
revised: number;
rejected: number;
kept: number;
total: number;
reason: string;
}

/**
* Check convergence and return a detailed summary with decision counts.
*/
export function checkConvergence(runDir: string, options?: ConvergenceOptions): ConvergenceSummary {
const mode = options?.lenient ? "lenient" : "strict";
const debate = parseDebateResult(runDir);

if (!debate) {
return { converged: false, mode, applied: 0, revised: 0, rejected: 0, kept: 0, total: 0, reason: "no debate.json found" };
}
if (debate.skipped) {
return { converged: true, mode, applied: 0, revised: 0, rejected: 0, kept: 0, total: 0, reason: debate.skipped };
}
if (!debate.arbitrator) {
return { converged: false, mode, applied: 0, revised: 0, rejected: 0, kept: 0, total: 0, reason: "no arbitrator result" };
}

const decisions = debate.arbitrator.decisions;
const applied = decisions.filter((d) => d.decision.trim().toLowerCase() === "applied").length;
const revised = decisions.filter((d) => d.decision.trim().toLowerCase() === "revised").length;
const rejected = decisions.filter((d) => d.decision.trim().toLowerCase() === "rejected").length;
const kept = decisions.length - applied - revised - rejected;
const total = decisions.length;

const converged = options?.lenient
? (applied + revised) === 0
: (applied + revised) === 0 && rejected === 0;

const parts: string[] = [];
if (applied > 0) parts.push(`${applied} applied`);
if (revised > 0) parts.push(`${revised} revised`);
if (rejected > 0) parts.push(`${rejected} rejected`);
if (kept > 0) parts.push(`${kept} kept`);
const countsStr = parts.length > 0 ? parts.join(", ") : "no decisions";
const verdict = converged ? "converged" : "not converged";
const reason = `${verdict} (${mode}) — ${countsStr} (${total} total)`;

return { converged, mode, applied, revised, rejected, kept, total, reason };
}

/** Options for convergence checking. */
export interface ConvergenceOptions {
/**
Expand Down
2 changes: 1 addition & 1 deletion src/agents/tuning-agent.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type { Severity } from "@/core/contracts/severity.js";
import type { Severity } from "../core/contracts/severity.js";
import type { Confidence } from "./contracts/tuning-agent.js";
import type {
TuningAgentInput,
Expand Down
Loading
Loading