Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 21 additions & 1 deletion src/agent-runtime/procedural-evolution.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ export interface ProceduralCandidateDecision {
readonly baselineDigest: string;
readonly candidateDigest: string;
readonly contextDigest: string;
readonly baselineReceiptDigest: string;
readonly candidateReceiptDigest: string;
readonly rejectionKey: string;
readonly baselineMean: number;
readonly candidateMean: number;
Expand Down Expand Up @@ -61,11 +63,27 @@ function observations(input: unknown, graphDigest: string, contextDigest: string
return holdout.map(id => byId.get(id)!);
}

async function evaluationReceiptDigest(
graphDigest: string,
contextDigest: string,
holdout: readonly string[],
rows: readonly Observation[],
): Promise<string> {
return proceduralHash([
"noema.procedural-evaluation-receipt/v1",
graphDigest,
contextDigest,
holdout.map((caseId, index) => [caseId, rows[index].score, rows[index].safetyViolations]),
]);
}

/**
* Screens supplied paired held-out evidence for a direct child graph while keeping activation and
* publication outside this pure port. The validator owner must authenticate receipts and pre-register
* the evaluation context; this function checks exact identities, complete paired cases, leakage,
* finite normalized scores, reported safety violations, mean non-regression, and contextual rejection.
* Returned receipt digests bind the exact validated paired evidence semantics for later authentication
* and durable retention, but a digest alone does not authenticate its producer or authorize approval.
* @param input Exact-key baseline, candidate, evaluation plan, paired receipts, and prior rejection keys.
* @returns Promise resolving to a frozen non-authoritative screening decision with activation always false.
*/
Expand All @@ -84,6 +102,8 @@ export async function assessProceduralCandidate(input: unknown): Promise<Procedu
if (holdout.some(id => training.has(id))) rejectProceduralInput("holdout_leakage");
const oldRows = observations(value.baselineReceipt, baseline.digest, contextDigest, holdout);
const newRows = observations(value.candidateReceipt, candidate.digest, contextDigest, holdout);
const baselineReceiptDigest = await evaluationReceiptDigest(baseline.digest, contextDigest, holdout, oldRows);
const candidateReceiptDigest = await evaluationReceiptDigest(candidate.digest, contextDigest, holdout, newRows);
const rejectedKeys = new Set(readProceduralArray(value.rejectedKeys, 0, 10_000).map(proceduralDigest));
const baselineMean = oldRows.reduce((total, row) => total + row.score, 0) / holdout.length;
const candidateMean = newRows.reduce((total, row) => total + row.score, 0) / holdout.length;
Expand All @@ -94,7 +114,7 @@ export async function assessProceduralCandidate(input: unknown): Promise<Procedu
else if (newRows.some(row => row.safetyViolations > 0)) reason = "safety_violation";
else if (candidateMean < baselineMean) reason = "score_regression";
else reason = "validation_non_regression";
const decision = Object.freeze({eligibleForApproval: reason === "validation_non_regression", activationAuthorized: false as const, reason, baselineDigest: baseline.digest, candidateDigest: candidate.digest, contextDigest, rejectionKey, baselineMean, candidateMean});
const decision = Object.freeze({eligibleForApproval: reason === "validation_non_regression", activationAuthorized: false as const, reason, baselineDigest: baseline.digest, candidateDigest: candidate.digest, contextDigest, baselineReceiptDigest, candidateReceiptDigest, rejectionKey, baselineMean, candidateMean});
admittedCandidateDecisions.add(decision);
return decision;
} catch (error) { return normalizeProceduralError(error); }
Expand Down
77 changes: 77 additions & 0 deletions test/procedural-evidence-identity.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
import { test } from "vitest";
import assert from "node:assert/strict";
import { createProceduralGraph } from "../src/agent-runtime/procedural-graph.ts";
import { assessProceduralCandidate } from "../src/agent-runtime/procedural-evolution.ts";

const raw = () => ({
schemaVersion: "noema.procedural-graph/v1",
tenantId: "tenant-a",
taskType: "repair",
graphId: "graph-a",
revision: 1,
parentDigest: null,
nodes: ["Start", "check"],
edges: [{
from: "Start",
relation: "requires",
to: "check",
condition: "",
guidance: "Check evidence",
pitfalls: "",
}],
});

async function fixture() {
const baseline = await createProceduralGraph(raw());
const candidateInput = raw();
candidateInput.revision = 2;
candidateInput.parentDigest = baseline.digest;
candidateInput.edges[0].guidance = "Check exact-head evidence";
const candidate = await createProceduralGraph(candidateInput);
const contextDigest = "c".repeat(64);
const plan = {
contextDigest,
minimumCases: 2,
trainingCaseIds: ["train-1"],
holdoutCaseIds: ["case-1", "case-2"],
};
const receipt = (graph, firstScore, secondScore) => ({
graphDigest: graph.digest,
contextDigest,
observations: [
{ caseId: "case-1", score: firstScore, safetyViolations: 0 },
{ caseId: "case-2", score: secondScore, safetyViolations: 0 },
],
});
return {
baseline,
candidate,
plan,
baselineReceipt: receipt(baseline, 0.5, 0.6),
candidateReceipt: receipt(candidate, 0.7, 0.8),
rejectedKeys: [],
};
}

test("screening decision binds canonical identities of both paired evaluation receipts", async () => {
const data = await fixture();
const first = await assessProceduralCandidate(data);
assert.match(first.baselineReceiptDigest, /^[0-9a-f]{64}$/);
assert.match(first.candidateReceiptDigest, /^[0-9a-f]{64}$/);
assert.notEqual(first.baselineReceiptDigest, first.candidateReceiptDigest);

data.baselineReceipt.observations.reverse();
data.candidateReceipt.observations.reverse();
const reordered = await assessProceduralCandidate(data);
assert.equal(reordered.baselineReceiptDigest, first.baselineReceiptDigest);
assert.equal(reordered.candidateReceiptDigest, first.candidateReceiptDigest);
});

test("receipt identity changes when validated evaluation evidence changes", async () => {
const data = await fixture();
const first = await assessProceduralCandidate(data);
data.candidateReceipt.observations[0].score = 0.71;
const changed = await assessProceduralCandidate(data);
assert.notEqual(changed.candidateReceiptDigest, first.candidateReceiptDigest);
assert.equal(changed.baselineReceiptDigest, first.baselineReceiptDigest);
});
Loading