From 892510d3a41f0c434ad393968603933c89071eef Mon Sep 17 00:00:00 2001 From: Sophie Neumann Date: Mon, 10 Aug 2026 10:29:06 +0200 Subject: [PATCH] feat(ci): golden-set regression eval for LLM verifier behaviour (#129) Frozen corpus run through a real VerifierPipeline (real ClaimExtractor + EvidenceJudge on the pinned VERIFIER_MODEL, fixture-backed sources) that fails when a known-good input regresses to a worse verdict class. Asserts the verdict class (approved / approved_with_disclaimer / blocked), the stable signal despite generation stochasticity. Flake policy: re-run a first-sample miss up to 3x, decide by majority. New golden-eval.yml runs on main push + dispatch + weekly cron (not per-PR, cost), gated on ANTHROPIC_API_KEY and skips cleanly when absent. - Split harness into a pure layer (goldenRunner.ts, type-only verifier import, key- and build-free) and a model layer (goldenModel.ts, real pipeline wiring) - goldenModel.test.ts drives the real pipeline to blocked/approved key-free via a stub LlmProvider, so the wiring is load-bearing in CI, not decorative - typecheck:golden (chained into typecheck) puts test/golden under tsc, which test/ otherwise escapes; caught pre-existing noUncheckedIndexedAccess bugs - Corpus: every judge-dependent answer carries a hard trigger signal, else shouldTriggerVerifier skips extraction and the entry silently approves - README documents the trigger trap and the add-a-corpus-entry-for-a-new-agent procedure; v2 (deterministic source + full-turn eval) tracked in #639 Refs #130, #131, #132. Follow-up #639. --- .github/workflows/golden-eval.yml | 86 ++++++ middleware/package.json | 4 +- middleware/test/golden/README.md | 202 ++++++++++++++ middleware/test/golden/corpus/approve.jsonl | 10 + .../test/golden/corpus/blocked-citation.jsonl | 7 + .../golden/corpus/blocked-contradiction.jsonl | 13 + .../corpus/blocked-tool-postcondition.jsonl | 5 + .../test/golden/corpus/disclaimer.jsonl | 13 + middleware/test/golden/goldenModel.ts | 107 ++++++++ middleware/test/golden/goldenRunner.ts | 257 ++++++++++++++++++ middleware/test/golden/goldenSet.eval.ts | 107 ++++++++ middleware/test/golden/tsconfig.json | 16 ++ middleware/test/goldenModel.test.ts | 135 +++++++++ middleware/test/goldenRunner.test.ts | 204 ++++++++++++++ 14 files changed, 1165 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/golden-eval.yml create mode 100644 middleware/test/golden/README.md create mode 100644 middleware/test/golden/corpus/approve.jsonl create mode 100644 middleware/test/golden/corpus/blocked-citation.jsonl create mode 100644 middleware/test/golden/corpus/blocked-contradiction.jsonl create mode 100644 middleware/test/golden/corpus/blocked-tool-postcondition.jsonl create mode 100644 middleware/test/golden/corpus/disclaimer.jsonl create mode 100644 middleware/test/golden/goldenModel.ts create mode 100644 middleware/test/golden/goldenRunner.ts create mode 100644 middleware/test/golden/goldenSet.eval.ts create mode 100644 middleware/test/golden/tsconfig.json create mode 100644 middleware/test/goldenModel.test.ts create mode 100644 middleware/test/goldenRunner.test.ts diff --git a/.github/workflows/golden-eval.yml b/.github/workflows/golden-eval.yml new file mode 100644 index 00000000..a250a543 --- /dev/null +++ b/.github/workflows/golden-eval.yml @@ -0,0 +1,86 @@ +# Golden-set behaviour regression eval (#129). +# +# CI's `ci.yml` verifies plumbing (lint / typecheck / unit tests) but never +# tests LLM behaviour against the pinned model — a model bump or prompt edit can +# silently regress agent quality (LLM weakness #13, version drift). This job +# runs a frozen corpus through a real VerifierPipeline and fails when a +# known-good input regresses to a worse verifier verdict class. +# +# NOT run on pull_request: it spends real Anthropic tokens, so it runs on main +# pushes + manual dispatch + a weekly cron (cost vs. signal). Because it never +# triggers on PRs, forks never attempt it. It is gated on the ANTHROPIC_API_KEY +# secret and skips cleanly with a notice when that secret is absent, so the job +# stays green on a repo where the key is not yet configured. + +name: golden-eval + +on: + push: + branches: [main] + workflow_dispatch: + schedule: + # Mondays 06:00 UTC — a weekly drift canary independent of merge cadence. + - cron: '0 6 * * 1' + +concurrency: + group: golden-eval-${{ github.ref }} + # Do NOT cancel a running eval when the next commit lands on main: this is a + # per-commit drift canary, and cancelling would leave the earlier commit range + # with no regression signal (the exact gap #129 exists to close). Back-to-back + # merges queue instead of racing; the cost is bounded by corpus size. Manual + # dispatch / cron on the same ref still serialise behind a running job. + cancel-in-progress: false + +permissions: + contents: read + +jobs: + golden: + name: golden-set verdict regression (#129) + runs-on: ubuntu-latest + timeout-minutes: 20 + defaults: + run: + working-directory: middleware + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + steps: + # Guard first so a repo without the secret does zero expensive work. + - name: Check ANTHROPIC_API_KEY is configured + id: guard + working-directory: . + run: | + if [ -z "${ANTHROPIC_API_KEY:-}" ]; then + echo "::notice::ANTHROPIC_API_KEY secret not configured — skipping golden-set eval (#129)." + echo "run=false" >> "$GITHUB_OUTPUT" + else + echo "run=true" >> "$GITHUB_OUTPUT" + fi + + - if: steps.guard.outputs.run == 'true' + uses: actions/checkout@v7 + + - if: steps.guard.outputs.run == 'true' + uses: actions/setup-node@v7 + with: + node-version: '22' + cache: 'npm' + cache-dependency-path: middleware/package-lock.json + + - name: npm ci (middleware + workspaces) + if: steps.guard.outputs.run == 'true' + run: npm ci --include=optional --no-audit --no-fund + + # sharp's native binary is platform-specific and skipped by npm ci on the + # linux runner (lockfile generated on darwin-arm64). Mirror ci.yml. + - name: Install sharp linux-x64 native binary + if: steps.guard.outputs.run == 'true' + run: npm install --no-save --no-audit --no-fund --os=linux --cpu=x64 sharp + + - name: Build workspace packages (@omadia/verifier + adapter dist/) + if: steps.guard.outputs.run == 'true' + run: npm run build + + - name: Run golden-set eval + if: steps.guard.outputs.run == 'true' + run: npm run eval:golden diff --git a/middleware/package.json b/middleware/package.json index d4a7c23c..99c60f24 100644 --- a/middleware/package.json +++ b/middleware/package.json @@ -27,7 +27,8 @@ "ensure-native-abi": "node scripts/ensure-native-abi.mjs", "lint": "eslint src/ packages/plugin-api/src/ packages/llm-provider-api/src/ packages/llm-provider/src/ packages/llm-adapter-anthropic/src/ packages/llm-adapter-openai/src/ packages/harness-ui-helpers/src/ packages/harness-api-key-auth/src/ packages/harness-channel-sdk/src/ packages/harness-diagrams/src/ packages/harness-memory/src/ packages/harness-memory-postgres/src/ packages/harness-embeddings/src/ packages/embedding-adapter-openai/src/ packages/harness-knowledge-graph-inmemory/src/ packages/harness-knowledge-graph-neon/src/ packages/harness-usage-telemetry/src/ packages/harness-orchestrator-extras/src/ packages/harness-verifier/src/ packages/harness-orchestrator/src/ packages/harness-plugin-web-search/src/ packages/harness-plugin-quality-guard/src/ packages/harness-plugin-privacy-guard/src/ packages/harness-plugin-office/src/ packages/omadia-ui-orchestrator/src/ packages/omadia-ui-channel/src/ packages/harness-channel-api/src/ packages/harness-plugin-plan-runner/src/", "lint:fix": "eslint src/ packages/plugin-api/src/ packages/llm-provider-api/src/ packages/llm-provider/src/ packages/llm-adapter-anthropic/src/ packages/llm-adapter-openai/src/ packages/harness-ui-helpers/src/ packages/harness-api-key-auth/src/ packages/harness-channel-sdk/src/ packages/harness-diagrams/src/ packages/harness-memory/src/ packages/harness-memory-postgres/src/ packages/harness-embeddings/src/ packages/embedding-adapter-openai/src/ packages/harness-knowledge-graph-inmemory/src/ packages/harness-knowledge-graph-neon/src/ packages/harness-usage-telemetry/src/ packages/harness-orchestrator-extras/src/ packages/harness-verifier/src/ packages/harness-orchestrator/src/ packages/harness-plugin-web-search/src/ packages/harness-plugin-quality-guard/src/ packages/harness-plugin-privacy-guard/src/ packages/harness-plugin-office/src/ packages/omadia-ui-orchestrator/src/ packages/omadia-ui-channel/src/ packages/harness-channel-api/src/ packages/harness-plugin-plan-runner/src/ --fix", - "typecheck": "npm run typecheck -w @omadia/plugin-api && npm run typecheck -w @omadia/llm-provider-api && npm run typecheck -w @omadia/llm-provider && npm run typecheck -w @omadia/llm-adapter-anthropic && npm run typecheck -w @omadia/llm-adapter-openai && npm run typecheck -w @omadia/canvas-core && npm run typecheck -w @omadia/conductor-core && npm run typecheck -w @omadia/plugin-ui-helpers && npm run typecheck -w @omadia/api-key-auth && npm run typecheck -w @omadia/channel-sdk && npm run typecheck -w @omadia/diagrams && npm run typecheck -w @omadia/memory && npm run typecheck -w @omadia/memory-postgres && npm run typecheck -w @omadia/embeddings && npm run typecheck -w @omadia/embedding-adapter-openai && npm run typecheck -w @omadia/knowledge-graph-inmemory && npm run typecheck -w @omadia/knowledge-graph-neon && npm run typecheck -w @omadia/orchestrator-extras && npm run typecheck -w @omadia/verifier && npm run typecheck -w @omadia/plugin-privacy-guard && npm run typecheck -w @omadia/orchestrator && npm run typecheck -w @omadia/ui-orchestrator && npm run typecheck -w @omadia/ui-channel && npm run typecheck -w @omadia/channel-api && npm run typecheck -w @omadia/plugin-office && npm run typecheck -w @omadia/plugin-web-search && npm run typecheck -w @omadia/plugin-quality-guard && npm run typecheck -w @omadia/agent-seo-analyst && npm run typecheck -w @omadia/agent-reference-maximum && npm run typecheck -w @omadia/plugin-plan-runner && tsc --noEmit", + "typecheck": "npm run typecheck -w @omadia/plugin-api && npm run typecheck -w @omadia/llm-provider-api && npm run typecheck -w @omadia/llm-provider && npm run typecheck -w @omadia/llm-adapter-anthropic && npm run typecheck -w @omadia/llm-adapter-openai && npm run typecheck -w @omadia/canvas-core && npm run typecheck -w @omadia/conductor-core && npm run typecheck -w @omadia/plugin-ui-helpers && npm run typecheck -w @omadia/api-key-auth && npm run typecheck -w @omadia/channel-sdk && npm run typecheck -w @omadia/diagrams && npm run typecheck -w @omadia/memory && npm run typecheck -w @omadia/memory-postgres && npm run typecheck -w @omadia/embeddings && npm run typecheck -w @omadia/embedding-adapter-openai && npm run typecheck -w @omadia/knowledge-graph-inmemory && npm run typecheck -w @omadia/knowledge-graph-neon && npm run typecheck -w @omadia/orchestrator-extras && npm run typecheck -w @omadia/verifier && npm run typecheck -w @omadia/plugin-privacy-guard && npm run typecheck -w @omadia/orchestrator && npm run typecheck -w @omadia/ui-orchestrator && npm run typecheck -w @omadia/ui-channel && npm run typecheck -w @omadia/channel-api && npm run typecheck -w @omadia/plugin-office && npm run typecheck -w @omadia/plugin-web-search && npm run typecheck -w @omadia/plugin-quality-guard && npm run typecheck -w @omadia/agent-seo-analyst && npm run typecheck -w @omadia/agent-reference-maximum && npm run typecheck -w @omadia/plugin-plan-runner && tsc --noEmit && npm run typecheck:golden", + "typecheck:golden": "tsc -p test/golden/tsconfig.json", "format": "prettier --write \"src/**/*.ts\"", "format:check": "prettier --check \"src/**/*.ts\"", "smoke:entity-refs": "tsx scripts/smoke-entity-refs.ts", @@ -36,6 +37,7 @@ "smoke:diagrams": "tsx scripts/smoke-diagrams.ts", "smoke:privacy-v2": "tsx scripts/smoke-privacy-v2.ts", "smoke:package-roundtrip": "tsx scripts/smoke-package-roundtrip.ts", + "eval:golden": "node --import tsx test/golden/goldenSet.eval.ts", "setup:tigris-lifecycle": "tsx scripts/setup-tigris-lifecycle.ts", "pretest": "node scripts/check-node-version.mjs", "test": "node --import tsx --test --test-timeout=120000 --test-reporter=spec 'test/**/*.test.ts'" diff --git a/middleware/test/golden/README.md b/middleware/test/golden/README.md new file mode 100644 index 00000000..b9380d96 --- /dev/null +++ b/middleware/test/golden/README.md @@ -0,0 +1,202 @@ +# Golden-set regression eval (#129) + +A behaviour-level regression gate for the verifier's stochastic LLM stages. CI's +`middleware` job proves the plumbing (lint / typecheck / unit tests); this proves +that the **pinned model still classifies known inputs into the same verifier +verdict class**. A model bump or a prompt edit that silently regresses agent +behaviour (LLM weakness #13, version drift) fails here instead of shipping. + +## What it asserts + +The assertion target is **not** the raw model string — it is the verifier +verdict *class*, which is stable despite generation stochasticity: + +- `approved` +- `approved_with_disclaimer` (the borderline path, `isBorderlineVerdict`) +- `blocked` — including the two synthetic claim paths: + - `tool_postcondition` (#130) + - `citation_missing` (#131) + +Each corpus entry is a frozen `(userMessage, answer, trace fields, fixture +evidence, expected status)` tuple run through a **real** `VerifierPipeline` — +real `ClaimExtractor` + real `EvidenceJudge` on the Anthropic adapter — with +**fixture-backed** deterministic sources (no Postgres, no Odoo, no orchestrator). +That keeps the job hermetic and cheap while still exercising the two stochastic +stages that actually drift. + +## Scope (v1) and what is v2 + +- **v1 (this):** verifier-stage eval. The stochastic stages run on + `VERIFIER_MODEL` (default `claude-haiku-4-5-20251001`), so that is the model + pinned here — asserting against the orchestrator model would be dishonest, + those stages never call it. Hard-claim verdicts resolve to `unverified` (no + Odoo/graph reader is wired), so no fixture relies on deterministic hard-claim + verification. Consequence: the judge's `verified → approved` path is **not** + exercised in v1 (a triggering answer's hard claim is always `unverified` → + disclaimer), so v1's `approved` fixtures cover only the trigger-skip path. +- **v2 (not built) — tracked in [#639](https://github.com/byte5ai/omadia/issues/639):** + a deterministic Odoo/graph fixture reader (enables a genuine judge/deterministic + `verified → approved` entry and a deterministic-contradiction `blocked` entry), + and a full-turn eval (`input → live orchestrator answer → verdict`) once a + headless turn runner exists. + +## Running locally + +```bash +export ANTHROPIC_API_KEY=sk-ant-... +npm run eval:golden # runs the whole corpus, exits non-zero on regression +GOLDEN_MODEL=claude-opus-4-8 npm run eval:golden # pin a different model +``` + +Without `ANTHROPIC_API_KEY` the runner **skips with a notice and exits 0** — the +same behaviour CI relies on so an unconfigured repo/fork stays green. `npm test` +never invokes this runner (it lives outside the `test/**/*.test.ts` glob); only +the key-free suites run there: + +- `test/goldenRunner.test.ts` covers the **pure** harness (`goldenRunner.ts`: + parsing, majority voting, flake-tolerant `runEntry`, summary). That module + imports `@omadia/verifier` as `import type` only, so this suite needs no build. +- `test/goldenModel.test.ts` covers the **model** layer (`goldenModel.ts`: the + real `VerifierPipeline` wiring) with a **stub** `LlmProvider`. The synthetic + block paths (`tool_postcondition`, `citation_missing`) need no model, so this + drives the real pipeline to a verdict — and asserts the trace fields are wired + through — without a key. It does need the workspace `dist/` built. + +## Type coverage + +The middleware `lint`/`typecheck` scripts only cover `src/`, so `test/` is +normally unchecked. `npm run typecheck:golden` (chained into `npm run typecheck`, +i.e. run on every PR) type-checks `test/golden/**` + the two suites via +`test/golden/tsconfig.json`, so a drift in `@omadia/verifier`'s types fails a PR +instead of surfacing only at the key-gated eval on `main`. + +## Flake policy + +Verdict classes are stable but not immune to model jitter. An entry that matches +its expected class on the first sample costs **one** model call. An entry that +misses on the first sample is re-run up to **3 samples total** and decided by +majority. Worst-case cost is therefore `3 × corpus size`. Per-entry token cost is +printed in the run summary and the GitHub job summary. + +## CI + +`.github/workflows/golden-eval.yml` runs on push to `main`, manual +`workflow_dispatch`, and a weekly cron — **not** on every PR (cost vs. signal). +It requires the `ANTHROPIC_API_KEY` GitHub Actions secret; when the secret is +absent the job skips cleanly with a notice. Because the workflow never triggers +on `pull_request`, forks never attempt to run it. + +## Adding a corpus entry (do this when you ship a new agent type or verdict path) + +1. Pick the file under `corpus/` that matches the **expected verdict class** + (`approve.jsonl`, `disclaimer.jsonl`, `blocked-citation.jsonl`, + `blocked-tool-postcondition.jsonl`, `blocked-contradiction.jsonl`), or add a + new `*.jsonl` file for a new class. Every `.jsonl` in `corpus/` is picked up + automatically. +2. Append one JSON object per line with this shape: + + ```jsonc + { + "id": "unique_stable_id", // required, unique across the corpus + "note": "why this class is expected", // recommended + "userMessage": "…", // required + "answer": "…", // required — the answer under eval + "trace": { // optional VerifierInput trace fields + "agent": "accounting", + "domainToolsCalled": ["query_odoo_accounting"], + "knowledgeGraphToolsCalled": true, + "toolPostconditionViolations": [ + { "toolName": "…", "callId": "…", "agentContext": "…", "issues": ["…"] } + ] + }, + "evidence": [ // optional fixture evidence for the judge + { "nodeId": "n1", "source": "graph", "content": "…", "title": "…" } + ], + "expected": { "status": "blocked" } // required: approved | approved_with_disclaimer | blocked + } + ``` + +3. **The trigger trap — read this before writing a judge fixture.** The + pipeline runs the stochastic extractor + judge only when + `shouldTriggerVerifier(answer)` fires, and it fires **only on a hard signal in + the answer**: a currency amount (`1.234,56 €`), a date (`01.03.2023` / + `2026-04-19`), an accounting ref (`INV/2026/0042`), a percentage, or a + duration (`12 Urlaubstage`) — an aggregate keyword next to a 3+ digit number + also counts. **A soft/qualitative claim alone never triggers.** If the answer + has no signal, `verify()` skips extraction and returns `approved` *before the + judge runs* — so a `disclaimer`/`contradiction` fixture whose answer lacks a + signal silently resolves to `approved` and fails. Every judge-dependent + fixture in this corpus therefore embeds a date or an amount in the answer; + keep that up. (Verify with the trigger check pattern used in review, or just + run `eval:golden`.) +4. **Author for robustness.** Prefer entries where the expected class does not + hinge on brittle extractor jitter: + - `blocked` via `trace.toolPostconditionViolations` or + `trace.knowledgeGraphToolsCalled=true` + no `[ref:]` marker is synthetic and + robust — the extractor need not cooperate, and no trigger signal is needed + (these fire independent of `shouldTriggerVerifier`). + - `approved_with_disclaimer` needs a soft (qualitative/name) claim in the + answer **plus a trigger signal**, with `evidence: []` (or on-topic-but-silent + evidence) → judge returns `unverified`. The signal's own hard claim also + resolves `unverified` (no Odoo reader), consistent with the outcome. + - `blocked` via judge contradiction needs a **trigger signal** plus evidence + that **explicitly** states something incompatible (the judge only + contradicts on explicit conflict); a contradiction dominates the co-extracted + `unverified` hard claim, so the verdict is `blocked`. + - `approved` is reachable in v1 **only** via the trigger-skip path (no signal → + extraction skipped). A judge-*verified* approve is not achievable in v1: any + triggering answer carries a hard claim that resolves `unverified` → disclaimer. + That path is v2 (needs the deterministic Odoo/graph reader). +5. Lines starting with `#` and blank lines are ignored — use them for section + headers. +6. Validate the shape without spending tokens: `npm test` runs the parser over + the harness unit suite. To smoke the real classification, run + `npm run eval:golden` with a key. + +## Adding coverage when a new agent type ships + +A new domain agent (a new Odoo-backed agent, a new tool surface, etc.) brings a +new way its output can regress — most importantly its **tool-postcondition** +contract. Add at least one fixture per new agent so a regression in that agent's +verification is caught, not just the agents that existed when the corpus was +written. The robust, model-independent choice is the synthetic +`tool_postcondition` path (it needs no trigger signal and no judge): + +1. Add one line to `corpus/blocked-tool-postcondition.jsonl`: + + ```jsonc + { + "id": "blocked_tool_postcondition_", // e.g. blocked_tool_postcondition_projects + "note": " tool returned a payload that failed its output-schema postcondition.", + "userMessage": "…a question that routes to the new agent…", + "answer": "…the agent's answer (a trigger signal is NOT required for this path)…", + "trace": { + "agent": "", // the new agent's id + "domainToolsCalled": [""], // the tool it calls + "toolPostconditionViolations": [ + { "toolName": "", "callId": "call_1", "agentContext": "", + "issues": [""] } + ] + }, + "expected": { "status": "blocked" } + } + ``` + + This blocks via a synthetic contradiction built directly from + `toolPostconditionViolations`, independent of the model — so it is stable the + day the agent ships, before you have a feel for how its answers read. + +2. If the agent cites the knowledge graph, also add a `citation_missing` line to + `corpus/blocked-citation.jsonl` (set `trace.knowledgeGraphToolsCalled: true` + and leave the answer without a `[ref:nodeId]` marker) — likewise synthetic and + robust. + +3. Only once you want to pin the agent's *judge* behaviour, add a + `disclaimer` / `contradiction` fixture — and then obey the **trigger trap** + above (the answer must carry a hard signal) and confirm the expected class on a + real `eval:golden` run. + +4. The `agent` id in `trace.agent` is free-form here (it flows into + `VerifierInput.agent`); use the same id the orchestrator tags that agent with, + so a `git grep` for the agent id turns up both its code and its golden + coverage. diff --git a/middleware/test/golden/corpus/approve.jsonl b/middleware/test/golden/corpus/approve.jsonl new file mode 100644 index 00000000..7f8e2c7c --- /dev/null +++ b/middleware/test/golden/corpus/approve.jsonl @@ -0,0 +1,10 @@ +# Expected verdict class: approved. In v1 this class is reachable ONLY via the +# trigger-skip path: shouldTriggerVerifier fires only on hard signals (currency / +# date / accounting-ref / percent / duration), so an answer without one skips +# extraction and approves BEFORE the judge runs. A judge-*verified* approve is +# not reachable in v1 — any triggering answer also carries a hard claim that +# resolves `unverified` with no Odoo/graph reader, which lands it in disclaimer, +# not clean approved. See README ("Scope (v1) and what is v2"). +{"id":"approve_smalltalk_hallo","note":"Pure smalltalk; no trigger signal, so shouldTriggerVerifier skips extraction entirely.","userMessage":"Hallo!","answer":"Hallo! Wie kann ich dir heute helfen?","expected":{"status":"approved"}} +{"id":"approve_smalltalk_danke","note":"Acknowledgement only; no factual claims and no trigger signal.","userMessage":"Danke, das war alles.","answer":"Gern geschehen! Melde dich jederzeit, wenn du noch etwas brauchst.","expected":{"status":"approved"}} +{"id":"approve_qualitative_no_trigger","note":"A qualitative claim but NO hard trigger signal in the answer -> extraction skipped -> approved. The judge never runs (evidence would be irrelevant, so none is given). Guards that plain qualitative statements are not over-verified into a disclaimer.","userMessage":"Wer leitet die Buchhaltung?","answer":"Anna Müller ist die Teamleiterin der Buchhaltung.","expected":{"status":"approved"}} diff --git a/middleware/test/golden/corpus/blocked-citation.jsonl b/middleware/test/golden/corpus/blocked-citation.jsonl new file mode 100644 index 00000000..951a9a50 --- /dev/null +++ b/middleware/test/golden/corpus/blocked-citation.jsonl @@ -0,0 +1,7 @@ +# Expected verdict class: blocked via the citation_missing claim path (#131). +# knowledgeGraphToolsCalled=true AND the answer carries no [ref:nodeId] marker +# => synthetic citation_missing contradiction => blocked. Robust to extraction +# jitter because the synthetic claim does not depend on the extractor. +{"id":"blocked_citation_kg_no_ref","note":"KG was queried but the answer cites nothing.","userMessage":"Wer ist der Ansprechpartner für Kunde ACME?","answer":"Der Ansprechpartner für ACME ist Julia Berg aus dem Key-Account-Team.","trace":{"knowledgeGraphToolsCalled":true},"expected":{"status":"blocked"}} +{"id":"blocked_citation_kg_no_ref_2","note":"Second KG-backed answer without any [ref:] marker.","userMessage":"Welche Projekte laufen aktuell mit ACME?","answer":"Aktuell laufen zwei Projekte mit ACME: eine Portal-Migration und ein Support-Retainer.","trace":{"knowledgeGraphToolsCalled":true},"expected":{"status":"blocked"}} +{"id":"blocked_citation_kg_no_ref_3","note":"KG-backed HR answer with no citation marker.","userMessage":"Wer ist der Vorgesetzte von Lena Fischer?","answer":"Der Vorgesetzte von Lena Fischer ist Markus Klein, Leiter der ERP-Abteilung.","trace":{"knowledgeGraphToolsCalled":true},"expected":{"status":"blocked"}} diff --git a/middleware/test/golden/corpus/blocked-contradiction.jsonl b/middleware/test/golden/corpus/blocked-contradiction.jsonl new file mode 100644 index 00000000..2d829db6 --- /dev/null +++ b/middleware/test/golden/corpus/blocked-contradiction.jsonl @@ -0,0 +1,13 @@ +# Expected verdict class: blocked via a judge contradiction. A soft claim whose +# fixture evidence explicitly states something incompatible -> judge returns +# `contradicted` -> blocked. This is the ONLY corpus path that exercises the +# stochastic judge on its strongest signal (per the judge prompt, 'contradicted +# ONLY when the evidence explicitly states something incompatible'). +# +# CRITICAL: each answer carries a hard trigger signal (a date or currency +# amount) so shouldTriggerVerifier fires and the extractor + judge actually run; +# without a signal the pipeline skips straight to `approved`. The co-extracted +# hard claim resolves `unverified`, but a contradiction dominates unverified in +# the aggregation, so the verdict is `blocked`. +{"id":"blocked_contradiction_role","note":"Date signal triggers; qualitative claim (IT-Abteilung) explicitly contradicted by evidence ('war nie in der IT-Abteilung') -> contradicted -> blocked.","userMessage":"In welcher Abteilung arbeitet Anna Müller?","answer":"Anna Müller wechselte am 01.03.2023 in die IT-Abteilung [ref:n_emp_anna].","evidence":[{"nodeId":"n_emp_anna","source":"graph","title":"Anna Müller","content":"Anna Müller ist Teamleiterin der Abteilung Buchhaltung. Sie war nie in der IT-Abteilung tätig."}],"expected":{"status":"blocked"}} +{"id":"blocked_contradiction_status","note":"Currency signal triggers; claim of an active contract explicitly contradicted by evidence (contract terminated) -> contradicted -> blocked.","userMessage":"Ist der Vertrag mit Kunde ACME noch aktiv?","answer":"Der Vertrag mit ACME ist mit einem Volumen von 250.000 € weiterhin aktiv [ref:n_contract_acme].","evidence":[{"nodeId":"n_contract_acme","source":"graph","title":"Vertrag ACME","content":"Der Vertrag mit ACME wurde zum 31.12.2025 gekündigt und ist seitdem beendet."}],"expected":{"status":"blocked"}} diff --git a/middleware/test/golden/corpus/blocked-tool-postcondition.jsonl b/middleware/test/golden/corpus/blocked-tool-postcondition.jsonl new file mode 100644 index 00000000..d6740fc6 --- /dev/null +++ b/middleware/test/golden/corpus/blocked-tool-postcondition.jsonl @@ -0,0 +1,5 @@ +# Expected verdict class: blocked via the tool_postcondition claim path (#130). +# A recorded tool-postcondition violation in the trace => synthetic +# tool_postcondition contradiction => blocked, independent of the model output. +{"id":"blocked_tool_postcondition_schema","note":"Domain tool returned a payload that failed its output-schema postcondition.","userMessage":"Wie hoch ist der offene Betrag von Rechnung INV/2026/0042?","answer":"Der offene Betrag beträgt 1.234,56 €.","trace":{"agent":"accounting","domainToolsCalled":["query_odoo_accounting"],"toolPostconditionViolations":[{"toolName":"query_odoo_accounting","callId":"call_1","agentContext":"accounting","issues":["result.amount_residual missing from tool output"]}]},"expected":{"status":"blocked"}} +{"id":"blocked_tool_postcondition_hr","note":"HR leave tool violated its postcondition (negative balance).","userMessage":"Wie viele Urlaubstage hat Anna Müller noch?","answer":"Anna Müller hat noch 12 Urlaubstage übrig.","trace":{"agent":"hr","domainToolsCalled":["query_odoo_hr"],"toolPostconditionViolations":[{"toolName":"query_odoo_hr","callId":"call_2","agentContext":"hr","issues":["remaining_leave was negative, which the schema forbids"]}]},"expected":{"status":"blocked"}} diff --git a/middleware/test/golden/corpus/disclaimer.jsonl b/middleware/test/golden/corpus/disclaimer.jsonl new file mode 100644 index 00000000..7a933948 --- /dev/null +++ b/middleware/test/golden/corpus/disclaimer.jsonl @@ -0,0 +1,13 @@ +# Expected verdict class: approved_with_disclaimer (the borderline path, #132 +# isBorderlineVerdict). A soft claim the fixture leaves WITHOUT confirming +# evidence -> judge returns `unverified` -> disclaimer. +# +# CRITICAL: each answer carries a hard trigger signal (a date or a currency +# amount) so shouldTriggerVerifier actually fires and the extractor + judge run. +# Without a signal the pipeline skips extraction and returns `approved` before +# the judge is ever consulted — the disclaimer class would then be untestable. +# The co-extracted hard claim (date/amount) also resolves `unverified` (no Odoo +# reader), which is consistent with the disclaimer outcome and does not mask it. +{"id":"disclaimer_soft_no_evidence","note":"Date signal makes the pipeline trigger; qualitative claim with empty evidence -> judge 'no evidence available' -> unverified -> disclaimer.","userMessage":"Was macht Anna Müller beruflich?","answer":"Anna Müller arbeitet seit dem 01.06.2022 als Senior-Beraterin im Vertriebsteam.","evidence":[],"expected":{"status":"approved_with_disclaimer"}} +{"id":"disclaimer_soft_silent_evidence","note":"Currency signal triggers; evidence is on-topic but silent about the Region-Süd responsibility -> ambiguous -> unverified -> disclaimer.","userMessage":"Ist Tomas Weber für die Region Süd zuständig?","answer":"Tomas Weber betreut die Region Süd mit einem Jahresumsatz von 1.250.000 €.","evidence":[{"nodeId":"n_emp_tomas","source":"graph","title":"Tomas Weber","content":"Tomas Weber ist Mitarbeiter im Vertrieb."}],"expected":{"status":"approved_with_disclaimer"}} +{"id":"disclaimer_soft_partial_evidence","note":"Date signal triggers; evidence covers the person but not the claimed certification -> unverified -> disclaimer.","userMessage":"Hat Lena Fischer eine SAP-Zertifizierung?","answer":"Lena Fischer besitzt seit dem 15.09.2023 eine SAP-Zertifizierung.","evidence":[{"nodeId":"n_emp_lena","source":"graph","title":"Lena Fischer","content":"Lena Fischer ist Projektmanagerin im Bereich ERP-Einführung."}],"expected":{"status":"approved_with_disclaimer"}} diff --git a/middleware/test/golden/goldenModel.ts b/middleware/test/golden/goldenModel.ts new file mode 100644 index 00000000..27b41ebc --- /dev/null +++ b/middleware/test/golden/goldenModel.ts @@ -0,0 +1,107 @@ +/** + * Golden-set regression runner — MODEL layer (#129). + * + * Split out of `goldenRunner.ts` on purpose: this is the only part that imports + * the verifier package as a VALUE (real `ClaimExtractor` + `EvidenceJudge` + + * `VerifierPipeline`), so it can only load once the workspace `dist/` is built. + * Anything importing this file therefore needs the build; the pure harness logic + * (voting, parsing, comparison) stays in `goldenRunner.ts` and does not. + * + * It wires a real `VerifierPipeline` — real ClaimExtractor + EvidenceJudge on an + * injected `LlmProvider`, fixture-backed DeterministicChecker + EvidenceFetcher — + * behind a `RunOnce`. The provider is injected, so this stays key-free: the CLI + * (`goldenSet.eval.ts`) supplies the Anthropic adapter with a real key, while + * `test/goldenModel.test.ts` supplies a stub provider and asserts the synthetic + * (no-model-needed) block paths without spending a token. + */ + +import { + ClaimExtractor, + DeterministicChecker, + EvidenceJudge, + VerifierPipeline, + type EvidenceFetcher, + type EvidenceSnippet, + type SoftClaim, + type VerifierInput, +} from '@omadia/verifier'; +import type { LlmProvider } from '@omadia/llm-provider-api'; + +import type { EntryRun, GoldenEntry, RunOnce } from './goldenRunner.js'; + +function toVerifierInput(entry: GoldenEntry): VerifierInput { + const t = entry.trace; + return { + runId: `golden_${entry.id}`, + userMessage: entry.userMessage, + answer: entry.answer, + ...(t?.agent !== undefined ? { agent: t.agent } : {}), + ...(t?.domainToolsCalled !== undefined + ? { domainToolsCalled: t.domainToolsCalled } + : {}), + ...(t?.knowledgeGraphToolsCalled !== undefined + ? { knowledgeGraphToolsCalled: t.knowledgeGraphToolsCalled } + : {}), + ...(t?.toolPostconditionViolations !== undefined + ? { toolPostconditionViolations: t.toolPostconditionViolations } + : {}), + }; +} + +/** Fixture evidence source — returns the entry's frozen snippets for any claim, + * so the judge stage is deterministic in its INPUT while still stochastic in + * its VERDICT. */ +function fixtureFetcher(evidence: EvidenceSnippet[]): EvidenceFetcher { + return { + fetch(_claim: SoftClaim): Promise { + return Promise.resolve(evidence); + }, + }; +} + +function buildPipeline( + llm: LlmProvider, + model: string, + evidence: EvidenceSnippet[], +): VerifierPipeline { + return new VerifierPipeline({ + extractor: new ClaimExtractor({ llm, model }), + // No odoo/graph reader: hard claims resolve to `unverified` rather than + // hitting Postgres. The verdict-class fixtures never depend on + // deterministic hard-claim verification (that path needs an Odoo fixture + // reader — v2, see README). + deterministic: new DeterministicChecker({}), + judge: new EvidenceJudge({ llm, fetcher: fixtureFetcher(evidence), model }), + log: (): void => { + /* silent */ + }, + }); +} + +/** + * Build a `RunOnce` backed by a real pipeline + injected provider. Wraps the + * provider in a per-run token counter so the summary can report cost without + * the pipeline having to surface usage. + */ +export function buildVerifierRunOnce( + provider: LlmProvider, + model: string, +): RunOnce { + return async (entry: GoldenEntry): Promise => { + let tokens = 0; + const counting: LlmProvider = { + id: provider.id, + capabilities: provider.capabilities, + async complete(req) { + const res = await provider.complete(req); + tokens += res.usage.inputTokens + res.usage.outputTokens; + return res; + }, + stream: provider.stream.bind(provider), + classifyError: provider.classifyError.bind(provider), + }; + const pipeline = buildPipeline(counting, model, entry.evidence ?? []); + const verdict = await pipeline.verify(toVerifierInput(entry)); + return { status: verdict.status, tokens }; + }; +} diff --git a/middleware/test/golden/goldenRunner.ts b/middleware/test/golden/goldenRunner.ts new file mode 100644 index 00000000..336da830 --- /dev/null +++ b/middleware/test/golden/goldenRunner.ts @@ -0,0 +1,257 @@ +/** + * Golden-set regression runner — PURE layer (#129). + * + * This file is deliberately SDK-, network- and key-free: corpus parsing, + * majority-of-K voting, verdict-class comparison and summary formatting, plus + * the shared types. Its only `@omadia/verifier` dependency is a `import type` + * (erased at runtime), so `test/goldenRunner.test.ts` can load it in the default + * `npm test` glob WITHOUT the verifier package being built. + * + * The stochastic model call lives behind the injected `RunOnce` type; the real + * wiring that produces one (`buildVerifierRunOnce`) is quarantined in the sibling + * `goldenModel.ts`, which DOES pull the verifier package as a value. Keeping the + * split at the module boundary (not just the function boundary) is what makes the + * pure suite genuinely independent of `dist/`. + * + * The assertion target is the verifier's verdict CLASS (approved / + * approved_with_disclaimer / blocked), not the raw model string — that is the + * stable signal despite generation stochasticity (see the issue's concept + * excerpt). Flake tolerance: an entry that misses on the first sample is + * re-run up to `maxRuns` total and decided by majority. + */ + +import type { EvidenceSnippet, VerifierVerdict } from '@omadia/verifier'; + +/** The three stable assertion targets — the live `VerifierVerdict` statuses. */ +export type StatusName = VerifierVerdict['status']; + +const STATUS_NAMES: readonly StatusName[] = [ + 'approved', + 'approved_with_disclaimer', + 'blocked', +]; + +/** Trace fields lifted verbatim into `VerifierInput`. */ +export interface GoldenTrace { + agent?: string; + domainToolsCalled?: string[]; + knowledgeGraphToolsCalled?: boolean; + toolPostconditionViolations?: { + toolName: string; + callId: string; + agentContext: string; + issues: string[]; + }[]; +} + +/** One frozen corpus fixture. */ +export interface GoldenEntry { + /** Stable id; unique within the corpus. Used in output + as the runId. */ + id: string; + /** Free-text note on WHY this entry has its expected class. */ + note?: string; + userMessage: string; + answer: string; + /** Trace evidence the pipeline reads (tool-postcondition / citation paths). */ + trace?: GoldenTrace; + /** + * Fixture-backed evidence the EvidenceJudge sees for any soft claim. Empty / + * omitted => the judge has "no evidence available" => `unverified` => + * approved_with_disclaimer. Confirming evidence => verified. Contradicting + * evidence => contradicted => blocked. + */ + evidence?: EvidenceSnippet[]; + expected: { status: StatusName }; +} + +export interface EntryRun { + status: StatusName; + /** input+output tokens spent across every LLM call in this single run. */ + tokens: number; +} + +/** One stochastic execution of a fixture. Injected so the pure layer never + * touches the SDK or a key. */ +export type RunOnce = (entry: GoldenEntry) => Promise; + +export interface GoldenResult { + id: string; + expected: StatusName; + /** verdict class of each sample, in order. */ + runs: StatusName[]; + /** the class after majority — what we compare against `expected`. */ + decided: StatusName; + pass: boolean; + /** summed tokens across every sample of this entry. */ + tokens: number; +} + +// --------------------------------------------------------------------------- +// Corpus parsing (pure) +// --------------------------------------------------------------------------- + +function isStatusName(v: unknown): v is StatusName { + return typeof v === 'string' && (STATUS_NAMES as readonly string[]).includes(v); +} + +/** + * Parse one JSONL line into a validated `GoldenEntry`. Throws a located error + * on malformed input — a bad fixture must fail loudly, not silently drop + * coverage. + */ +export function parseCorpusLine( + line: string, + file: string, + lineNo: number, +): GoldenEntry { + const where = `${file}:${lineNo}`; + let raw: unknown; + try { + raw = JSON.parse(line); + } catch (err) { + throw new Error(`${where}: invalid JSON — ${(err as Error).message}`, { + cause: err, + }); + } + if (typeof raw !== 'object' || raw === null) { + throw new Error(`${where}: entry must be a JSON object`); + } + const o = raw as Record; + const req = (k: string): string => { + const v = o[k]; + if (typeof v !== 'string' || v.length === 0) { + throw new Error(`${where}: missing/empty string field "${k}"`); + } + return v; + }; + const id = req('id'); + const userMessage = req('userMessage'); + const answer = req('answer'); + const expected = o['expected'] as { status?: unknown } | undefined; + if (!expected || !isStatusName(expected.status)) { + throw new Error( + `${where}: "expected.status" must be one of ${STATUS_NAMES.join(' | ')}`, + ); + } + const entry: GoldenEntry = { + id, + userMessage, + answer, + expected: { status: expected.status }, + }; + if (typeof o['note'] === 'string') entry.note = o['note']; + if (o['trace'] !== undefined) entry.trace = o['trace'] as GoldenTrace; + if (o['evidence'] !== undefined) { + entry.evidence = o['evidence'] as EvidenceSnippet[]; + } + return entry; +} + +/** Parse a whole `.jsonl` file body. Blank lines and `#`-comment lines skipped. */ +export function loadCorpusFromText(text: string, file: string): GoldenEntry[] { + const out: GoldenEntry[] = []; + const lines = text.split('\n'); + for (let i = 0; i < lines.length; i++) { + const line = (lines[i] ?? '').trim(); + if (line === '' || line.startsWith('#')) continue; + out.push(parseCorpusLine(line, file, i + 1)); + } + return out; +} + +// --------------------------------------------------------------------------- +// Voting + comparison (pure) +// --------------------------------------------------------------------------- + +/** Mode of a non-empty status list. Ties resolve to the earliest sample, which + * keeps a 1-1-1 three-way split a deterministic (and, vs. any single expected, + * usually failing) outcome rather than a coin flip. */ +export function majority(items: StatusName[]): { winner: StatusName; count: number } { + const first = items[0]; + if (first === undefined) { + throw new Error('majority(): empty sample list'); + } + const counts = new Map(); + for (const s of items) counts.set(s, (counts.get(s) ?? 0) + 1); + let winner: StatusName = first; + let best = 0; + for (const s of items) { + const c = counts.get(s) ?? 0; + if (c > best) { + best = c; + winner = s; + } + } + return { winner, count: best }; +} + +/** + * Run a single fixture with flake tolerance. Cost-aware: a sample that already + * matches `expected` on the first try costs ONE model call; only a first-sample + * miss pays for the re-runs (up to `maxRuns` total) that a majority vote needs. + * This bounds worst-case cost at `maxRuns × corpus size` while keeping the + * common (green) path cheap. + */ +export async function runEntry( + entry: GoldenEntry, + runOnce: RunOnce, + maxRuns = 3, +): Promise { + const first = await runOnce(entry); + const runs: EntryRun[] = [first]; + if (first.status !== entry.expected.status) { + while (runs.length < maxRuns) { + runs.push(await runOnce(entry)); + } + } + const statuses = runs.map((r) => r.status); + // `majority` returns the sole sample unchanged for a one-element list, so it + // doubles as the single-run case — no separate index into `statuses`. + const decided = majority(statuses).winner; + return { + id: entry.id, + expected: entry.expected.status, + runs: statuses, + decided, + pass: decided === entry.expected.status, + tokens: runs.reduce((sum, r) => sum + r.tokens, 0), + }; +} + +export async function runCorpus( + entries: GoldenEntry[], + runOnce: RunOnce, + maxRuns = 3, +): Promise { + const results: GoldenResult[] = []; + // Sequential on purpose: shared per-account Anthropic rate limits make a + // fan-out flaky and the cost signal harder to attribute per entry. + for (const entry of entries) { + results.push(await runEntry(entry, runOnce, maxRuns)); + } + return results; +} + +export function hasRegression(results: GoldenResult[]): boolean { + return results.some((r) => !r.pass); +} + +/** Human + CI-log summary. Failures first, then a totals line with token cost. */ +export function formatSummary(results: GoldenResult[]): string { + const totalTokens = results.reduce((s, r) => s + r.tokens, 0); + const failed = results.filter((r) => !r.pass); + const lines: string[] = []; + for (const r of results) { + const mark = r.pass ? 'PASS' : 'FAIL'; + lines.push( + `${mark} ${r.id.padEnd(28)} expected=${r.expected} decided=${r.decided} ` + + `runs=[${r.runs.join(',')}] tokens=${r.tokens}`, + ); + } + lines.push(''); + lines.push( + `golden-eval: ${results.length - failed.length}/${results.length} passed, ` + + `${failed.length} regressed, ${totalTokens} tokens total`, + ); + return lines.join('\n'); +} diff --git a/middleware/test/golden/goldenSet.eval.ts b/middleware/test/golden/goldenSet.eval.ts new file mode 100644 index 00000000..0187663a --- /dev/null +++ b/middleware/test/golden/goldenSet.eval.ts @@ -0,0 +1,107 @@ +/** + * Golden-set regression eval CLI — #129. `npm run eval:golden`. + * + * Runs the frozen corpus through a REAL VerifierPipeline built on the pinned + * model and exits non-zero on any post-majority verdict-class mismatch. It is + * deliberately OUTSIDE the `test/**\/*.test.ts` glob so `npm test` never needs a + * key; the harness logic it relies on is unit-tested in + * `test/goldenRunner.test.ts`. + * + * Model pinned: the verifier's stochastic stages (ClaimExtractor, EvidenceJudge) + * run on VERIFIER_MODEL — so THAT is the honest thing to regression-test here, + * not ORCHESTRATOR_MODEL, which these stages never call. Override with + * GOLDEN_MODEL. Default mirrors config.ts (`claude-haiku-4-5-20251001`). + * Full-turn generation eval against the orchestrator model is v2 (see README). + * + * Key handling: absent ANTHROPIC_API_KEY => skip-with-notice, exit 0. That keeps + * the CI job green on a repo/fork where the secret is not configured instead of + * failing red on missing infrastructure. + */ + +import { appendFileSync, readFileSync, readdirSync } from 'node:fs'; + +import { + createAnthropicClient, + createAnthropicProvider, +} from '@omadia/llm-adapter-anthropic'; + +import { + formatSummary, + hasRegression, + loadCorpusFromText, + runCorpus, + type GoldenEntry, + type GoldenResult, +} from './goldenRunner.js'; +import { buildVerifierRunOnce } from './goldenModel.js'; + +const DEFAULT_MODEL = 'claude-haiku-4-5-20251001'; + +function loadCorpus(): GoldenEntry[] { + const dir = new URL('./corpus/', import.meta.url); + const files = readdirSync(dir) + .filter((f) => f.endsWith('.jsonl')) + .sort(); + const entries: GoldenEntry[] = []; + for (const file of files) { + const text = readFileSync(new URL(file, dir), 'utf8'); + entries.push(...loadCorpusFromText(text, file)); + } + return entries; +} + +function writeJobSummary(results: GoldenResult[], model: string): void { + const path = process.env.GITHUB_STEP_SUMMARY; + if (!path) return; + const total = results.reduce((s, r) => s + r.tokens, 0); + const failed = results.filter((r) => !r.pass); + const rows = results + .map( + (r) => + `| ${r.pass ? '✅' : '❌'} | \`${r.id}\` | ${r.expected} | ${r.decided} | ${r.runs.join(', ')} | ${r.tokens} |`, + ) + .join('\n'); + const md = [ + `## Golden-set eval (#129) — model \`${model}\``, + '', + `${results.length - failed.length}/${results.length} passed · ${failed.length} regressed · ${total} tokens total`, + '', + '| | entry | expected | decided | samples | tokens |', + '| --- | --- | --- | --- | --- | --- |', + rows, + '', + ].join('\n'); + appendFileSync(path, md); +} + +async function main(): Promise { + const apiKey = process.env.ANTHROPIC_API_KEY; + if (!apiKey) { + console.log( + '::notice::ANTHROPIC_API_KEY not set — skipping golden-set eval (#129). ' + + 'No regression signal collected this run.', + ); + process.exit(0); + } + + const model = process.env.GOLDEN_MODEL ?? process.env.VERIFIER_MODEL ?? DEFAULT_MODEL; + const entries = loadCorpus(); + console.log( + `golden-eval: ${entries.length} corpus entries against model=${model}\n`, + ); + + const client = createAnthropicClient({ apiKey }); + const provider = createAnthropicProvider({ client }); + const runOnce = buildVerifierRunOnce(provider, model); + + const results = await runCorpus(entries, runOnce); + console.log(formatSummary(results)); + writeJobSummary(results, model); + + process.exit(hasRegression(results) ? 1 : 0); +} + +main().catch((err: unknown) => { + console.error('golden-eval crashed:', err); + process.exit(1); +}); diff --git a/middleware/test/golden/tsconfig.json b/middleware/test/golden/tsconfig.json new file mode 100644 index 00000000..0d7634b4 --- /dev/null +++ b/middleware/test/golden/tsconfig.json @@ -0,0 +1,16 @@ +{ + // #129 — static type coverage for the golden-set harness. The middleware + // `lint`/`typecheck` scripts only cover `src/` and package `src/` dirs, so + // nothing under `test/` is checked by default. This project puts the golden + // runner + its two suites back under `tsc --noEmit`, wired into the top-level + // `typecheck` script so a drift in `@omadia/verifier`'s types (e.g. a new + // required `VerifierInput` field) fails on every PR rather than silently at + // the key-gated eval on `main`. Requires the workspace `dist/` to be built + // first (same as the suites themselves), which CI already does before typecheck. + "extends": "../../tsconfig.json", + "compilerOptions": { + "noEmit": true, + "rootDir": "../.." + }, + "include": ["*.ts", "../goldenRunner.test.ts", "../goldenModel.test.ts"] +} diff --git a/middleware/test/goldenModel.test.ts b/middleware/test/goldenModel.test.ts new file mode 100644 index 00000000..e4139dff --- /dev/null +++ b/middleware/test/goldenModel.test.ts @@ -0,0 +1,135 @@ +/** + * #129 — key-free tests for the golden-set MODEL layer (`goldenModel.ts`). + * + * The pure harness (voting/parsing) is covered in `goldenRunner.test.ts`. This + * suite covers the part that actually wires a real `VerifierPipeline`: + * `buildVerifierRunOnce` → `toVerifierInput` → `pipeline.verify`. Without a test + * here that wiring is decorative in CI — a bug that drops a trace field (say + * `knowledgeGraphToolsCalled`) would only surface against a real API key on + * `main`, i.e. after merge. + * + * We exploit that the two synthetic block paths need NO model: a + * `tool_postcondition` violation and a `citation_missing` condition + * (knowledge-graph used, answer carries no `[ref:]` marker) both inject a + * `contradicted` verdict before/independent of claim extraction. A STUB + * `LlmProvider` that returns empty content (the extractor treats that as "no + * claims") lets us drive the real pipeline to a `blocked` verdict with zero + * tokens and no key — and a control entry proves we are not merely + * always-blocking. + */ + +import { describe, it } from 'node:test'; +import { strict as assert } from 'node:assert'; + +import type { LlmProvider } from '@omadia/llm-provider-api'; + +import { buildVerifierRunOnce } from './golden/goldenModel.js'; +import type { GoldenEntry } from './golden/goldenRunner.js'; + +/** A provider whose `complete` returns no content — so `ClaimExtractor` finds no + * tool call and yields zero claims. `stream`/`classifyError` are never reached + * on the verify path, so they throw if the wiring ever routes through them. */ +function stubProvider(onComplete?: () => void): LlmProvider { + return { + id: 'stub', + capabilities: { + tools: true, + vision: false, + streaming: false, + promptCaching: false, + forcedToolChoice: true, + parallelToolCalls: false, + interleavedToolUse: false, + }, + complete(_req) { + onComplete?.(); + return Promise.resolve({ + content: [], + finishReason: 'stop', + model: 'stub', + usage: { inputTokens: 0, outputTokens: 0 }, + }); + }, + stream() { + throw new Error('stub: stream() must not be called on the verify path'); + }, + classifyError() { + throw new Error('stub: classifyError() must not be called on the verify path'); + }, + }; +} + +describe('goldenModel/buildVerifierRunOnce (synthetic paths, key-free)', () => { + it('blocks via a recorded tool_postcondition violation (#130)', async () => { + const entry: GoldenEntry = { + id: 'tp', + userMessage: 'Wie hoch ist der offene Betrag?', + answer: 'Der offene Betrag beträgt 1.234,56 €.', + trace: { + agent: 'accounting', + domainToolsCalled: ['query_odoo_accounting'], + toolPostconditionViolations: [ + { + toolName: 'query_odoo_accounting', + callId: 'call_1', + agentContext: 'accounting', + issues: ['result.amount_residual missing'], + }, + ], + }, + expected: { status: 'blocked' }, + }; + const r = await buildVerifierRunOnce(stubProvider(), 'stub-model')(entry); + assert.equal(r.status, 'blocked'); + }); + + it('blocks via citation_missing when KG was used but no [ref:] marker (#131)', async () => { + const entry: GoldenEntry = { + id: 'cm', + userMessage: 'Wer ist der Ansprechpartner für ACME?', + answer: 'Der Ansprechpartner für ACME ist Julia Berg.', + trace: { knowledgeGraphToolsCalled: true }, + expected: { status: 'blocked' }, + }; + const r = await buildVerifierRunOnce(stubProvider(), 'stub-model')(entry); + assert.equal(r.status, 'blocked'); + }); + + it('does NOT block a KG answer that carries a [ref:] marker (control)', async () => { + // Same trace flag as above, but the marker is present, so the citation + // synthetic must NOT fire. If `toVerifierInput` dropped + // `knowledgeGraphToolsCalled`, the block-above case would silently pass for + // the wrong reason and this control would still be green — the two together + // pin the wiring. The answer trips no trigger signal, so extraction is + // skipped and the stub `complete` is never called. + const entry: GoldenEntry = { + id: 'ok', + userMessage: 'Ist die Aufgabe erledigt?', + answer: 'Alles erledigt [ref:n1].', + trace: { knowledgeGraphToolsCalled: true }, + expected: { status: 'approved' }, + }; + let calls = 0; + const r = await buildVerifierRunOnce(stubProvider(() => { + calls += 1; + }), 'stub-model')(entry); + assert.notEqual(r.status, 'blocked'); + assert.equal(calls, 0); + }); + + it('reports zero tokens when the stub reports zero usage', async () => { + const entry: GoldenEntry = { + id: 'tok', + userMessage: 'q', + answer: 'Der offene Betrag beträgt 1.234,56 €.', + trace: { + toolPostconditionViolations: [ + { toolName: 't', callId: 'c', agentContext: 'a', issues: ['x'] }, + ], + }, + expected: { status: 'blocked' }, + }; + const r = await buildVerifierRunOnce(stubProvider(), 'stub-model')(entry); + assert.equal(r.tokens, 0); + }); +}); diff --git a/middleware/test/goldenRunner.test.ts b/middleware/test/goldenRunner.test.ts new file mode 100644 index 00000000..df4ade0e --- /dev/null +++ b/middleware/test/goldenRunner.test.ts @@ -0,0 +1,204 @@ +/** + * #129 — unit tests for the golden-set harness LOGIC (corpus parsing, majority + * voting, flake-tolerant runEntry, regression detection). No Anthropic key: the + * stochastic model call is replaced by a scripted `RunOnce`, so this suite runs + * in the default `npm test` glob and stays green key-free. + * + * Every assertion here is mutation-checked in review: flip the comparator in + * runEntry, the tie rule in majority, or the re-run guard, and a case below goes + * red. A green harness that catches nothing would make the whole eval decorative. + */ + +import { describe, it } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { readFileSync, readdirSync } from 'node:fs'; + +import { + formatSummary, + hasRegression, + loadCorpusFromText, + majority, + parseCorpusLine, + runEntry, + type EntryRun, + type GoldenEntry, + type RunOnce, + type StatusName, +} from './golden/goldenRunner.js'; + +function entry(expected: StatusName, id = 'e'): GoldenEntry { + return { id, userMessage: 'u', answer: 'a', expected: { status: expected } }; +} + +/** A RunOnce that yields a scripted sequence of statuses, then repeats the last. + * Records how many times it was called so the cost-aware path is observable. */ +function scriptedRunOnce(seq: StatusName[]): RunOnce & { calls: () => number } { + let i = 0; + let calls = 0; + const fn = ((): Promise => { + calls++; + const status = seq[Math.min(i, seq.length - 1)] ?? seq[seq.length - 1]!; + i++; + return Promise.resolve({ status, tokens: 10 }); + }) as unknown as RunOnce & { calls: () => number }; + fn.calls = (): number => calls; + return fn; +} + +describe('goldenRunner/parseCorpusLine', () => { + it('parses a valid entry with trace + evidence', () => { + const e = parseCorpusLine( + JSON.stringify({ + id: 'x', + userMessage: 'q', + answer: 'a [ref:n1]', + trace: { knowledgeGraphToolsCalled: true }, + evidence: [{ nodeId: 'n1', source: 'graph', content: 'c' }], + expected: { status: 'blocked' }, + }), + 'f.jsonl', + 3, + ); + assert.equal(e.id, 'x'); + assert.equal(e.expected.status, 'blocked'); + assert.equal(e.trace?.knowledgeGraphToolsCalled, true); + assert.equal(e.evidence?.[0]?.nodeId, 'n1'); + }); + + it('throws with file:line on a missing required field', () => { + assert.throws( + () => parseCorpusLine(JSON.stringify({ id: 'x', answer: 'a', expected: { status: 'approved' } }), 'f.jsonl', 7), + /f\.jsonl:7: missing\/empty string field "userMessage"/, + ); + }); + + it('throws on an unknown verdict status', () => { + assert.throws( + () => parseCorpusLine(JSON.stringify({ id: 'x', userMessage: 'q', answer: 'a', expected: { status: 'corrected' } }), 'f.jsonl', 1), + /expected\.status.*approved.*approved_with_disclaimer.*blocked/, + ); + }); + + it('throws on malformed JSON', () => { + assert.throws(() => parseCorpusLine('{not json', 'f.jsonl', 2), /f\.jsonl:2: invalid JSON/); + }); +}); + +describe('goldenRunner/loadCorpusFromText', () => { + it('skips blank lines and # comments', () => { + const text = [ + '# a header comment', + '', + JSON.stringify({ id: 'a', userMessage: 'u', answer: 'a', expected: { status: 'approved' } }), + ' ', + JSON.stringify({ id: 'b', userMessage: 'u', answer: 'a', expected: { status: 'blocked' } }), + ].join('\n'); + const entries = loadCorpusFromText(text, 'c.jsonl'); + assert.equal(entries.length, 2); + assert.deepEqual(entries.map((e) => e.id), ['a', 'b']); + }); +}); + +describe('goldenRunner/majority', () => { + it('returns the mode', () => { + assert.deepEqual(majority(['blocked', 'approved', 'blocked']), { winner: 'blocked', count: 2 }); + }); + it('breaks a 3-way tie toward the earliest sample', () => { + assert.equal(majority(['approved', 'blocked', 'approved_with_disclaimer']).winner, 'approved'); + }); + it('throws on an empty sample list', () => { + assert.throws(() => majority([]), /empty sample list/); + }); +}); + +describe('goldenRunner/runEntry', () => { + it('passes on the first sample and pays exactly ONE model call', async () => { + const run = scriptedRunOnce(['approved']); + const r = await runEntry(entry('approved'), run); + assert.equal(r.pass, true); + assert.equal(r.runs.length, 1); + assert.equal(run.calls(), 1); // cost-aware: no re-runs on a green first try + assert.equal(r.tokens, 10); + }); + + it('re-runs up to 3× on a first-sample miss and corrects a lone flake by majority', async () => { + // first sample wrong, next two right -> majority approved -> pass + const run = scriptedRunOnce(['blocked', 'approved', 'approved']); + const r = await runEntry(entry('approved'), run); + assert.equal(run.calls(), 3); + assert.deepEqual(r.runs, ['blocked', 'approved', 'approved']); + assert.equal(r.decided, 'approved'); + assert.equal(r.pass, true); + assert.equal(r.tokens, 30); + }); + + it('fails when the majority genuinely disagrees with expected (real regression)', async () => { + const run = scriptedRunOnce(['blocked', 'blocked', 'approved']); + const r = await runEntry(entry('approved'), run); + assert.equal(r.decided, 'blocked'); + assert.equal(r.pass, false); + }); + + it('honours a custom maxRuns', async () => { + const run = scriptedRunOnce(['blocked', 'approved']); + const r = await runEntry(entry('approved'), run, 2); + assert.equal(run.calls(), 2); + assert.deepEqual(r.runs, ['blocked', 'approved']); + }); +}); + +describe('goldenRunner/corpus integrity (#129 acceptance, key-free)', () => { + const dir = new URL('./golden/corpus/', import.meta.url); + const files = readdirSync(dir).filter((f) => f.endsWith('.jsonl')); + const entries = files.flatMap((f) => + loadCorpusFromText(readFileSync(new URL(f, dir), 'utf8'), f), + ); + + it('every corpus line parses and there are at least 12 entries', () => { + assert.ok(files.length > 0, 'corpus dir must contain .jsonl files'); + assert.ok(entries.length >= 12, `expected >= 12 entries, got ${entries.length}`); + }); + + it('entry ids are unique', () => { + const ids = entries.map((e) => e.id); + assert.equal(new Set(ids).size, ids.length); + }); + + it('covers all three VerifierVerdict statuses', () => { + const statuses = new Set(entries.map((e) => e.expected.status)); + assert.ok(statuses.has('approved')); + assert.ok(statuses.has('approved_with_disclaimer')); + assert.ok(statuses.has('blocked')); + }); + + it('covers the tool_postcondition and citation_missing claim paths', () => { + const hasToolPostcondition = entries.some( + (e) => (e.trace?.toolPostconditionViolations?.length ?? 0) > 0, + ); + const hasCitationPath = entries.some( + (e) => e.trace?.knowledgeGraphToolsCalled === true, + ); + assert.ok(hasToolPostcondition, 'need a tool_postcondition (#130) fixture'); + assert.ok(hasCitationPath, 'need a citation_missing (#131) fixture'); + }); +}); + +describe('goldenRunner/hasRegression + formatSummary', () => { + it('detects at least one failing entry', async () => { + const pass = await runEntry(entry('approved', 'ok'), scriptedRunOnce(['approved'])); + const fail = await runEntry(entry('blocked', 'bad'), scriptedRunOnce(['approved', 'approved', 'approved'])); + assert.equal(hasRegression([pass]), false); + assert.equal(hasRegression([pass, fail]), true); + }); + + it('summary reports counts, per-entry token cost and total', async () => { + const results = [ + await runEntry(entry('approved', 'ok'), scriptedRunOnce(['approved'])), + await runEntry(entry('blocked', 'bad'), scriptedRunOnce(['approved', 'approved', 'approved'])), + ]; + const summary = formatSummary(results); + assert.match(summary, /PASS {2}ok/); + assert.match(summary, /FAIL {2}bad/); + assert.match(summary, /1\/2 passed, 1 regressed, 40 tokens total/); + }); +});