From a2f184cc5752fcd0fb360014476bf7047bd2a5a1 Mon Sep 17 00:00:00 2001 From: Marcel Wege Date: Thu, 20 Aug 2026 16:22:44 +0200 Subject: [PATCH 1/2] fix(verifier): check Odoo record existence for anchored soft claims (#129 golden flake) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit golden-eval.yml flaked on `blocked_deterministic_id_absent` (2 of 15 main runs today, ~30-50 % per-sample miss locally). Root cause: the Haiku claim extractor types "die Rechnung INV/2026/0099 ist verbucht" as `id` in some samples and as `qualitative` in others — in BOTH cases with `odoo_record: {model: 'account.move', ref: 'INV/2026/0099'}` populated. `qualitative` claims bypassed the DeterministicChecker entirely and went to the EvidenceJudge, which returned `unverified` → `approved_with_disclaimer` instead of `blocked`. That is not eval noise: a hallucinated invoice number passed with a disclaimer in roughly a third of real turns. Fix, two layers: 1. Pipeline: soft claims anchored on an Odoo record (`hasOdooRecordAnchor`) get `DeterministicChecker.checkRecordExists()` before the judge. A record that does not exist is a contradiction regardless of how the extractor typed the claim; those claims never reach the judge. Existing records (or unverifiable lookups) go to the judge unchanged. 2. Extractor prompt: a record reference is always its own `id` claim, in addition to any qualitative claim about the same record. Verification: new unit tests (checker ×4, pipeline ×3) red→green; full middleware suite 7165/0 fail; golden corpus 15/15 locally; the flaky case went from 10/16 to 12/12 `blocked` against claude-haiku-4-5 with both claim types now contradicted. --- .../harness-verifier/src/claimExtractor.ts | 3 +- .../harness-verifier/src/claimTypes.ts | 14 +++ .../src/deterministicChecker.ts | 26 ++++- .../packages/harness-verifier/src/index.ts | 1 + .../harness-verifier/src/verifierPipeline.ts | 35 +++++- .../test/verifierDeterministicChecker.test.ts | 67 ++++++++++++ middleware/test/verifierPipeline.test.ts | 103 ++++++++++++++++++ 7 files changed, 244 insertions(+), 5 deletions(-) diff --git a/middleware/packages/harness-verifier/src/claimExtractor.ts b/middleware/packages/harness-verifier/src/claimExtractor.ts index 8b52eeca..8bdffde3 100644 --- a/middleware/packages/harness-verifier/src/claimExtractor.ts +++ b/middleware/packages/harness-verifier/src/claimExtractor.ts @@ -91,7 +91,7 @@ const toolSpec: ToolSpec = { type: 'string', enum: [...CLAIM_TYPES], description: - 'amount=money/number+unit; id=record reference; date=calendar date; name=person/customer with context; aggregate=sum/count/avg over a set (especially HR leave totals); qualitative=non-numeric claim about an entity.', + 'amount=money/number+unit; id=record reference (invoice/order/document number such as "INV/2026/0042", or a numeric record id) — ALWAYS emit a separate id claim for every record reference, even when the sentence also makes a qualitative statement about that record; date=calendar date; name=person/customer with context; aggregate=sum/count/avg over a set (especially HR leave totals); qualitative=non-numeric claim about an entity.', }, expected_source: { type: 'string', @@ -188,6 +188,7 @@ Strict rules: - Do NOT extract the user's question, instructions, or meta-commentary. - Do NOT invent claims that are "implied" but not stated. - When in doubt, skip the claim rather than invent one. +- A record reference (invoice, order or document number, numeric record id) is ALWAYS its own claim of type "id" with odoo_record.model and odoo_record.ref/id set — in addition to any qualitative claim about the same record. - Return at most ${String(this.opts.maxClaims)} claims via the ${TOOL_NAME} tool.`; const user = `USER MESSAGE: diff --git a/middleware/packages/harness-verifier/src/claimTypes.ts b/middleware/packages/harness-verifier/src/claimTypes.ts index 7ec0e84b..f5f086ee 100644 --- a/middleware/packages/harness-verifier/src/claimTypes.ts +++ b/middleware/packages/harness-verifier/src/claimTypes.ts @@ -185,3 +185,17 @@ export function isHardClaim(claim: Claim): claim is HardClaim { export function isSoftClaim(claim: Claim): claim is SoftClaim { return claim.type === 'name' || claim.type === 'qualitative'; } + +/** + * #129 — a claim is *anchored* when it names a concrete Odoo record + * (`odooRecord.id` or `.ref`) with `expectedSource: 'odoo'`. Whether that + * record EXISTS is checkable deterministically no matter how the extractor + * typed the claim — Haiku types "INV/2026/0099 ist verbucht" as `id` in + * some samples and `qualitative` in others. Pure predicate; no I/O. + */ +export function hasOdooRecordAnchor(claim: Claim): boolean { + if (claim.expectedSource !== 'odoo') return false; + const ref = claim.odooRecord; + if (!ref || typeof ref.model !== 'string' || ref.model.length === 0) return false; + return typeof ref.id === 'number' || (typeof ref.ref === 'string' && ref.ref.length > 0); +} diff --git a/middleware/packages/harness-verifier/src/deterministicChecker.ts b/middleware/packages/harness-verifier/src/deterministicChecker.ts index e8bae71d..bb679a9f 100644 --- a/middleware/packages/harness-verifier/src/deterministicChecker.ts +++ b/middleware/packages/harness-verifier/src/deterministicChecker.ts @@ -123,6 +123,28 @@ export class DeterministicChecker { return Promise.all(claims.map((c) => this.check(c))); } + /** + * #129 — existence check for ANY claim anchored on an Odoo record + * (`hasOdooRecordAnchor`), independent of `claim.type`. The extractor + * is an LLM and types "INV/2026/0099 ist verbucht" as `id` in some + * samples and `qualitative` in others; the record either exists or it + * doesn't either way. Same transport as the `id` path: `read` by id, + * `search` by `name = ref`. Always resolves — never throws. + */ + async checkRecordExists(claim: Claim): Promise { + if (claim.expectedSource !== 'odoo') { + return unverified(claim, `existence check needs odoo source, got ${claim.expectedSource}`); + } + if (!this.odoo) return unverified(claim, 'no odoo reader configured'); + try { + return await this.checkOdooId(claim, claim.odooRecord); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + this.log(`[verifier/deterministic] FAIL exists claim=${claim.id} err=${msg}`); + return unverified(claim, `re-query error: ${msg}`); + } + } + // --- Odoo --------------------------------------------------------------- private async checkOdoo(claim: HardClaim): Promise { @@ -250,8 +272,10 @@ export class DeterministicChecker { return verified(claim, 'odoo'); } + /** Record-existence check — shared by the `id` path and + * {@link checkRecordExists}, hence typed on `Claim` not `HardClaim`. */ private async checkOdooId( - claim: HardClaim, + claim: Claim, ref: OdooRecordRef | undefined, ): Promise { if (!ref) return unverified(claim, 'id claim without odoo model'); diff --git a/middleware/packages/harness-verifier/src/index.ts b/middleware/packages/harness-verifier/src/index.ts index 4da3caae..dcadaea9 100644 --- a/middleware/packages/harness-verifier/src/index.ts +++ b/middleware/packages/harness-verifier/src/index.ts @@ -41,6 +41,7 @@ export type { // claimTypes — shared vocabulary used by every other verifier file plus // the kernel-side `verifierService.ts` until sub-commit 2b moves it. export { + hasOdooRecordAnchor, isBorderlineVerdict, isHardClaim, isSoftClaim, diff --git a/middleware/packages/harness-verifier/src/verifierPipeline.ts b/middleware/packages/harness-verifier/src/verifierPipeline.ts index cb812464..c31c81cb 100644 --- a/middleware/packages/harness-verifier/src/verifierPipeline.ts +++ b/middleware/packages/harness-verifier/src/verifierPipeline.ts @@ -6,7 +6,7 @@ import type { VerifierInput, VerifierVerdict, } from './claimTypes.js'; -import { isHardClaim, isSoftClaim } from './claimTypes.js'; +import { hasOdooRecordAnchor, isHardClaim, isSoftClaim } from './claimTypes.js'; import type { ClaimExtractor } from './claimExtractor.js'; import type { DeterministicChecker } from './deterministicChecker.js'; import type { EvidenceJudge } from './evidenceJudge.js'; @@ -18,7 +18,11 @@ import { shouldTriggerVerifier } from './triggerRouter.js'; * * answer → triggerRouter → claimExtractor → classify * ├─► DeterministicChecker (hard claims, parallel) - * └─► EvidenceJudge (soft claims, parallel) + * ├─► DeterministicChecker.checkRecordExists + * │ (soft claims anchored on an Odoo record — + * │ #129: a missing record blocks before any + * │ judge call, whatever the extractor typed) + * └─► EvidenceJudge (remaining soft claims) * → aggregate → VerifierVerdict * * Never throws. On any failure below the API level the pipeline returns @@ -131,7 +135,7 @@ export class VerifierPipeline { // own — we never need to wait on one to start the other. const [hardVerdicts, softVerdicts] = await Promise.all([ this.deterministic.checkAll(hardToActuallyCheck), - this.judge.checkAll(soft), + this.checkSoftClaims(soft), ]); const all: ClaimVerdict[] = [ @@ -142,6 +146,31 @@ export class VerifierPipeline { ]; return aggregate(all, started); } + + /** + * #129 — soft claims anchored on an Odoo record get a deterministic + * existence check first. A record that does not exist is a contradiction + * no judge can talk away, so those claims never reach the judge; claims + * whose record exists (or could not be checked) go to the judge unchanged. + */ + private async checkSoftClaims(soft: SoftClaim[]): Promise { + const anchored = soft.filter(hasOdooRecordAnchor); + if (anchored.length === 0) return this.judge.checkAll(soft); + + const existence = await Promise.all( + anchored.map((c) => this.deterministic.checkRecordExists(c)), + ); + const contradicted = existence.filter((v) => v.status === 'contradicted'); + const blockedIds = new Set(contradicted.map((v) => v.claim.id)); + if (blockedIds.size > 0) { + this.log( + `[verifier/pipeline] anchored soft claim(s) refuted by record re-query: ${[...blockedIds].join(',')}`, + ); + } + const forJudge = soft.filter((c) => !blockedIds.has(c.id)); + const judged = await this.judge.checkAll(forJudge); + return [...contradicted, ...judged]; + } } /** diff --git a/middleware/test/verifierDeterministicChecker.test.ts b/middleware/test/verifierDeterministicChecker.test.ts index 87cf1c40..ab94b0e7 100644 --- a/middleware/test/verifierDeterministicChecker.test.ts +++ b/middleware/test/verifierDeterministicChecker.test.ts @@ -2,6 +2,7 @@ import { describe, it } from 'node:test'; import { strict as assert } from 'node:assert'; import { DeterministicChecker, + type Claim, type GraphReader, type HardClaim, type OdooReader, @@ -326,3 +327,69 @@ describe('verifier/deterministicChecker - error handling', () => { } }); }); + +// #129 golden flake — the extractor sometimes types an invoice-reference claim +// as `qualitative` instead of `id` while still populating `odooRecord.ref`. +// Record existence is checkable regardless of claim type. +describe('verifier/deterministicChecker - checkRecordExists (any claim type)', () => { + function anchoredQualitative(overrides: Partial = {}): Claim { + return { + id: 'c_q', + text: 'die Rechnung INV/2026/0099 ist verbucht und abgeschlossen', + type: 'qualitative', + expectedSource: 'odoo', + odooRecord: { model: 'account.move', ref: 'INV/2026/0099' }, + relatedEntities: [], + ...overrides, + }; + } + + it('contradicts a qualitative claim whose anchored ref does not exist', async () => { + const calls: OdooCall[] = []; + const odoo = stubOdoo(() => [], calls); + const checker = new DeterministicChecker({ odoo }); + const verdict = await checker.checkRecordExists(anchoredQualitative()); + assert.equal(verdict.status, 'contradicted'); + assert.equal(calls[0]!.method, 'search'); + assert.deepEqual(calls[0]!.positionalArgs, [[['name', '=', 'INV/2026/0099']]]); + }); + + it('verifies a qualitative claim whose anchored id exists', async () => { + const odoo = stubOdoo(() => [{ id: 42 }]); + const checker = new DeterministicChecker({ odoo }); + const verdict = await checker.checkRecordExists( + anchoredQualitative({ odooRecord: { model: 'account.move', id: 42 } }), + ); + assert.equal(verdict.status, 'verified'); + }); + + it('returns unverified (never throws) when the reader fails or is missing', async () => { + const throwing = new DeterministicChecker({ + odoo: stubOdoo(() => { + throw new Error('boom'); + }), + log: () => undefined, + }); + assert.equal( + (await throwing.checkRecordExists(anchoredQualitative())).status, + 'unverified', + ); + const none = new DeterministicChecker({}); + assert.equal( + (await none.checkRecordExists(anchoredQualitative())).status, + 'unverified', + ); + }); + + it('returns unverified for a non-odoo or unanchored claim', async () => { + const checker = new DeterministicChecker({ odoo: stubOdoo(() => [1]) }); + assert.equal( + (await checker.checkRecordExists(anchoredQualitative({ expectedSource: 'graph' }))).status, + 'unverified', + ); + assert.equal( + (await checker.checkRecordExists(anchoredQualitative({ odooRecord: undefined }))).status, + 'unverified', + ); + }); +}); diff --git a/middleware/test/verifierPipeline.test.ts b/middleware/test/verifierPipeline.test.ts index ac8a21cb..5d01bb2a 100644 --- a/middleware/test/verifierPipeline.test.ts +++ b/middleware/test/verifierPipeline.test.ts @@ -407,3 +407,106 @@ describe('verifier/pipeline', () => { assert.equal(verdict.status, 'approved'); }); }); + +// #129 golden flake — `blocked_deterministic_id_absent`: the extractor types +// "INV/2026/0099 ist verbucht" as `qualitative` in ~⅓ of samples while still +// anchoring `odooRecord.ref`. The pipeline must check record existence +// deterministically BEFORE the judge, independent of the claim type. +describe('verifier/pipeline - anchored soft claims', () => { + function anchoredSoft(): SoftClaim { + return softClaim({ + id: 'c_anchor', + text: 'die Rechnung INV/2026/0099 ist im Odoo-Modell account.move verbucht und abgeschlossen', + type: 'qualitative', + expectedSource: 'odoo', + odooRecord: { model: 'account.move', ref: 'INV/2026/0099' }, + }); + } + + function stubDeterministicWithExists( + existsVerdict: (c: Claim) => ClaimVerdict, + ): DeterministicChecker { + return { + ...stubDeterministic((c) => ({ status: 'verified', claim: c, source: 'odoo' })), + checkRecordExists(c: Claim): Promise { + return Promise.resolve(existsVerdict(c)); + }, + } as unknown as DeterministicChecker; + } + + it('blocks a qualitative claim whose anchored Odoo record does not exist — judge never asked', async () => { + let judgeCalls = 0; + const pipeline = new VerifierPipeline({ + extractor: stubExtractor([anchoredSoft()]), + deterministic: stubDeterministicWithExists((c) => ({ + status: 'contradicted', + claim: c, + truth: null, + source: 'odoo', + detail: 'no account.move with name="INV/2026/0099"', + })), + judge: stubJudge((c) => { + judgeCalls += 1; + return { status: 'unverified', claim: c, reason: 'no evidence' }; + }), + log: SILENT_LOG, + }); + const verdict = await pipeline.verify({ + runId: 'r_anchor_absent', + userMessage: 'Ist die Rechnung INV/2026/0099 bereits verbucht?', + answer: + 'Ja, die Rechnung INV/2026/0099 ist im Odoo-Modell account.move verbucht und abgeschlossen.', + domainToolsCalled: ['query_odoo_accounting'], + }); + assert.equal(verdict.status, 'blocked'); + assert.equal(judgeCalls, 0); + if (verdict.status === 'blocked') { + assert.equal(verdict.contradictions[0]!.claim.type, 'qualitative'); + } + }); + + it('hands an anchored qualitative claim to the judge when the record exists', async () => { + let judgeCalls = 0; + const pipeline = new VerifierPipeline({ + extractor: stubExtractor([anchoredSoft()]), + deterministic: stubDeterministicWithExists((c) => ({ + status: 'verified', + claim: c, + source: 'odoo', + })), + judge: stubJudge((c) => { + judgeCalls += 1; + return { status: 'unverified', claim: c, reason: 'no evidence' }; + }), + log: SILENT_LOG, + }); + const verdict = await pipeline.verify({ + runId: 'r_anchor_present', + userMessage: 'Ist die Rechnung bereits verbucht?', + answer: 'Ja, die Rechnung INV/2026/0099 ist verbucht und abgeschlossen.', + domainToolsCalled: ['query_odoo_accounting'], + }); + assert.equal(judgeCalls, 1); + assert.equal(verdict.status, 'approved_with_disclaimer'); + }); + + it('leaves unanchored soft claims on the judge path untouched', async () => { + let existsCalls = 0; + const pipeline = new VerifierPipeline({ + extractor: stubExtractor([softClaim()]), + deterministic: stubDeterministicWithExists((c) => { + existsCalls += 1; + return { status: 'verified', claim: c, source: 'graph' }; + }), + judge: stubJudge((c) => ({ status: 'verified', claim: c, source: 'graph' })), + log: SILENT_LOG, + }); + const verdict = await pipeline.verify({ + runId: 'r_plain', + userMessage: 'wer?', + answer: 'John Doe ist Senior Dev.', + }); + assert.equal(existsCalls, 0); + assert.equal(verdict.status, 'approved'); + }); +}); From fbf7aad5e2af21a90fe5fab4968c966e15a5b056 Mon Sep 17 00:00:00 2001 From: Marcel Wege Date: Thu, 20 Aug 2026 16:34:27 +0200 Subject: [PATCH 2/2] fix(verifier): narrow anchored-claim existence check (review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review findings on PR #781: - HIGH: `name` claims were anchored too — an exact `name = "John Doe"` search would refute "Doe, John" and block a correct answer. Anchor is now `qualitative` only, and a textual `ref` must look like a document sequence (contains a digit). - HIGH: `ref` was searched on `name` only; vendor bills keep the supplier number in `ref`, sale orders in `client_order_ref`, … Added `SOFT_ANCHOR_REF_FIELDS` (per-model field list, tried in order) and an allow-list: unknown model ⇒ `unverified`, judge decides. - MEDIUM: anchors already covered by a hard `id` claim in the same turn are no longer re-queried — one contradiction per record, and the hard path's replay guard owns that record's verdict (asymmetry documented). - LOW: `checkRecordExists` on an unanchored claim now says so. - Tests: fail-open (unverified → judge), hard/soft twin dedupe, `name` exclusion, ref-field fallback, allow-list miss, predicate edge cases. - docs/CHANGELOG.md entry (blocking behaviour change). Verification: verifier tests 55/55, middleware suite 7217/0 fail, golden corpus 15/15 locally, flaky case 10/10 `blocked` with exactly one contradiction per run. --- docs/CHANGELOG.md | 19 +++++ .../harness-verifier/src/claimTypes.ts | 43 ++++++++++-- .../src/deterministicChecker.ts | 45 +++++++++--- .../packages/harness-verifier/src/index.ts | 1 + .../harness-verifier/src/verifierPipeline.ts | 32 +++++++-- .../test/verifierDeterministicChecker.test.ts | 67 +++++++++++++++++- middleware/test/verifierPipeline.test.ts | 69 +++++++++++++++++++ 7 files changed, 253 insertions(+), 23 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index e70dfd48..ea201ed4 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -18,6 +18,25 @@ entry. See `CONTRIBUTING.md` § Releases & changelog. ## [Unreleased] +### Fixed — verifier: hallucinated record references no longer pass with a disclaimer (#129, PR #781) + +- **Behaviour change (blocking).** A qualitative answer that names a concrete + Odoo document (`INV/2026/0099`, `SO0123`, `RE-4711`) which does **not** + exist is now `blocked`, not `approved_with_disclaimer`. Root cause: the + LLM claim extractor typed such sentences as `qualitative` in roughly a + third of samples, and qualitative claims bypassed the deterministic + re-query entirely. `golden-eval.yml` flaked on exactly this + (`blocked_deterministic_id_absent`). +- Scope is deliberately narrow: only `qualitative` claims with a + document-style `ref` (contains a digit) or numeric `id`; only models with + known reference fields (`account.move` `name|ref`, `sale.order` + `name|client_order_ref`, `purchase.order` `name|partner_ref`, + `stock.picking` `name|origin`, `account.payment`, `hr.expense.sheet`). + Person/company names, unknown models and reader errors stay on the judge + path (fail-open). Anchors already covered by a hard `id` claim in the same + turn are not re-queried twice. +- Extractor prompt now asks for a separate `id` claim per record reference. + ### Added — provenance verification surface: verify API, signed export, offline verifier (#761) - **`GET /api/v1/operator/provenance/verify`** walks the stored chain, diff --git a/middleware/packages/harness-verifier/src/claimTypes.ts b/middleware/packages/harness-verifier/src/claimTypes.ts index f5f086ee..9d68765a 100644 --- a/middleware/packages/harness-verifier/src/claimTypes.ts +++ b/middleware/packages/harness-verifier/src/claimTypes.ts @@ -187,15 +187,44 @@ export function isSoftClaim(claim: Claim): claim is SoftClaim { } /** - * #129 — a claim is *anchored* when it names a concrete Odoo record - * (`odooRecord.id` or `.ref`) with `expectedSource: 'odoo'`. Whether that - * record EXISTS is checkable deterministically no matter how the extractor - * typed the claim — Haiku types "INV/2026/0099 ist verbucht" as `id` in - * some samples and `qualitative` in others. Pure predicate; no I/O. + * #129 — a *qualitative* claim is *anchored* when it names a concrete Odoo + * record (`odooRecord.id` or a document-style `.ref`) with + * `expectedSource: 'odoo'`. Whether that record EXISTS is checkable + * deterministically no matter how the extractor typed the claim — Haiku + * types "INV/2026/0099 ist verbucht" as `id` in some samples and + * `qualitative` in others. + * + * Deliberately narrow (review on PR #781): `name` claims are excluded — + * an exact `name = "John Doe"` search would refute "Doe, John" and block a + * correct answer — and a `ref` only counts when it looks like a document + * sequence (contains a digit), never a bare person/company name. + * Pure predicate; no I/O. */ +/** A reference that starts with a non-space and carries at least one digit — + * "INV/2026/0042", "SO0123", "RE-4711"; not "ACME GmbH" or "John Doe". */ +const DOCUMENT_REF_PATTERN = /^\S.*\d/; + export function hasOdooRecordAnchor(claim: Claim): boolean { - if (claim.expectedSource !== 'odoo') return false; + if (claim.type !== 'qualitative' || claim.expectedSource !== 'odoo') return false; const ref = claim.odooRecord; if (!ref || typeof ref.model !== 'string' || ref.model.length === 0) return false; - return typeof ref.id === 'number' || (typeof ref.ref === 'string' && ref.ref.length > 0); + if (typeof ref.id === 'number' && Number.isInteger(ref.id) && ref.id > 0) return true; + return typeof ref.ref === 'string' && DOCUMENT_REF_PATTERN.test(ref.ref); } + +/** + * Per-model fields that may hold a human-readable record reference. `name` + * is the sequence on customer invoices / orders / pickings, but vendor bills + * carry the supplier number in `ref`, sale orders the customer's PO in + * `client_order_ref`, purchase orders the vendor's in `partner_ref`. + * A model outside this map has no safe reference field → the existence + * check stays `unverified` (judge decides) instead of guessing. + */ +export const SOFT_ANCHOR_REF_FIELDS: Readonly> = { + 'account.move': ['name', 'ref'], + 'account.payment': ['name', 'ref'], + 'sale.order': ['name', 'client_order_ref'], + 'purchase.order': ['name', 'partner_ref'], + 'stock.picking': ['name', 'origin'], + 'hr.expense.sheet': ['name'], +}; diff --git a/middleware/packages/harness-verifier/src/deterministicChecker.ts b/middleware/packages/harness-verifier/src/deterministicChecker.ts index bb679a9f..657c74e2 100644 --- a/middleware/packages/harness-verifier/src/deterministicChecker.ts +++ b/middleware/packages/harness-verifier/src/deterministicChecker.ts @@ -4,6 +4,7 @@ import type { HardClaim, OdooRecordRef, } from './claimTypes.js'; +import { SOFT_ANCHOR_REF_FIELDS, hasOdooRecordAnchor } from './claimTypes.js'; /** * Deterministic verifier for HardClaims. Runs an INDEPENDENT read-only @@ -124,20 +125,46 @@ export class DeterministicChecker { } /** - * #129 — existence check for ANY claim anchored on an Odoo record - * (`hasOdooRecordAnchor`), independent of `claim.type`. The extractor - * is an LLM and types "INV/2026/0099 ist verbucht" as `id` in some - * samples and `qualitative` in others; the record either exists or it - * doesn't either way. Same transport as the `id` path: `read` by id, - * `search` by `name = ref`. Always resolves — never throws. + * #129 — existence check for a qualitative claim anchored on an Odoo + * record (`hasOdooRecordAnchor`). The extractor is an LLM and types + * "INV/2026/0099 ist verbucht" as `id` in some samples and `qualitative` + * in others; the record either exists or it doesn't either way. + * + * Narrower than the `id` path on purpose: a numeric id is `read` on any + * model, but a textual `ref` is only searched on models whose reference + * fields are known ({@link SOFT_ANCHOR_REF_FIELDS}) — `name` first, then + * the model's secondary reference field (vendor bill `ref`, sale order + * `client_order_ref`, …). Unknown model ⇒ `unverified`, the judge decides. + * Always resolves — never throws. */ async checkRecordExists(claim: Claim): Promise { - if (claim.expectedSource !== 'odoo') { - return unverified(claim, `existence check needs odoo source, got ${claim.expectedSource}`); + if (!hasOdooRecordAnchor(claim)) { + return unverified(claim, 'claim has no odoo record anchor'); } if (!this.odoo) return unverified(claim, 'no odoo reader configured'); + const ref = claim.odooRecord!; try { - return await this.checkOdooId(claim, claim.odooRecord); + if (typeof ref.id === 'number') { + return await this.checkOdooId(claim, ref); + } + const fields = SOFT_ANCHOR_REF_FIELDS[ref.model]; + if (!fields) { + return unverified(claim, `no known reference field for ${ref.model}`); + } + for (const field of fields) { + const ids = (await this.odoo.execute({ + model: ref.model, + method: 'search', + positionalArgs: [[[field, '=', ref.ref]]], + kwargs: { limit: 1 }, + })) as number[] | undefined; + if (Array.isArray(ids) && ids.length > 0) return verified(claim, 'odoo'); + } + return contradicted( + claim, + null, + `no ${ref.model} with ${fields.join('|')}="${String(ref.ref)}"`, + ); } catch (err) { const msg = err instanceof Error ? err.message : String(err); this.log(`[verifier/deterministic] FAIL exists claim=${claim.id} err=${msg}`); diff --git a/middleware/packages/harness-verifier/src/index.ts b/middleware/packages/harness-verifier/src/index.ts index dcadaea9..d24cfb59 100644 --- a/middleware/packages/harness-verifier/src/index.ts +++ b/middleware/packages/harness-verifier/src/index.ts @@ -41,6 +41,7 @@ export type { // claimTypes — shared vocabulary used by every other verifier file plus // the kernel-side `verifierService.ts` until sub-commit 2b moves it. export { + SOFT_ANCHOR_REF_FIELDS, hasOdooRecordAnchor, isBorderlineVerdict, isHardClaim, diff --git a/middleware/packages/harness-verifier/src/verifierPipeline.ts b/middleware/packages/harness-verifier/src/verifierPipeline.ts index c31c81cb..6ed0ecad 100644 --- a/middleware/packages/harness-verifier/src/verifierPipeline.ts +++ b/middleware/packages/harness-verifier/src/verifierPipeline.ts @@ -135,7 +135,7 @@ export class VerifierPipeline { // own — we never need to wait on one to start the other. const [hardVerdicts, softVerdicts] = await Promise.all([ this.deterministic.checkAll(hardToActuallyCheck), - this.checkSoftClaims(soft), + this.checkSoftClaims(soft, hard), ]); const all: ClaimVerdict[] = [ @@ -148,13 +148,26 @@ export class VerifierPipeline { } /** - * #129 — soft claims anchored on an Odoo record get a deterministic + * #129 — qualitative claims anchored on an Odoo record get a deterministic * existence check first. A record that does not exist is a contradiction * no judge can talk away, so those claims never reach the judge; claims - * whose record exists (or could not be checked) go to the judge unchanged. + * whose record exists (or could not be checked — reader error, unknown + * model) go to the judge unchanged (fail-open on the soft path). + * + * Anchors already covered by a hard claim in the same turn are skipped: + * the extractor is told to emit the `id` claim alongside the qualitative + * one, and the hard path (incl. the context-replay guard, which does not + * run on soft claims) already produces the verdict for that record — a + * second re-query would only duplicate the contradiction. */ - private async checkSoftClaims(soft: SoftClaim[]): Promise { - const anchored = soft.filter(hasOdooRecordAnchor); + private async checkSoftClaims( + soft: SoftClaim[], + hard: readonly HardClaim[], + ): Promise { + const coveredByHard = new Set(hard.map(anchorKey).filter(Boolean)); + const anchored = soft.filter( + (c) => hasOdooRecordAnchor(c) && !coveredByHard.has(anchorKey(c)), + ); if (anchored.length === 0) return this.judge.checkAll(soft); const existence = await Promise.all( @@ -275,6 +288,15 @@ function traceMissingCallVerdict( }; } +/** `model#id` / `model@ref` identity of a claim's Odoo anchor; '' when none. */ +function anchorKey(claim: Claim): string { + const ref = claim.odooRecord; + if (!ref || claim.expectedSource !== 'odoo') return ''; + if (typeof ref.id === 'number') return `${ref.model}#${String(ref.id)}`; + if (typeof ref.ref === 'string' && ref.ref.length > 0) return `${ref.model}@${ref.ref}`; + return ''; +} + function classify(claims: readonly Claim[]): { hard: HardClaim[]; soft: SoftClaim[]; diff --git a/middleware/test/verifierDeterministicChecker.test.ts b/middleware/test/verifierDeterministicChecker.test.ts index ab94b0e7..311ab677 100644 --- a/middleware/test/verifierDeterministicChecker.test.ts +++ b/middleware/test/verifierDeterministicChecker.test.ts @@ -2,6 +2,7 @@ import { describe, it } from 'node:test'; import { strict as assert } from 'node:assert'; import { DeterministicChecker, + hasOdooRecordAnchor, type Claim, type GraphReader, type HardClaim, @@ -387,9 +388,71 @@ describe('verifier/deterministicChecker - checkRecordExists (any claim type)', ( (await checker.checkRecordExists(anchoredQualitative({ expectedSource: 'graph' }))).status, 'unverified', ); + const noAnchor = await checker.checkRecordExists(anchoredQualitative({ odooRecord: undefined })); + assert.equal(noAnchor.status, 'unverified'); + if (noAnchor.status === 'unverified') assert.match(noAnchor.reason, /no odoo record anchor/); + }); + + it('falls back to the model-specific reference field (vendor bill `ref`) before contradicting', async () => { + const calls: OdooCall[] = []; + const odoo = stubOdoo((call) => { + const [[field]] = call.positionalArgs[0] as [[string, string, string]]; + return field === 'ref' ? [7] : []; + }, calls); + const checker = new DeterministicChecker({ odoo }); + const verdict = await checker.checkRecordExists( + anchoredQualitative({ + text: 'Die Lieferantenrechnung RE-4711 ist verbucht', + odooRecord: { model: 'account.move', ref: 'RE-4711' }, + }), + ); + assert.equal(verdict.status, 'verified'); + assert.deepEqual( + calls.map((c) => (c.positionalArgs[0] as [[string]])[0][0]), + ['name', 'ref'], + ); + }); + + it('stays unverified (judge decides) for a model without a known reference field', async () => { + const calls: OdooCall[] = []; + const checker = new DeterministicChecker({ odoo: stubOdoo(() => [], calls) }); + const verdict = await checker.checkRecordExists( + anchoredQualitative({ odooRecord: { model: 'hr.leave', ref: 'Urlaub 2026-03' } }), + ); + assert.equal(verdict.status, 'unverified'); + assert.equal(calls.length, 0); + }); + + it('does not treat `name` claims or bare person/company refs as anchors', () => { assert.equal( - (await checker.checkRecordExists(anchoredQualitative({ odooRecord: undefined }))).status, - 'unverified', + hasOdooRecordAnchor(anchoredQualitative({ type: 'name' })), + false, + 'name claims stay on the judge path', + ); + assert.equal( + hasOdooRecordAnchor( + anchoredQualitative({ odooRecord: { model: 'res.partner', ref: 'ACME GmbH' } }), + ), + false, + 'ref without a digit is not a document reference', + ); + assert.equal( + hasOdooRecordAnchor(anchoredQualitative({ odooRecord: { model: 'account.move', ref: '' } })), + false, + ); + assert.equal( + hasOdooRecordAnchor(anchoredQualitative({ odooRecord: { model: '', ref: 'INV/1' } })), + false, + ); + assert.equal( + hasOdooRecordAnchor(anchoredQualitative({ odooRecord: { model: 'account.move', id: 0 } })), + false, + ); + assert.equal( + hasOdooRecordAnchor(anchoredQualitative({ odooRecord: { model: 'hr.leave', id: 12 } })), + true, + 'numeric id anchors on any model', ); + assert.equal(hasOdooRecordAnchor(anchoredQualitative()), true); }); }); diff --git a/middleware/test/verifierPipeline.test.ts b/middleware/test/verifierPipeline.test.ts index 5d01bb2a..889df908 100644 --- a/middleware/test/verifierPipeline.test.ts +++ b/middleware/test/verifierPipeline.test.ts @@ -490,6 +490,75 @@ describe('verifier/pipeline - anchored soft claims', () => { assert.equal(verdict.status, 'approved_with_disclaimer'); }); + it('fails open: an unverifiable existence lookup still sends the claim to the judge', async () => { + let judgeCalls = 0; + const pipeline = new VerifierPipeline({ + extractor: stubExtractor([anchoredSoft()]), + deterministic: stubDeterministicWithExists((c) => ({ + status: 'unverified', + claim: c, + reason: 're-query error: ECONNRESET', + })), + judge: stubJudge((c) => { + judgeCalls += 1; + return { status: 'verified', claim: c, source: 'odoo' }; + }), + log: SILENT_LOG, + }); + const verdict = await pipeline.verify({ + runId: 'r_anchor_err', + userMessage: 'Ist die Rechnung verbucht?', + answer: 'Ja, die Rechnung INV/2026/0099 ist verbucht.', + domainToolsCalled: ['query_odoo_accounting'], + }); + assert.equal(judgeCalls, 1); + assert.equal(verdict.status, 'approved'); + }); + + it('skips the existence re-query when a hard id claim already covers the same anchor (no double contradiction)', async () => { + let existsCalls = 0; + let judgeCalls = 0; + const twinId = hardClaim({ + id: 'c_id', + text: 'INV/2026/0099', + type: 'id', + value: undefined, + odooRecord: { model: 'account.move', ref: 'INV/2026/0099' }, + }); + const pipeline = new VerifierPipeline({ + extractor: stubExtractor([twinId, anchoredSoft()]), + deterministic: { + ...stubDeterministic((c) => ({ + status: 'contradicted', + claim: c, + truth: null, + source: 'odoo', + })), + checkRecordExists(c: Claim): Promise { + existsCalls += 1; + return Promise.resolve({ status: 'contradicted', claim: c, truth: null, source: 'odoo' }); + }, + } as unknown as DeterministicChecker, + judge: stubJudge((c) => { + judgeCalls += 1; + return { status: 'unverified', claim: c, reason: 'no evidence' }; + }), + log: SILENT_LOG, + }); + const verdict = await pipeline.verify({ + runId: 'r_twin', + userMessage: 'Ist die Rechnung INV/2026/0099 bereits verbucht?', + answer: 'Ja, die Rechnung INV/2026/0099 ist verbucht und abgeschlossen.', + domainToolsCalled: ['query_odoo_accounting'], + }); + assert.equal(existsCalls, 0, 'hard twin owns the record verdict'); + assert.equal(judgeCalls, 1, 'qualitative twin still judged'); + assert.equal(verdict.status, 'blocked'); + if (verdict.status === 'blocked') { + assert.equal(verdict.contradictions.length, 1); + } + }); + it('leaves unanchored soft claims on the judge path untouched', async () => { let existsCalls = 0; const pipeline = new VerifierPipeline({