fix(verifier): check Odoo record existence for anchored soft claims (#129 golden flake)

docs/CHANGELOG.md

@@ -18,6 +18,25 @@ entry. See `CONTRIBUTING.md` § Releases & changelog.
 
 ## [Unreleased]
 
+### Fixed — verifier: hallucinated record references no longer pass with a disclaimer (#129, PR #781)
+
+- **Behaviour change (blocking).** A qualitative answer that names a concrete
+  Odoo document (`INV/2026/0099`, `SO0123`, `RE-4711`) which does **not**
+  exist is now `blocked`, not `approved_with_disclaimer`. Root cause: the
+  LLM claim extractor typed such sentences as `qualitative` in roughly a
+  third of samples, and qualitative claims bypassed the deterministic
+  re-query entirely. `golden-eval.yml` flaked on exactly this
+  (`blocked_deterministic_id_absent`).
+- Scope is deliberately narrow: only `qualitative` claims with a
+  document-style `ref` (contains a digit) or numeric `id`; only models with
+  known reference fields (`account.move` `name|ref`, `sale.order`
+  `name|client_order_ref`, `purchase.order` `name|partner_ref`,
+  `stock.picking` `name|origin`, `account.payment`, `hr.expense.sheet`).
+  Person/company names, unknown models and reader errors stay on the judge
+  path (fail-open). Anchors already covered by a hard `id` claim in the same
+  turn are not re-queried twice.
+- Extractor prompt now asks for a separate `id` claim per record reference.
+
 ### Added — provenance verification surface: verify API, signed export, offline verifier (#761)
 
 - **`GET /api/v1/operator/provenance/verify`** walks the stored chain,

middleware/packages/harness-verifier/src/claimExtractor.ts

@@ -91,7 +91,7 @@ const toolSpec: ToolSpec = {
               type: 'string',
               enum: [...CLAIM_TYPES],
               description:
-                'amount=money/number+unit; id=record reference; date=calendar date; name=person/customer with context; aggregate=sum/count/avg over a set (especially HR leave totals); qualitative=non-numeric claim about an entity.',
+                'amount=money/number+unit; id=record reference (invoice/order/document number such as "INV/2026/0042", or a numeric record id) — ALWAYS emit a separate id claim for every record reference, even when the sentence also makes a qualitative statement about that record; date=calendar date; name=person/customer with context; aggregate=sum/count/avg over a set (especially HR leave totals); qualitative=non-numeric claim about an entity.',
             },
             expected_source: {
               type: 'string',
@@ -188,6 +188,7 @@ Strict rules:
 - Do NOT extract the user's question, instructions, or meta-commentary.
 - Do NOT invent claims that are "implied" but not stated.
 - When in doubt, skip the claim rather than invent one.
+- A record reference (invoice, order or document number, numeric record id) is ALWAYS its own claim of type "id" with odoo_record.model and odoo_record.ref/id set — in addition to any qualitative claim about the same record.
 - Return at most ${String(this.opts.maxClaims)} claims via the ${TOOL_NAME} tool.`;
 
     const user = `USER MESSAGE:

middleware/packages/harness-verifier/src/claimTypes.ts

@@ -185,3 +185,46 @@ export function isHardClaim(claim: Claim): claim is HardClaim {
 export function isSoftClaim(claim: Claim): claim is SoftClaim {
   return claim.type === 'name' || claim.type === 'qualitative';
 }
+
+/**
+ * #129 — a *qualitative* claim is *anchored* when it names a concrete Odoo
+ * record (`odooRecord.id` or a document-style `.ref`) with
+ * `expectedSource: 'odoo'`. Whether that record EXISTS is checkable
+ * deterministically no matter how the extractor typed the claim — Haiku
+ * types "INV/2026/0099 ist verbucht" as `id` in some samples and
+ * `qualitative` in others.
+ *
+ * Deliberately narrow (review on PR #781): `name` claims are excluded —
+ * an exact `name = "John Doe"` search would refute "Doe, John" and block a
+ * correct answer — and a `ref` only counts when it looks like a document
+ * sequence (contains a digit), never a bare person/company name.
+ * Pure predicate; no I/O.
+ */
+/** A reference that starts with a non-space and carries at least one digit —
+ *  "INV/2026/0042", "SO0123", "RE-4711"; not "ACME GmbH" or "John Doe". */
+const DOCUMENT_REF_PATTERN = /^\S.*\d/;
+
+export function hasOdooRecordAnchor(claim: Claim): boolean {
+  if (claim.type !== 'qualitative' || claim.expectedSource !== 'odoo') return false;
+  const ref = claim.odooRecord;
+  if (!ref || typeof ref.model !== 'string' || ref.model.length === 0) return false;
+  if (typeof ref.id === 'number' && Number.isInteger(ref.id) && ref.id > 0) return true;
+  return typeof ref.ref === 'string' && DOCUMENT_REF_PATTERN.test(ref.ref);
+}
+
+/**
+ * Per-model fields that may hold a human-readable record reference. `name`
+ * is the sequence on customer invoices / orders / pickings, but vendor bills
+ * carry the supplier number in `ref`, sale orders the customer's PO in
+ * `client_order_ref`, purchase orders the vendor's in `partner_ref`.
+ * A model outside this map has no safe reference field → the existence
+ * check stays `unverified` (judge decides) instead of guessing.
+ */
+export const SOFT_ANCHOR_REF_FIELDS: Readonly<Record<string, readonly string[]>> = {
+  'account.move': ['name', 'ref'],
+  'account.payment': ['name', 'ref'],
+  'sale.order': ['name', 'client_order_ref'],
+  'purchase.order': ['name', 'partner_ref'],
+  'stock.picking': ['name', 'origin'],
+  'hr.expense.sheet': ['name'],
+};

middleware/packages/harness-verifier/src/deterministicChecker.ts

@@ -4,6 +4,7 @@ import type {
   HardClaim,
   OdooRecordRef,
 } from './claimTypes.js';
+import { SOFT_ANCHOR_REF_FIELDS, hasOdooRecordAnchor } from './claimTypes.js';
 
 /**
  * Deterministic verifier for HardClaims. Runs an INDEPENDENT read-only
@@ -123,6 +124,54 @@ export class DeterministicChecker {
     return Promise.all(claims.map((c) => this.check(c)));
   }
 
+  /**
+   * #129 — existence check for a qualitative claim anchored on an Odoo
+   * record (`hasOdooRecordAnchor`). The extractor is an LLM and types
+   * "INV/2026/0099 ist verbucht" as `id` in some samples and `qualitative`
+   * in others; the record either exists or it doesn't either way.
+   *
+   * Narrower than the `id` path on purpose: a numeric id is `read` on any
+   * model, but a textual `ref` is only searched on models whose reference
+   * fields are known ({@link SOFT_ANCHOR_REF_FIELDS}) — `name` first, then
+   * the model's secondary reference field (vendor bill `ref`, sale order
+   * `client_order_ref`, …). Unknown model ⇒ `unverified`, the judge decides.
+   * Always resolves — never throws.
+   */
+  async checkRecordExists(claim: Claim): Promise<ClaimVerdict> {
+    if (!hasOdooRecordAnchor(claim)) {
+      return unverified(claim, 'claim has no odoo record anchor');
+    }
+    if (!this.odoo) return unverified(claim, 'no odoo reader configured');
+    const ref = claim.odooRecord!;
+    try {
+      if (typeof ref.id === 'number') {
+        return await this.checkOdooId(claim, ref);
+      }
+      const fields = SOFT_ANCHOR_REF_FIELDS[ref.model];
+      if (!fields) {
+        return unverified(claim, `no known reference field for ${ref.model}`);
+      }
+      for (const field of fields) {
+        const ids = (await this.odoo.execute({
+          model: ref.model,
+          method: 'search',
+          positionalArgs: [[[field, '=', ref.ref]]],
+          kwargs: { limit: 1 },
+        })) as number[] | undefined;
+        if (Array.isArray(ids) && ids.length > 0) return verified(claim, 'odoo');
+      }
+      return contradicted(
+        claim,
+        null,
+        `no ${ref.model} with ${fields.join('|')}="${String(ref.ref)}"`,
+      );
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      this.log(`[verifier/deterministic] FAIL exists claim=${claim.id} err=${msg}`);
+      return unverified(claim, `re-query error: ${msg}`);
+    }
+  }
+
   // --- Odoo ---------------------------------------------------------------
 
   private async checkOdoo(claim: HardClaim): Promise<ClaimVerdict> {
@@ -250,8 +299,10 @@ export class DeterministicChecker {
     return verified(claim, 'odoo');
   }
 
+  /** Record-existence check — shared by the `id` path and
+   *  {@link checkRecordExists}, hence typed on `Claim` not `HardClaim`. */
   private async checkOdooId(
-    claim: HardClaim,
+    claim: Claim,
     ref: OdooRecordRef | undefined,
   ): Promise<ClaimVerdict> {
     if (!ref) return unverified(claim, 'id claim without odoo model');

middleware/packages/harness-verifier/src/index.ts

@@ -41,6 +41,8 @@ export type {
 // claimTypes — shared vocabulary used by every other verifier file plus
 // the kernel-side `verifierService.ts` until sub-commit 2b moves it.
 export {
+  SOFT_ANCHOR_REF_FIELDS,
+  hasOdooRecordAnchor,
   isBorderlineVerdict,
   isHardClaim,
   isSoftClaim,

middleware/packages/harness-verifier/src/verifierPipeline.ts

@@ -6,7 +6,7 @@ import type {
   VerifierInput,
   VerifierVerdict,
 } from './claimTypes.js';
-import { isHardClaim, isSoftClaim } from './claimTypes.js';
+import { hasOdooRecordAnchor, isHardClaim, isSoftClaim } from './claimTypes.js';
 import type { ClaimExtractor } from './claimExtractor.js';
 import type { DeterministicChecker } from './deterministicChecker.js';
 import type { EvidenceJudge } from './evidenceJudge.js';
@@ -18,7 +18,11 @@ import { shouldTriggerVerifier } from './triggerRouter.js';
  *
  *   answer → triggerRouter → claimExtractor → classify
  *                            ├─► DeterministicChecker (hard claims, parallel)
- *                            └─► EvidenceJudge        (soft claims, parallel)
+ *                            ├─► DeterministicChecker.checkRecordExists
+ *                            │     (soft claims anchored on an Odoo record —
+ *                            │      #129: a missing record blocks before any
+ *                            │      judge call, whatever the extractor typed)
+ *                            └─► EvidenceJudge        (remaining soft claims)
  *                            → aggregate → VerifierVerdict
  *
  * Never throws. On any failure below the API level the pipeline returns
@@ -131,7 +135,7 @@ export class VerifierPipeline {
     // own — we never need to wait on one to start the other.
     const [hardVerdicts, softVerdicts] = await Promise.all([
       this.deterministic.checkAll(hardToActuallyCheck),
-      this.judge.checkAll(soft),
+      this.checkSoftClaims(soft, hard),
     ]);
 
     const all: ClaimVerdict[] = [
@@ -142,6 +146,44 @@ export class VerifierPipeline {
     ];
     return aggregate(all, started);
   }
+
+  /**
+   * #129 — qualitative claims anchored on an Odoo record get a deterministic
+   * existence check first. A record that does not exist is a contradiction
+   * no judge can talk away, so those claims never reach the judge; claims
+   * whose record exists (or could not be checked — reader error, unknown
+   * model) go to the judge unchanged (fail-open on the soft path).
+   *
+   * Anchors already covered by a hard claim in the same turn are skipped:
+   * the extractor is told to emit the `id` claim alongside the qualitative
+   * one, and the hard path (incl. the context-replay guard, which does not
+   * run on soft claims) already produces the verdict for that record — a
+   * second re-query would only duplicate the contradiction.
+   */
+  private async checkSoftClaims(
+    soft: SoftClaim[],
+    hard: readonly HardClaim[],
+  ): Promise<ClaimVerdict[]> {
+    const coveredByHard = new Set(hard.map(anchorKey).filter(Boolean));
+    const anchored = soft.filter(
+      (c) => hasOdooRecordAnchor(c) && !coveredByHard.has(anchorKey(c)),
+    );
+    if (anchored.length === 0) return this.judge.checkAll(soft);
+
+    const existence = await Promise.all(
+      anchored.map((c) => this.deterministic.checkRecordExists(c)),
+    );
+    const contradicted = existence.filter((v) => v.status === 'contradicted');
+    const blockedIds = new Set(contradicted.map((v) => v.claim.id));
+    if (blockedIds.size > 0) {
+      this.log(
+        `[verifier/pipeline] anchored soft claim(s) refuted by record re-query: ${[...blockedIds].join(',')}`,
+      );
+    }
+    const forJudge = soft.filter((c) => !blockedIds.has(c.id));
+    const judged = await this.judge.checkAll(forJudge);
+    return [...contradicted, ...judged];
+  }
 }
 
 /**
@@ -246,6 +288,15 @@ function traceMissingCallVerdict(
   };
 }
 
+/** `model#id` / `model@ref` identity of a claim's Odoo anchor; '' when none. */
+function anchorKey(claim: Claim): string {
+  const ref = claim.odooRecord;
+  if (!ref || claim.expectedSource !== 'odoo') return '';
+  if (typeof ref.id === 'number') return `${ref.model}#${String(ref.id)}`;
+  if (typeof ref.ref === 'string' && ref.ref.length > 0) return `${ref.model}@${ref.ref}`;
+  return '';
+}
+
 function classify(claims: readonly Claim[]): {
   hard: HardClaim[];
   soft: SoftClaim[];