fix(verifier): give the judge the sentence a fragment claim was cut from

docs/CHANGELOG.md

@@ -18,6 +18,22 @@ entry. See `CONTRIBUTING.md` § Releases & changelog.
 
 ## [Unreleased]
 
+### Fixed — verifier: judge sees the sentence a fragment claim was cut from (#129 follow-up)
+
+- The claim extractor sometimes emits a subject-less fragment ("in die
+  IT-Abteilung") as the qualitative claim. The evidence judge deliberately
+  never sees the answer, so it could not know *who* moved where and returned
+  `unverified` — `golden-eval.yml` flaked on `blocked_contradiction_role`.
+- `Claim.context` now carries the enclosing sentence, cut deterministically
+  (no LLM) at `.`/`!`/`?`+whitespace or newline; a dot after a number or a
+  known abbreviation (`01.03.2023`, `1. März`, `z.B.`, `Dr.`) is not a
+  boundary; capped at 400 chars around the span. No context is attached when
+  the span occurs in more than one sentence (no guessing the subject) or
+  when it already is the whole sentence. The judge gets it as a `CONTEXT:`
+  line for disambiguation only and may not base a verdict on facts that
+  appear only there. Extractor prompt additionally asks for self-contained
+  qualitative claims. The contradiction double-check is unchanged.
+
 ### Added — Conductor workflows can be deleted from the library
 
 - **`DELETE /api/v1/operator/conductors/:slug`** removes a workflow with the

middleware/packages/harness-verifier/src/claimExtractor.ts

@@ -189,6 +189,7 @@ Strict rules:
 - Do NOT invent claims that are "implied" but not stated.
 - When in doubt, skip the claim rather than invent one.
 - A record reference (invoice, order or document number, numeric record id) is ALWAYS its own claim of type "id" with odoo_record.model and odoo_record.ref/id set — in addition to any qualitative claim about the same record.
+- A qualitative claim must be a self-contained statement: include the subject it is about in the verbatim span ("Anna Müller wechselte in die IT-Abteilung"), never a bare fragment ("in die IT-Abteilung"). An independent reviewer will judge the claim WITHOUT seeing the answer.
 - Return at most ${String(this.opts.maxClaims)} claims via the ${TOOL_NAME} tool.`;
 
     const user = `USER MESSAGE:
@@ -274,6 +275,87 @@ function readToolClaims(response: LlmResponse): unknown[] | null {
  * meet minimum invariants (known type, known source, text is verbatim,
  * text length sane). Returns null when the claim should be dropped.
  */
+export const MAX_CONTEXT_CHARS = 400;
+
+/** Tokens whose trailing dot does not end a sentence: ordinals ("1.", "3."),
+ *  and the common German/English abbreviations an ERP answer uses. */
+const NON_TERMINAL_BEFORE_DOT = /(?:\d+|z\.b|d\.h|u\.a|bzw|ca|dr|prof|nr|str|evtl|ggf|inkl|exkl|vgl|etc|usw|vs|abs|art|no|approx|e\.g|i\.e)$/i;
+
+/**
+ * #129 — the sentence of `answer` that contains `text` (case-insensitive),
+ * or `undefined` when `text` is absent, already spans the whole sentence,
+ * or occurs in more than one sentence (then we would only be guessing
+ * which subject the fragment belongs to — better no context than a wrong
+ * one, which could turn an `unverified` into a false `contradicted`).
+ *
+ * Sentence boundaries are `.`, `!`, `?` followed by whitespace/end, or a
+ * newline; a dot after a number or a known abbreviation ("01.03.2023",
+ * "1. März", "z.B.", "Dr.") is not a boundary. Pure string work, no LLM:
+ * the extractor sometimes emits a subject-less fragment ("in die
+ * IT-Abteilung") and the judge, which never sees the answer, needs the
+ * enclosing sentence to know *who* moved where.
+ */
+export function claimContext(text: string, answer: string): string | undefined {
+  const needle = text.trim().toLowerCase();
+  if (needle.length === 0) return undefined;
+  const hay = answer.toLowerCase();
+  // Lower-casing can change the code-unit length (e.g. U+0130) and would
+  // shift every offset — bail out rather than slice at the wrong place.
+  if (hay.length !== answer.length) return undefined;
+  const at = hay.indexOf(needle);
+  if (at < 0) return undefined;
+
+  const [start, end] = sentenceBounds(answer, at, needle.length);
+  const again = hay.indexOf(needle, at + 1);
+  if (again >= 0 && again >= end) return undefined; // second occurrence in another sentence
+
+  let [s, e] = [start, end];
+  if (e - s > MAX_CONTEXT_CHARS) {
+    // Over-long sentence: keep a window around the span, not its head.
+    const room = Math.floor((MAX_CONTEXT_CHARS - needle.length) / 2);
+    s = Math.max(s, at - room);
+    e = Math.min(e, at + needle.length + room);
+  }
+  const sentence = answer.slice(s, e).trim();
+  if (sentence.length === 0) return undefined;
+  if (stripTrailingPunctuation(sentence.toLowerCase()) === stripTrailingPunctuation(needle)) {
+    return undefined;
+  }
+  return sentence;
+}
+
+/** `[start, end)` of the sentence containing the span `[at, at+len)`. */
+function sentenceBounds(s: string, at: number, len: number): [number, number] {
+  let start = at;
+  while (start > 0 && !isSentenceBoundaryBefore(s, start)) start -= 1;
+  let end = at + len;
+  while (end < s.length && !isSentenceBoundaryAfter(s, end)) end += 1;
+  return [start, end];
+}
+
+/** True when position `i` starts a new sentence (previous char ends one). */
+function isSentenceBoundaryBefore(s: string, i: number): boolean {
+  const prev = s[i - 1];
+  if (prev === '\n') return true;
+  if (prev !== '.' && prev !== '!' && prev !== '?') return false;
+  if (!/\s/.test(s[i] ?? ' ')) return false;
+  return prev !== '.' || !NON_TERMINAL_BEFORE_DOT.test(s.slice(Math.max(0, i - 8), i - 1));
+}
+
+/** True when position `i` (exclusive end) closes a sentence — `i` is the
+ *  index just past the terminator. */
+function isSentenceBoundaryAfter(s: string, i: number): boolean {
+  const ch = s[i - 1];
+  if (s[i] === '\n') return true;
+  if (ch !== '.' && ch !== '!' && ch !== '?') return false;
+  if (!(i >= s.length || /\s/.test(s[i] ?? ''))) return false;
+  return ch !== '.' || !NON_TERMINAL_BEFORE_DOT.test(s.slice(Math.max(0, i - 9), i - 1));
+}
+
+function stripTrailingPunctuation(v: string): string {
+  return v.replace(/[.!?\s]+$/u, '');
+}
+
 function normaliseClaim(raw: unknown, idx: number, answer: string): Claim | null {
   if (!raw || typeof raw !== 'object') return null;
   const r = raw as RawClaim;
@@ -310,6 +392,9 @@ function normaliseClaim(raw: unknown, idx: number, answer: string): Claim | null
   const odoo = asOdooRecord(r.odoo_record);
   if (odoo) claim.odooRecord = odoo;
 
+  const context = claimContext(text, answer);
+  if (context) claim.context = context;
+
   return claim;
 }
 

middleware/packages/harness-verifier/src/claimTypes.ts

@@ -51,6 +51,12 @@ export type Aggregation = 'sum' | 'count' | 'avg' | 'max' | 'min';
 export interface Claim {
   id: string;                           // local: "c_001"
   text: string;                         // verbatim snippet from the answer
+  /** #129 — the sentence of the answer that contains `text`, cut
+   *  deterministically by the extractor (no LLM). Present only when it adds
+   *  something beyond `text`, i.e. the claim is a fragment ("in die
+   *  IT-Abteilung") whose subject lives elsewhere in the sentence. The judge
+   *  reads it for disambiguation; it never sees the whole answer. */
+  context?: string;
   type: ClaimType;
   expectedSource: ClaimSource;
   value?: number | string;              // parsed numeric or normalised literal

middleware/packages/harness-verifier/src/evidenceJudge.ts

@@ -1,6 +1,7 @@
 import type { LlmProvider, LlmResponse, ToolSpec } from '@omadia/llm-provider';
 import { textMessage, toolCalls } from '@omadia/llm-provider';
 import type { ClaimVerdict, SoftClaim } from './claimTypes.js';
+import { MAX_CONTEXT_CHARS } from './claimExtractor.js';
 
 /**
  * LLM-as-Judge for SoftClaims (names, qualitative statements) that can't
@@ -175,7 +176,8 @@ Rules:
 - verdict = "verified": evidence directly states the claim.
 - verdict = "unverified": evidence is silent, ambiguous, or only tangentially related. This is the DEFAULT when unsure.
 - verdict = "contradicted": evidence explicitly says something incompatible with the claim. Requires evidence_node_id.
-- Do NOT reward plausibility. If the evidence doesn't mention it, it's unverified — not verified.`;
+- Do NOT reward plausibility. If the evidence doesn't mention it, it's unverified — not verified.
+- When a CONTEXT line is present it is the single sentence the claim was cut from. Use it only to resolve what the claim refers to (its subject, tense); judge the CLAIM as meant in that sentence. Never base "contradicted" or "verified" on a fact that appears only in CONTEXT and not in CLAIM.`;
 
     const evidenceBlock = evidence
       .map(
@@ -184,7 +186,11 @@ Rules:
       )
       .join('\n\n');
 
-    const user = `CLAIM: ${claim.text}
+    const context =
+      claim.context && claim.context.trim().toLowerCase() !== claim.text.trim().toLowerCase()
+        ? `\nCONTEXT: ${truncate(claim.context, MAX_CONTEXT_CHARS)}`
+        : '';
+    const user = `CLAIM: ${claim.text}${context}
 CLAIM TYPE: ${claim.type}
 RELATED: ${claim.relatedEntities.join(', ') || '(none)'}
 

middleware/packages/harness-verifier/src/index.ts

@@ -32,7 +32,7 @@ export { activate } from './plugin.js';
 export type { VerifierBundle, VerifierPluginHandle } from './plugin.js';
 
 // ClaimExtractor
-export { ClaimExtractor } from './claimExtractor.js';
+export { ClaimExtractor, claimContext } from './claimExtractor.js';
 export type {
   ClaimExtractorOptions,
   ExtractInput,

middleware/test/verifierClaimExtractor.test.ts

@@ -0,0 +1,122 @@
+import { describe, it } from 'node:test';
+import { strict as assert } from 'node:assert';
+import { ClaimExtractor, claimContext } from '@omadia/verifier';
+
+// --- Stubs ---------------------------------------------------------------
+
+function stubLlm(claims: unknown[]): unknown {
+  return {
+    complete(): Promise<{ content: unknown[] }> {
+      return Promise.resolve({
+        content: [
+          { type: 'tool_call', name: 'record_claims', id: 'toolu_x', input: { claims } },
+        ],
+      });
+    },
+  };
+}
+
+const ANSWER =
+  'Anna Müller wechselte am 01.03.2023 in die IT-Abteilung [ref:n_emp_anna]. Sie leitet dort das Team. Fragen? Gern!';
+
+// --- Tests ---------------------------------------------------------------
+
+describe('verifier/claimExtractor - claimContext (enclosing sentence)', () => {
+  it('returns the sentence around a fragment, cut at sentence boundaries', () => {
+    assert.equal(
+      claimContext('in die IT-Abteilung', ANSWER),
+      'Anna Müller wechselte am 01.03.2023 in die IT-Abteilung [ref:n_emp_anna].',
+    );
+    assert.equal(claimContext('das Team', ANSWER), 'Sie leitet dort das Team.');
+  });
+
+  it('does not treat the dot inside a date, an ordinal or an abbreviation as a sentence end', () => {
+    assert.equal(
+      claimContext('IT-Abteilung', 'Sie kam am 01.03.2023 zur IT-Abteilung. Ende.'),
+      'Sie kam am 01.03.2023 zur IT-Abteilung.',
+    );
+    assert.equal(
+      claimContext('in die IT-Abteilung', 'Anna wechselte z.B. in die IT-Abteilung. Ende.'),
+      'Anna wechselte z.B. in die IT-Abteilung.',
+    );
+    assert.equal(
+      claimContext('in die IT', 'Anna wechselte am 1. März in die IT. Ende.'),
+      'Anna wechselte am 1. März in die IT.',
+    );
+    assert.equal(
+      claimContext('Buchhaltung', 'Vorher. Dr. Müller leitet die Buchhaltung. Danach.'),
+      'Dr. Müller leitet die Buchhaltung.',
+    );
+  });
+
+  it('returns undefined when the span occurs in more than one sentence (no guessing the subject)', () => {
+    assert.equal(
+      claimContext('in der IT', 'Bob ist in der IT. Anna wechselte in der IT-Abteilung.'),
+      undefined,
+    );
+    // Twice inside the SAME sentence is fine.
+    assert.equal(
+      claimContext('IT', 'Anna ist in der IT, genauer der IT-Leitung. Ende.'),
+      'Anna ist in der IT, genauer der IT-Leitung.',
+    );
+  });
+
+  it('treats markdown bullets / newlines as sentence boundaries', () => {
+    assert.equal(
+      claimContext('IT-Abteilung', '- Anna: IT-Abteilung\n- Bob: Sales'),
+      '- Anna: IT-Abteilung',
+    );
+  });
+
+  it('suppresses a context that equals the claim modulo trailing punctuation', () => {
+    assert.equal(claimContext('Der Vertrag ist beendet', 'Der Vertrag ist beendet.'), undefined);
+    assert.equal(
+      claimContext('Der Vertrag ist beendet.', 'Der Vertrag ist beendet. Mehr dazu.'),
+      undefined,
+    );
+  });
+
+  it('bails out when lower-casing would shift offsets (U+0130)', () => {
+    assert.equal(claimContext('Sales', 'İstanbul-Team: Sales. Ende.'), undefined);
+  });
+
+  it('matches case-insensitively and returns undefined when the span is absent or already a full sentence', () => {
+    assert.equal(
+      claimContext('anna müller wechselte am 01.03.2023 in die it-abteilung [ref:n_emp_anna].', ANSWER),
+      undefined,
+      'claim is the whole sentence → no extra context',
+    );
+    assert.equal(claimContext('Buchhaltung', ANSWER), undefined);
+    assert.equal(claimContext('', ANSWER), undefined);
+  });
+
+  it('caps the context length', () => {
+    const long = `${'x'.repeat(600)} Kern ${'y'.repeat(600)}`;
+    const ctx = claimContext('Kern', long);
+    assert.ok(ctx && ctx.length <= 400, `got ${String(ctx?.length)}`);
+    assert.match(ctx!, /Kern/);
+  });
+});
+
+describe('verifier/claimExtractor - extract', () => {
+  it('attaches context to fragment claims and leaves full-sentence claims without', async () => {
+    const extractor = new ClaimExtractor({
+      llm: stubLlm([
+        { text: 'in die IT-Abteilung', type: 'qualitative', expected_source: 'graph' },
+        {
+          text: 'Anna Müller wechselte am 01.03.2023 in die IT-Abteilung [ref:n_emp_anna].',
+          type: 'qualitative',
+          expected_source: 'graph',
+        },
+      ]) as never,
+      log: () => undefined,
+    });
+    const claims = await extractor.extract({ userMessage: 'Wo arbeitet Anna?', answer: ANSWER });
+    assert.equal(claims.length, 2);
+    assert.equal(
+      claims[0]!.context,
+      'Anna Müller wechselte am 01.03.2023 in die IT-Abteilung [ref:n_emp_anna].',
+    );
+    assert.equal(claims[1]!.context, undefined);
+  });
+});

middleware/test/verifierEvidenceJudge.test.ts

@@ -188,3 +188,52 @@ describe('verifier/evidenceJudge', () => {
     }
   });
 });
+
+// #129 golden flake `blocked_contradiction_role`: the extractor sometimes
+// emits a subject-less fragment ("in die IT-Abteilung") as the qualitative
+// claim. The judge deliberately never sees the answer, so it cannot know who
+// moved where → `unverified`. `claim.context` (the enclosing sentence, cut
+// deterministically by the extractor) restores the subject without leaking
+// the whole answer.
+describe('verifier/evidenceJudge - claim context', () => {
+  function capturingProvider(v: RecordedVerdict): { llm: unknown; prompts: string[] } {
+    const prompts: string[] = [];
+    const provider = {
+      complete(req: { messages: Array<{ content: unknown }> }): Promise<{ content: unknown[] }> {
+        const first = req.messages[0]?.content;
+        const text = Array.isArray(first)
+          ? first.map((p) => (p as { text?: string }).text ?? '').join('')
+          : String(first);
+        prompts.push(text);
+        return Promise.resolve({
+          content: [{ type: 'tool_call', name: 'record_verdict', id: 'toolu_x', input: v }],
+        });
+      },
+    };
+    return { llm: provider, prompts };
+  }
+
+  it('passes the enclosing sentence as CONTEXT when the claim carries one', async () => {
+    const { llm, prompts } = capturingProvider({ verdict: 'unverified' });
+    const judge = new EvidenceJudge({ llm: llm as never, fetcher: stubFetcher([SNIPPET]) });
+    await judge.check(
+      makeSoftClaim({
+        text: 'in die IT-Abteilung',
+        context: 'Anna Müller wechselte am 01.03.2023 in die IT-Abteilung.',
+      }),
+    );
+    assert.equal(prompts.length, 1);
+    assert.match(prompts[0]!, /CLAIM: in die IT-Abteilung/);
+    assert.match(prompts[0]!, /CONTEXT: Anna Müller wechselte am 01\.03\.2023 in die IT-Abteilung\./);
+  });
+
+  it('omits CONTEXT when the claim has none or it equals the claim text', async () => {
+    const { llm, prompts } = capturingProvider({ verdict: 'unverified' });
+    const judge = new EvidenceJudge({ llm: llm as never, fetcher: stubFetcher([SNIPPET]) });
+    await judge.check(makeSoftClaim());
+    await judge.check(makeSoftClaim({ context: 'John Doe ist Senior Developer bei byte5' }));
+    assert.equal(prompts.length, 2);
+    assert.doesNotMatch(prompts[0]!, /CONTEXT:/);
+    assert.doesNotMatch(prompts[1]!, /CONTEXT:/);
+  });
+});