diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 377ee288..bc7adc27 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -18,6 +18,22 @@ entry. See `CONTRIBUTING.md` § Releases & changelog. ## [Unreleased] +### Fixed — verifier: judge sees the sentence a fragment claim was cut from (#129 follow-up) + +- The claim extractor sometimes emits a subject-less fragment ("in die + IT-Abteilung") as the qualitative claim. The evidence judge deliberately + never sees the answer, so it could not know *who* moved where and returned + `unverified` — `golden-eval.yml` flaked on `blocked_contradiction_role`. +- `Claim.context` now carries the enclosing sentence, cut deterministically + (no LLM) at `.`/`!`/`?`+whitespace or newline; a dot after a number or a + known abbreviation (`01.03.2023`, `1. März`, `z.B.`, `Dr.`) is not a + boundary; capped at 400 chars around the span. No context is attached when + the span occurs in more than one sentence (no guessing the subject) or + when it already is the whole sentence. The judge gets it as a `CONTEXT:` + line for disambiguation only and may not base a verdict on facts that + appear only there. Extractor prompt additionally asks for self-contained + qualitative claims. The contradiction double-check is unchanged. + ### Added — Conductor workflows can be deleted from the library - **`DELETE /api/v1/operator/conductors/:slug`** removes a workflow with the diff --git a/middleware/packages/harness-verifier/src/claimExtractor.ts b/middleware/packages/harness-verifier/src/claimExtractor.ts index 8bdffde3..1f99a454 100644 --- a/middleware/packages/harness-verifier/src/claimExtractor.ts +++ b/middleware/packages/harness-verifier/src/claimExtractor.ts @@ -189,6 +189,7 @@ Strict rules: - Do NOT invent claims that are "implied" but not stated. - When in doubt, skip the claim rather than invent one. - A record reference (invoice, order or document number, numeric record id) is ALWAYS its own claim of type "id" with odoo_record.model and odoo_record.ref/id set — in addition to any qualitative claim about the same record. +- A qualitative claim must be a self-contained statement: include the subject it is about in the verbatim span ("Anna Müller wechselte in die IT-Abteilung"), never a bare fragment ("in die IT-Abteilung"). An independent reviewer will judge the claim WITHOUT seeing the answer. - Return at most ${String(this.opts.maxClaims)} claims via the ${TOOL_NAME} tool.`; const user = `USER MESSAGE: @@ -274,6 +275,87 @@ function readToolClaims(response: LlmResponse): unknown[] | null { * meet minimum invariants (known type, known source, text is verbatim, * text length sane). Returns null when the claim should be dropped. */ +export const MAX_CONTEXT_CHARS = 400; + +/** Tokens whose trailing dot does not end a sentence: ordinals ("1.", "3."), + * and the common German/English abbreviations an ERP answer uses. */ +const NON_TERMINAL_BEFORE_DOT = /(?:\d+|z\.b|d\.h|u\.a|bzw|ca|dr|prof|nr|str|evtl|ggf|inkl|exkl|vgl|etc|usw|vs|abs|art|no|approx|e\.g|i\.e)$/i; + +/** + * #129 — the sentence of `answer` that contains `text` (case-insensitive), + * or `undefined` when `text` is absent, already spans the whole sentence, + * or occurs in more than one sentence (then we would only be guessing + * which subject the fragment belongs to — better no context than a wrong + * one, which could turn an `unverified` into a false `contradicted`). + * + * Sentence boundaries are `.`, `!`, `?` followed by whitespace/end, or a + * newline; a dot after a number or a known abbreviation ("01.03.2023", + * "1. März", "z.B.", "Dr.") is not a boundary. Pure string work, no LLM: + * the extractor sometimes emits a subject-less fragment ("in die + * IT-Abteilung") and the judge, which never sees the answer, needs the + * enclosing sentence to know *who* moved where. + */ +export function claimContext(text: string, answer: string): string | undefined { + const needle = text.trim().toLowerCase(); + if (needle.length === 0) return undefined; + const hay = answer.toLowerCase(); + // Lower-casing can change the code-unit length (e.g. U+0130) and would + // shift every offset — bail out rather than slice at the wrong place. + if (hay.length !== answer.length) return undefined; + const at = hay.indexOf(needle); + if (at < 0) return undefined; + + const [start, end] = sentenceBounds(answer, at, needle.length); + const again = hay.indexOf(needle, at + 1); + if (again >= 0 && again >= end) return undefined; // second occurrence in another sentence + + let [s, e] = [start, end]; + if (e - s > MAX_CONTEXT_CHARS) { + // Over-long sentence: keep a window around the span, not its head. + const room = Math.floor((MAX_CONTEXT_CHARS - needle.length) / 2); + s = Math.max(s, at - room); + e = Math.min(e, at + needle.length + room); + } + const sentence = answer.slice(s, e).trim(); + if (sentence.length === 0) return undefined; + if (stripTrailingPunctuation(sentence.toLowerCase()) === stripTrailingPunctuation(needle)) { + return undefined; + } + return sentence; +} + +/** `[start, end)` of the sentence containing the span `[at, at+len)`. */ +function sentenceBounds(s: string, at: number, len: number): [number, number] { + let start = at; + while (start > 0 && !isSentenceBoundaryBefore(s, start)) start -= 1; + let end = at + len; + while (end < s.length && !isSentenceBoundaryAfter(s, end)) end += 1; + return [start, end]; +} + +/** True when position `i` starts a new sentence (previous char ends one). */ +function isSentenceBoundaryBefore(s: string, i: number): boolean { + const prev = s[i - 1]; + if (prev === '\n') return true; + if (prev !== '.' && prev !== '!' && prev !== '?') return false; + if (!/\s/.test(s[i] ?? ' ')) return false; + return prev !== '.' || !NON_TERMINAL_BEFORE_DOT.test(s.slice(Math.max(0, i - 8), i - 1)); +} + +/** True when position `i` (exclusive end) closes a sentence — `i` is the + * index just past the terminator. */ +function isSentenceBoundaryAfter(s: string, i: number): boolean { + const ch = s[i - 1]; + if (s[i] === '\n') return true; + if (ch !== '.' && ch !== '!' && ch !== '?') return false; + if (!(i >= s.length || /\s/.test(s[i] ?? ''))) return false; + return ch !== '.' || !NON_TERMINAL_BEFORE_DOT.test(s.slice(Math.max(0, i - 9), i - 1)); +} + +function stripTrailingPunctuation(v: string): string { + return v.replace(/[.!?\s]+$/u, ''); +} + function normaliseClaim(raw: unknown, idx: number, answer: string): Claim | null { if (!raw || typeof raw !== 'object') return null; const r = raw as RawClaim; @@ -310,6 +392,9 @@ function normaliseClaim(raw: unknown, idx: number, answer: string): Claim | null const odoo = asOdooRecord(r.odoo_record); if (odoo) claim.odooRecord = odoo; + const context = claimContext(text, answer); + if (context) claim.context = context; + return claim; } diff --git a/middleware/packages/harness-verifier/src/claimTypes.ts b/middleware/packages/harness-verifier/src/claimTypes.ts index 9d68765a..dbcb59b1 100644 --- a/middleware/packages/harness-verifier/src/claimTypes.ts +++ b/middleware/packages/harness-verifier/src/claimTypes.ts @@ -51,6 +51,12 @@ export type Aggregation = 'sum' | 'count' | 'avg' | 'max' | 'min'; export interface Claim { id: string; // local: "c_001" text: string; // verbatim snippet from the answer + /** #129 — the sentence of the answer that contains `text`, cut + * deterministically by the extractor (no LLM). Present only when it adds + * something beyond `text`, i.e. the claim is a fragment ("in die + * IT-Abteilung") whose subject lives elsewhere in the sentence. The judge + * reads it for disambiguation; it never sees the whole answer. */ + context?: string; type: ClaimType; expectedSource: ClaimSource; value?: number | string; // parsed numeric or normalised literal diff --git a/middleware/packages/harness-verifier/src/evidenceJudge.ts b/middleware/packages/harness-verifier/src/evidenceJudge.ts index d2622764..536a0417 100644 --- a/middleware/packages/harness-verifier/src/evidenceJudge.ts +++ b/middleware/packages/harness-verifier/src/evidenceJudge.ts @@ -1,6 +1,7 @@ import type { LlmProvider, LlmResponse, ToolSpec } from '@omadia/llm-provider'; import { textMessage, toolCalls } from '@omadia/llm-provider'; import type { ClaimVerdict, SoftClaim } from './claimTypes.js'; +import { MAX_CONTEXT_CHARS } from './claimExtractor.js'; /** * LLM-as-Judge for SoftClaims (names, qualitative statements) that can't @@ -175,7 +176,8 @@ Rules: - verdict = "verified": evidence directly states the claim. - verdict = "unverified": evidence is silent, ambiguous, or only tangentially related. This is the DEFAULT when unsure. - verdict = "contradicted": evidence explicitly says something incompatible with the claim. Requires evidence_node_id. -- Do NOT reward plausibility. If the evidence doesn't mention it, it's unverified — not verified.`; +- Do NOT reward plausibility. If the evidence doesn't mention it, it's unverified — not verified. +- When a CONTEXT line is present it is the single sentence the claim was cut from. Use it only to resolve what the claim refers to (its subject, tense); judge the CLAIM as meant in that sentence. Never base "contradicted" or "verified" on a fact that appears only in CONTEXT and not in CLAIM.`; const evidenceBlock = evidence .map( @@ -184,7 +186,11 @@ Rules: ) .join('\n\n'); - const user = `CLAIM: ${claim.text} + const context = + claim.context && claim.context.trim().toLowerCase() !== claim.text.trim().toLowerCase() + ? `\nCONTEXT: ${truncate(claim.context, MAX_CONTEXT_CHARS)}` + : ''; + const user = `CLAIM: ${claim.text}${context} CLAIM TYPE: ${claim.type} RELATED: ${claim.relatedEntities.join(', ') || '(none)'} diff --git a/middleware/packages/harness-verifier/src/index.ts b/middleware/packages/harness-verifier/src/index.ts index d24cfb59..014122a4 100644 --- a/middleware/packages/harness-verifier/src/index.ts +++ b/middleware/packages/harness-verifier/src/index.ts @@ -32,7 +32,7 @@ export { activate } from './plugin.js'; export type { VerifierBundle, VerifierPluginHandle } from './plugin.js'; // ClaimExtractor -export { ClaimExtractor } from './claimExtractor.js'; +export { ClaimExtractor, claimContext } from './claimExtractor.js'; export type { ClaimExtractorOptions, ExtractInput, diff --git a/middleware/test/verifierClaimExtractor.test.ts b/middleware/test/verifierClaimExtractor.test.ts new file mode 100644 index 00000000..45500288 --- /dev/null +++ b/middleware/test/verifierClaimExtractor.test.ts @@ -0,0 +1,122 @@ +import { describe, it } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { ClaimExtractor, claimContext } from '@omadia/verifier'; + +// --- Stubs --------------------------------------------------------------- + +function stubLlm(claims: unknown[]): unknown { + return { + complete(): Promise<{ content: unknown[] }> { + return Promise.resolve({ + content: [ + { type: 'tool_call', name: 'record_claims', id: 'toolu_x', input: { claims } }, + ], + }); + }, + }; +} + +const ANSWER = + 'Anna Müller wechselte am 01.03.2023 in die IT-Abteilung [ref:n_emp_anna]. Sie leitet dort das Team. Fragen? Gern!'; + +// --- Tests --------------------------------------------------------------- + +describe('verifier/claimExtractor - claimContext (enclosing sentence)', () => { + it('returns the sentence around a fragment, cut at sentence boundaries', () => { + assert.equal( + claimContext('in die IT-Abteilung', ANSWER), + 'Anna Müller wechselte am 01.03.2023 in die IT-Abteilung [ref:n_emp_anna].', + ); + assert.equal(claimContext('das Team', ANSWER), 'Sie leitet dort das Team.'); + }); + + it('does not treat the dot inside a date, an ordinal or an abbreviation as a sentence end', () => { + assert.equal( + claimContext('IT-Abteilung', 'Sie kam am 01.03.2023 zur IT-Abteilung. Ende.'), + 'Sie kam am 01.03.2023 zur IT-Abteilung.', + ); + assert.equal( + claimContext('in die IT-Abteilung', 'Anna wechselte z.B. in die IT-Abteilung. Ende.'), + 'Anna wechselte z.B. in die IT-Abteilung.', + ); + assert.equal( + claimContext('in die IT', 'Anna wechselte am 1. März in die IT. Ende.'), + 'Anna wechselte am 1. März in die IT.', + ); + assert.equal( + claimContext('Buchhaltung', 'Vorher. Dr. Müller leitet die Buchhaltung. Danach.'), + 'Dr. Müller leitet die Buchhaltung.', + ); + }); + + it('returns undefined when the span occurs in more than one sentence (no guessing the subject)', () => { + assert.equal( + claimContext('in der IT', 'Bob ist in der IT. Anna wechselte in der IT-Abteilung.'), + undefined, + ); + // Twice inside the SAME sentence is fine. + assert.equal( + claimContext('IT', 'Anna ist in der IT, genauer der IT-Leitung. Ende.'), + 'Anna ist in der IT, genauer der IT-Leitung.', + ); + }); + + it('treats markdown bullets / newlines as sentence boundaries', () => { + assert.equal( + claimContext('IT-Abteilung', '- Anna: IT-Abteilung\n- Bob: Sales'), + '- Anna: IT-Abteilung', + ); + }); + + it('suppresses a context that equals the claim modulo trailing punctuation', () => { + assert.equal(claimContext('Der Vertrag ist beendet', 'Der Vertrag ist beendet.'), undefined); + assert.equal( + claimContext('Der Vertrag ist beendet.', 'Der Vertrag ist beendet. Mehr dazu.'), + undefined, + ); + }); + + it('bails out when lower-casing would shift offsets (U+0130)', () => { + assert.equal(claimContext('Sales', 'İstanbul-Team: Sales. Ende.'), undefined); + }); + + it('matches case-insensitively and returns undefined when the span is absent or already a full sentence', () => { + assert.equal( + claimContext('anna müller wechselte am 01.03.2023 in die it-abteilung [ref:n_emp_anna].', ANSWER), + undefined, + 'claim is the whole sentence → no extra context', + ); + assert.equal(claimContext('Buchhaltung', ANSWER), undefined); + assert.equal(claimContext('', ANSWER), undefined); + }); + + it('caps the context length', () => { + const long = `${'x'.repeat(600)} Kern ${'y'.repeat(600)}`; + const ctx = claimContext('Kern', long); + assert.ok(ctx && ctx.length <= 400, `got ${String(ctx?.length)}`); + assert.match(ctx!, /Kern/); + }); +}); + +describe('verifier/claimExtractor - extract', () => { + it('attaches context to fragment claims and leaves full-sentence claims without', async () => { + const extractor = new ClaimExtractor({ + llm: stubLlm([ + { text: 'in die IT-Abteilung', type: 'qualitative', expected_source: 'graph' }, + { + text: 'Anna Müller wechselte am 01.03.2023 in die IT-Abteilung [ref:n_emp_anna].', + type: 'qualitative', + expected_source: 'graph', + }, + ]) as never, + log: () => undefined, + }); + const claims = await extractor.extract({ userMessage: 'Wo arbeitet Anna?', answer: ANSWER }); + assert.equal(claims.length, 2); + assert.equal( + claims[0]!.context, + 'Anna Müller wechselte am 01.03.2023 in die IT-Abteilung [ref:n_emp_anna].', + ); + assert.equal(claims[1]!.context, undefined); + }); +}); diff --git a/middleware/test/verifierEvidenceJudge.test.ts b/middleware/test/verifierEvidenceJudge.test.ts index 8369423c..0299aae2 100644 --- a/middleware/test/verifierEvidenceJudge.test.ts +++ b/middleware/test/verifierEvidenceJudge.test.ts @@ -188,3 +188,52 @@ describe('verifier/evidenceJudge', () => { } }); }); + +// #129 golden flake `blocked_contradiction_role`: the extractor sometimes +// emits a subject-less fragment ("in die IT-Abteilung") as the qualitative +// claim. The judge deliberately never sees the answer, so it cannot know who +// moved where → `unverified`. `claim.context` (the enclosing sentence, cut +// deterministically by the extractor) restores the subject without leaking +// the whole answer. +describe('verifier/evidenceJudge - claim context', () => { + function capturingProvider(v: RecordedVerdict): { llm: unknown; prompts: string[] } { + const prompts: string[] = []; + const provider = { + complete(req: { messages: Array<{ content: unknown }> }): Promise<{ content: unknown[] }> { + const first = req.messages[0]?.content; + const text = Array.isArray(first) + ? first.map((p) => (p as { text?: string }).text ?? '').join('') + : String(first); + prompts.push(text); + return Promise.resolve({ + content: [{ type: 'tool_call', name: 'record_verdict', id: 'toolu_x', input: v }], + }); + }, + }; + return { llm: provider, prompts }; + } + + it('passes the enclosing sentence as CONTEXT when the claim carries one', async () => { + const { llm, prompts } = capturingProvider({ verdict: 'unverified' }); + const judge = new EvidenceJudge({ llm: llm as never, fetcher: stubFetcher([SNIPPET]) }); + await judge.check( + makeSoftClaim({ + text: 'in die IT-Abteilung', + context: 'Anna Müller wechselte am 01.03.2023 in die IT-Abteilung.', + }), + ); + assert.equal(prompts.length, 1); + assert.match(prompts[0]!, /CLAIM: in die IT-Abteilung/); + assert.match(prompts[0]!, /CONTEXT: Anna Müller wechselte am 01\.03\.2023 in die IT-Abteilung\./); + }); + + it('omits CONTEXT when the claim has none or it equals the claim text', async () => { + const { llm, prompts } = capturingProvider({ verdict: 'unverified' }); + const judge = new EvidenceJudge({ llm: llm as never, fetcher: stubFetcher([SNIPPET]) }); + await judge.check(makeSoftClaim()); + await judge.check(makeSoftClaim({ context: 'John Doe ist Senior Developer bei byte5' })); + assert.equal(prompts.length, 2); + assert.doesNotMatch(prompts[0]!, /CONTEXT:/); + assert.doesNotMatch(prompts[1]!, /CONTEXT:/); + }); +});