From 48aebe2f1596c32634c9c4a83e648d62987b84be Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 17:27:42 +0900 Subject: [PATCH 1/9] fix: preserve semantic document evidence units --- CHANGELOG.md | 10 ++ ...101-semantic-document-evidence-contract.md | 65 ++++++++++ .../PRODUCT_TECHNICAL_GAP_REFERENCES.md | 24 ++++ docs/product-technical-gap-baseline.md | 76 +++++++++++ frontend/src/PostBody.stories.tsx | 51 ++++++++ frontend/src/PostBody.test.tsx | 44 +++++++ frontend/src/PostBody.tsx | 46 ++++++- frontend/src/postBodyDisplay.test.ts | 25 +++- frontend/src/postBodyDisplay.ts | 91 +++++++++++-- lineageweave/chunking.py | 120 +++++++++++++++++- lineageweave/post_content_normalization.py | 9 +- tests/test_chunking.py | 49 +++++++ tests/test_person_mention_projection.py | 4 + tests/test_post_content_normalization.py | 14 ++ 14 files changed, 610 insertions(+), 18 deletions(-) create mode 100644 docs/adr/0101-semantic-document-evidence-contract.md create mode 100644 docs/doctoring/PRODUCT_TECHNICAL_GAP_REFERENCES.md create mode 100644 docs/product-technical-gap-baseline.md create mode 100644 frontend/src/PostBody.stories.tsx diff --git a/CHANGELOG.md b/CHANGELOG.md index f5719013a..bb905daae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to this project are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versioning follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Fixed + +- Preserve source-order nested list units, numeric superscript footnotes, + HTML/OOXML table rows, and recognizable Markdown table rows across the + semantic-unit parser and buyer body renderer. See the [product and + technical gap baseline](docs/product-technical-gap-baseline.md) and + [ADR 0101](docs/adr/0101-semantic-document-evidence-contract.md). + ## [2.10.0] - 2026-08-18 ### Added diff --git a/docs/adr/0101-semantic-document-evidence-contract.md b/docs/adr/0101-semantic-document-evidence-contract.md new file mode 100644 index 000000000..8875bde34 --- /dev/null +++ b/docs/adr/0101-semantic-document-evidence-contract.md @@ -0,0 +1,65 @@ +# ADR 0101 — Preserve semantic document evidence across source and buyer views + +**Decision status:** Proposed on the stacked product-gap branch +**Date:** 2026-08-20 +**Figma File ID:** `1Su3lDRmiZdcUs47t1QwIX` +**Related baseline:** [Product and Technical Gap Baseline](../product-technical-gap-baseline.md) + +## Context + +Live aggregate inspection identified recurring buyer-visible loss at the +source boundary: superscript footnotes, nested list order/depth, table rows, +and Markdown tables were not represented consistently between Python +ingestion, PostgreSQL units, and the React popup. A flattened string cannot +reconstruct a table row, a branch in a list, or the position of an image. It +also makes a later LLM summary less auditable. + +The HTML Living Standard defines the semantic elements used by the source +boundary, including `ol`, `li`, `table`, `tr`, and `sup`. CommonMark provides a +versioned baseline for Markdown block parsing; table syntax remains an +extension in many Markdown dialects, so the implementation accepts only a +recognizable header/separator/data shape and otherwise preserves plain text. + +## Decision + +1. Parse source content into ordered semantic units before embedding or + summarization. A unit retains its source label, source order, indentation + metadata, and image position. +2. Group HTML and OOXML table cells into row units. Markdown tables are + recognized only when a header row is immediately followed by a separator + row; the separator itself is not evidence content. +3. Treat explicit CSS/OOXML indentation as authoritative. List-container + nesting contributes structural depth but must not double-count an explicit + source width. +4. Mark a numeric footnote only when the source uses a numeric `sup` marker; + a numeric table cell or ordinary numbered text is not a footnote by itself. +5. Keep the frontend's raw-source fallback aligned with the persisted unit + labels. Persisted row units render as accessible tables; unresolved + structure remains visibly unresolved and actionable. + +## Rejected alternatives + +- Flattening all bodies into one embedding string: loses row, list, and image + boundaries and cannot be repaired at display time. +- Treating every leading number as a footnote: mislabels table rows and + numbered instructions. +- Calling a provider directly from the parser: violates the orchestrator + trust boundary and makes evidence/cost lineage incomplete. +- Creating a separate parsing service: the existing shared chunker and + persistence boundary are sufficient; Ponytail favors the smaller change. + +## Consequences + +- Search and summaries receive smaller, meaningful units without exposing raw + markup or image base64. +- The database keeps the existing normalized unit tables; this decision adds + no denormalized JSON field or new service. +- Markdown dialects outside the narrow recognized shape remain plain text and + are reported as a future parser extension rather than guessed. + +## Verification + +The baseline's synthetic tests cover numeric superscript footnotes, marker +footnotes, nested `ol`/`ul`/`oi` order and depth, HTML/OOXML rows, Markdown +rows, React table rendering, and unresolved indentation. Full CI remains the +release gate. diff --git a/docs/doctoring/PRODUCT_TECHNICAL_GAP_REFERENCES.md b/docs/doctoring/PRODUCT_TECHNICAL_GAP_REFERENCES.md new file mode 100644 index 000000000..60e2d8c00 --- /dev/null +++ b/docs/doctoring/PRODUCT_TECHNICAL_GAP_REFERENCES.md @@ -0,0 +1,24 @@ +# Product technical-gap references + +These references are the normative basis for the semantic-unit boundary in +[ADR 0101](../adr/0101-semantic-document-evidence-contract.md). Dates and +versions are recorded so a later standards refresh can be reviewed rather +than silently changing parser behavior. + +## APA 7th edition + +CommonMark. (2024). *CommonMark spec (Version 0.31.2)*. https://spec.commonmark.org/0.31.2/ + +WHATWG. (2026). *HTML: Living Standard*. https://html.spec.whatwg.org/multipage/ + +## Applied mapping + +| Source | Boundary used in LineageWeave | +| --- | --- | +| CommonMark (2024) | Recognizable Markdown block/header/separator shape; unrecognized dialects remain source text. | +| WHATWG (2026) | HTML list, table-row, and `sup` semantics; source order and element identity are retained as unit metadata. | + +These standards define syntax and semantics, not an LLM extraction license. +Provider-derived summaries, image descriptions, project boundaries, and +5W1H values still require contextual-orchestrator provenance and explicit +source evidence. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md new file mode 100644 index 000000000..d7458abc8 --- /dev/null +++ b/docs/product-technical-gap-baseline.md @@ -0,0 +1,76 @@ +# Product and Technical Gap Baseline + +**Status:** active delivery baseline +**Owner boundary:** LineageWeave source ingestion, evidence projection, and buyer popup +**Design boundary:** Figma file `1Su3lDRmiZdcUs47t1QwIX` (see [ADR 0002](adr/0002-figma-access-boundary.md)) +**Data rule:** this document uses synthetic case labels only; production post identifiers, names, bodies, and screenshots never enter the repository. + +## Buyer outcome + +When a buyer opens a source record, the product must preserve the meaning that +was visible in the source: document order, table rows, list depth, footnotes, +image regions, named actors, project boundaries, time/place/method/reason, and +the next person or team action. Every displayed fact must link back to a +persisted source unit or an explicitly unavailable channel. The UI must never +turn missing evidence into a confident negative or an invented actor. + +## Current gap register + +| Gap | Buyer-visible failure | Durable contract | Verification | +| --- | --- | --- | --- | +| `structure-footnote` | Superscript or marker footnotes look like ordinary body text. | A footnote keeps its source unit and `footnote` label; numeric superscripts are accepted only when the source marks them as superscript. | Synthetic HTML tests cover `*`, `†`, `‡`, and numeric `` markers. | +| `structure-list-depth` | Nested `li`/exporter `oi` items are flattened or appear in reverse order. | Parent items precede children; list depth remains explicit metadata and is never inferred from unrelated whitespace. | Synthetic nested `ol`/`ul`/`oi` tests cover order and depth. | +| `structure-table-rows` | HTML/OOXML tables lose row and cell relationships. | One persisted unit represents one row, with cells joined by a stable delimiter and a source label. | Synthetic HTML and Word-table tests cover headers, data rows, and nested cell blocks. | +| `structure-markdown-table` | A Markdown table becomes one flattened paragraph in the API or popup. | A CommonMark-style header/separator/data block becomes row units and renders as a table in the buyer surface. | Python normalization, persistence contract, React render, and Storybook checks cover a synthetic table. | +| `structure-explicit-indent` | CSS/OOXML indentation is displayed at the wrong level. | Explicit source width is authoritative; list-container depth is structural and does not double-count explicit width. | CSS shorthand, OOXML, nested list, and unresolved-structure tests. | +| `semantic-project-boundary` | Events from distinct projects or matters are blended into one narrative. | Each event, project mention, and evidence phrase carries a stable project/matter boundary; ambiguous candidates remain below grouping threshold. | Multi-matter synthetic summary contract and persistence tests. | +| `semantic-actor-identity` | A company factory is mislabeled as a partner/supplier, or a group of PMs is shown without names and affiliations. | Actor type, stated affiliation, requester, and processor remain separate source-grounded fields; no organization relationship is inferred without evidence. | Parser rejection tests, role catalog foreign-key tests, and ambiguous-relationship tests. | +| `semantic-five-w-one-h` | `when`, `where`, `why`, or `how` disappears from the summary. | Each supported slot stores value plus evidence; absent slots expose a next action rather than a placeholder fact. | Slot parser, API contract, and buyer popup tests cover present, absent, and conflicting evidence. | +| `vision-region-evidence` | Image tables receive generic OCR and only a partial visual region is described. | Region discovery covers the full image, persists coordinates, OCR, caption, tags, and independent embeddings through contextual-orchestrator. | Synthetic multi-region image client tests cover full coverage, table-like regions, failures, and document order. | +| `lineage-dag` | Related records exist but the buyer cannot see the branch history. | The focused post subgraph is rendered as a source-order DAG with branch points and clickable nodes; unrelated project groups are excluded. | Layout unit tests, browser interaction tests, and Storybook inventory checks. | + +## Implementation order + +1. **Source structure:** finish the shared Python/TypeScript semantic-unit + contract for footnotes, lists, HTML/OOXML rows, and Markdown rows. +2. **Evidence persistence:** ensure the queue considers structure, unit + embeddings, image-region descriptions, and region embeddings complete as a + single invariant; retry only bounded, changed, or incomplete work. +3. **Semantic extraction:** keep contextual-orchestrator as the only LLM/VISION + boundary, and persist project, actor, requester, processor, and 5W1H + evidence in normalized relations. +4. **Buyer surface:** render tables, footnotes, image evidence, focused DAG + branches, and actionable empty states with shared design tokens and + Storybook coverage. +5. **Release gate:** run the complete backend, PostgreSQL, frontend, Storybook, + security, coverage, and protected-merge checks. Version and changelog + changes are made only after the current protected head has formal review and + terminal required checks. + +## Evidence and safety rules + +- Synthetic fixtures are the only committed test data. Live validation returns + aggregate counts and status codes, never source text or identifiable names. +- Missing provider channels remain unavailable. They do not produce zero + vectors, guessed roles, generic captions, or fabricated 5W1H values. +- Provider credentials and model selection remain in contextual-orchestrator; + this repository never calls a provider API directly. +- A relationship involving the organization's own factory is not classified + from a job title or catalog hint alone. The source must state the relation or + the result remains unresolved. +- Structural rendering is additive: the raw source remains available so a + buyer can compare the derived view with the original evidence. + +## Definition of done + +A gap is closed only when its synthetic behavior test fails before the change, +passes after the change, its persistence/API contract is covered where +applicable, the buyer action is visible in frontend tests and Storybook, and +the corresponding ADR and APA 7 references are updated. A green local test is +not a protected merge, release, or production-data success claim. + +## References + +See [ADR 0101](adr/0101-semantic-document-evidence-contract.md) and +[`docs/doctoring/PRODUCT_TECHNICAL_GAP_REFERENCES.md`](doctoring/PRODUCT_TECHNICAL_GAP_REFERENCES.md) +for the normative standards boundary. diff --git a/frontend/src/PostBody.stories.tsx b/frontend/src/PostBody.stories.tsx new file mode 100644 index 000000000..43260ec2d --- /dev/null +++ b/frontend/src/PostBody.stories.tsx @@ -0,0 +1,51 @@ +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { PostBody } from "./PostBody"; + +const meta = { + title: "Evidence/PostBody", + component: PostBody, +} satisfies Meta; + +export default meta; + +type Story = StoryObj; + +export const MarkdownTableEvidence: Story = { + args: { + body: "| Workstream | State |\n| --- | --- |\n| Alpha | Ready |", + structureUnits: [ + { + unit_index: 0, + unit_kind_code: "plain_text", + unit_label: "markdown_tr", + unit_text: "Workstream | State", + indent_level: 0, + indent_source_code: "unresolved", + indent_confidence: 0, + indent_evidence: "Markdown table row", + }, + { + unit_index: 1, + unit_kind_code: "plain_text", + unit_label: "markdown_tr", + unit_text: "Alpha | Ready", + indent_level: 0, + indent_source_code: "unresolved", + indent_confidence: 0, + indent_evidence: "Markdown table row", + }, + ], + }, +}; + +export const MarkdownTableFallback: Story = { + args: { + body: "Intro.\n\n| Workstream | State |\n| --- | --- |\n| Alpha | Ready |\n\nNext action.", + }, +}; + +export const NumericFootnote: Story = { + args: { + body: "

Evidence remains attached to the source.

1 Source note.

", + }, +}; diff --git a/frontend/src/PostBody.test.tsx b/frontend/src/PostBody.test.tsx index 63a4e3e81..cb5cfd064 100644 --- a/frontend/src/PostBody.test.tsx +++ b/frontend/src/PostBody.test.tsx @@ -120,6 +120,50 @@ describe("PostBody", () => { expect(screen.queryAllByText("No.")).toHaveLength(1); }); + it("renders persisted Markdown table rows as a table", () => { + render( + , + ); + + expect(screen.getByRole("table")).toBeInTheDocument(); + expect(screen.getAllByRole("row")).toHaveLength(2); + }); + + it("renders a raw Markdown table when persisted structure is unavailable", () => { + render( + , + ); + + expect(screen.getByRole("table")).toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: "Project" })).toBeInTheDocument(); + expect(screen.getByText("Intro.")).toBeInTheDocument(); + expect(screen.getByText("Next action.")).toBeInTheDocument(); + }); + it("marks persisted footnotes as footnote evidence", () => { render( { + if (block.kind === "prose") { + return

{block.text}

; + } + const [header, ...rows] = block.rows; + return ( + + + + {header.map((cell, cellIndex) => ( + + ))} + + + + {rows.map((row, rowIndex) => ( + + {row.map((cell, cellIndex) => ( + + ))} + + ))} + +
+ {cell} +
+ {cell} +
+ ); + }); +} + export function PostBody({ body, imageContent = [], @@ -146,6 +184,10 @@ export function PostBody({ if (hasPersistedStructuralUnits) { return
{renderStructuredUnits(body, structureUnits, imageContent)}
; } + const markdownBlocks = !hasPersistedStructuralUnits ? splitMarkdownTableBody(body) : null; + if (markdownBlocks) { + return
{renderMarkdownBlocks(markdownBlocks)}
; + } return (
{splitPostBody(body).map((segment, index) => { diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts index 07ca9514d..2b82ab068 100644 --- a/frontend/src/postBodyDisplay.test.ts +++ b/frontend/src/postBodyDisplay.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { splitPostBody } from "./postBodyDisplay"; +import { splitMarkdownTableBody, splitPostBody } from "./postBodyDisplay"; /** 1x1 transparent PNG — the same synthetic fixture the Python vision tests use. */ const TINY_PNG_B64 = @@ -61,6 +61,29 @@ describe("splitPostBody", () => { ]); }); + it("recognizes numeric superscript-style footnotes", () => { + expect(splitPostBody("

1 Source note

")).toEqual([ + { kind: "text", text: "1 Source note", role: "footnote" }, + ]); + }); + + it("keeps exporter list containers in the visible nesting hierarchy", () => { + expect(splitPostBody("
  • Parent
    • Child
    • ")).toEqual([ + { kind: "text", text: "Parent", indentLevel: 1 }, + { kind: "text", text: "Child", indentLevel: 2 }, + ]); + }); + + it("preserves prose around the supported Markdown table shape", () => { + expect( + splitMarkdownTableBody("Intro.\n\n| Project | Status |\n| --- | --- |\n| Alpha | Ready |\n\nNext action."), + ).toEqual([ + { kind: "prose", text: "Intro." }, + { kind: "table", rows: [["Project", "Status"], ["Alpha", "Ready"]] }, + { kind: "prose", text: "Next action." }, + ]); + }); + it("leaves a plain-text post unchanged so existing popups keep their wording", () => { expect(splitPostBody("The full body text.")).toEqual([ { kind: "text", text: "The full body text." }, diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts index e84af2fb0..e7d57c101 100644 --- a/frontend/src/postBodyDisplay.ts +++ b/frontend/src/postBodyDisplay.ts @@ -11,16 +11,20 @@ export type PostBodySegment = | { kind: "text"; text: string; indentLevel?: number; role?: "footnote" } | { kind: "image"; src: string; mimeType: string; position: number }; +export type MarkdownBodyBlock = + | { kind: "prose"; text: string } + | { kind: "table"; rows: string[][] }; + const DATA_URI_IMG = /]*\bsrc\s*=\s*["']data:(image\/[a-zA-Z0-9.+-]+);base64,([A-Za-z0-9+/=\s]+)["'][^>]*>/gi; const HTML_TAG = /<\/?[a-zA-Z][^>]*>/g; const BREAK_TAG = /]*>/gi; const BLOCK_TAG = - /<\/?(?:article|blockquote|div|h[1-6]|li|ol|p|section|table|tbody|td|tfoot|th|thead|tr|ul|w:p|w:tbl|w:tr|w:tc)\b[^>]*>/gi; + /<\/?(?:article|blockquote|div|h[1-6]|li|oi|ol|p|section|table|tbody|td|tfoot|th|thead|tr|ul|w:p|w:tbl|w:tr|w:tc)\b[^>]*>/gi; const WORD_INDENT_TAG = /]*\/?\s*>/gi; const LIST_ITEM_START = /^\s*(?:[-*•·]\s+|[*†‡](?=\S)|(?:\d{1,3}|[A-Za-z가-힣])[.)]\s+|[①-⑳]\s+)/; -const FOOTNOTE_START = /^\s*[*†‡](?=\S)/; +const FOOTNOTE_START = /^\s*[*†‡]+(?=\S)/; const INDENT_MARKER = "\u0001lw-indent:"; const INDENT_MARKER_END = "\u0002"; const INDENT_MARKER_PATTERN = /lw-indent:(\d+)/g; @@ -65,7 +69,7 @@ function lengthToIndentUnits(value: string): number { function declaredIndentWidth(tag: string): number { const name = tag.match(/^<\/?\s*([a-z0-9:]+)/i)?.[1]?.toLowerCase() ?? ""; - let width = name === "blockquote" || name === "ul" || name === "ol" ? 4 : 0; + let width = name === "blockquote" || name === "ul" || name === "ol" || name === "oi" ? 4 : 0; const style = tag.match(/\bstyle\s*=\s*(["'])(.*?)\1/i)?.[2] ?? ""; for (const match of style.matchAll( /(?:^|;)\s*(?:margin-left|padding-left|padding-inline-start|text-indent)\s*:\s*([^;]+)/gi, @@ -90,16 +94,30 @@ function indentMarker(width: number): string { } function stripHtmlTags(text: string): string { + let listDepth = 0; const withBoundaries = text .replace(BREAK_TAG, "\n") .replace(BLOCK_TAG, (tag) => { - if (/^<\//.test(tag)) return "\n\n"; + const closing = /^<\//.test(tag); + const listContainer = /^<\s*(?:ul|ol|oi)\b/i.test(tag); + if (closing) { + if (listContainer) listDepth = Math.max(0, listDepth - 1); + return "\n\n"; + } + if (listContainer) { + listDepth += 1; + return "\n\n"; + } + if (/^<\s*li\b/i.test(tag)) { + return indentMarker(Math.max(listDepth * 4, declaredIndentWidth(tag))); + } return `\n\n${indentMarker(declaredIndentWidth(tag))}`; }) .replace(WORD_INDENT_TAG, (tag) => indentMarker(declaredIndentWidth(tag))); - const withoutTags = withBoundaries.replace(HTML_TAG, (tag) => - /^<\/?w:/i.test(tag) ? "" : " ", - ); + const withoutTags = withBoundaries.replace(HTML_TAG, (tag) => { + if (/^<\/?w:/i.test(tag) || /^<\/?sup\b/i.test(tag)) return ""; + return " "; + }); const decoded = decodeHtmlEntities(withoutTags); return decoded .split("\n") @@ -197,6 +215,7 @@ function isDecodableBase64(raw: string): boolean { } function pushText(segments: PostBodySegment[], raw: string, indentUnit: number): void { + const hasNumericSuperscriptMarker = /]*>\s*\d{1,3}\s*<\/sup>/i.test(raw); const text = stripHtmlTags(raw); for (const paragraph of splitSemanticParagraphs(text)) { const indentLevel = indentationLevel(paragraph, indentUnit); @@ -208,12 +227,68 @@ function pushText(segments: PostBodySegment[], raw: string, indentUnit: number): kind: "text", text: normalized, ...(indentLevel > 0 ? { indentLevel } : {}), - ...(FOOTNOTE_START.test(normalized) ? { role: "footnote" as const } : {}), + ...(hasNumericSuperscriptMarker || FOOTNOTE_START.test(normalized) + ? { role: "footnote" as const } + : {}), }); } } } +function markdownCells(line: string): string[] | null { + const value = line.trim().replace(/^\|/, "").replace(/(? cell.trim().replace(/\\\|/g, "|")); + return cells.length >= 2 && cells.every(Boolean) ? cells : null; +} + +function isMarkdownSeparatorRow(cells: string[] | null): boolean { + return Boolean(cells?.every((cell) => /^:?-{3,}:?$/.test(cell))); +} + +/** + * Split the narrow Markdown-table shape supported by the ingestion boundary. + * The return value preserves prose and skips only the delimiter row. + */ +export function splitMarkdownTableBody(body: string): MarkdownBodyBlock[] | null { + const lines = body.replace(/\r\n?/g, "\n").split("\n"); + const blocks: MarkdownBodyBlock[] = []; + let prose: string[] = []; + let foundTable = false; + + const flushProse = () => { + const text = prose.join("\n").trim(); + if (text) blocks.push({ kind: "prose", text }); + prose = []; + }; + + let index = 0; + while (index < lines.length) { + const header = markdownCells(lines[index]); + const separator = markdownCells(lines[index + 1] ?? ""); + if (!header || !isMarkdownSeparatorRow(separator)) { + prose.push(lines[index]); + index += 1; + continue; + } + + foundTable = true; + flushProse(); + const rows = [header]; + index += 2; + while (index < lines.length && lines[index].trim()) { + const row = markdownCells(lines[index]); + if (!row) break; + rows.push(row); + index += 1; + } + blocks.push({ kind: "table", rows }); + } + + flushProse(); + return foundTable ? blocks : null; +} + export function splitPostBody(body: string): PostBodySegment[] { const segments: PostBodySegment[] = []; const indentUnit = inferIndentationUnit(stripHtmlTags(body)); diff --git a/lineageweave/chunking.py b/lineageweave/chunking.py index a0c62c1b5..3adb84690 100644 --- a/lineageweave/chunking.py +++ b/lineageweave/chunking.py @@ -66,6 +66,9 @@ "div", "p", "li", + "ul", + "ol", + "oi", "tr", "blockquote", "h1", @@ -91,11 +94,12 @@ # readable and attributable as one unit. _TABLE_ROW_TAGS = frozenset({"tr", "w:tr"}) _TABLE_CELL_TAGS = frozenset({"td", "th", "w:tc"}) +_LIST_CONTAINER_TAGS = frozenset({"ul", "ol", "oi"}) _LIST_ITEM_START = re.compile( r"^(?:[-*•·]\s+|[*†‡](?=\S)|(?:\d{1,3}|[A-Za-z가-힣])[.)]\s+|[①-⑳]\s+)" ) -_FOOTNOTE_START = re.compile(r"^[*†‡](?=\S)") +_FOOTNOTE_START = re.compile(r"^[*†‡]+(?=\S)") def normalize_semantic_text(text: str) -> str: @@ -163,7 +167,7 @@ def _shorthand_left_value(raw: str) -> str: def _declared_indent_width(tag: str, attrs: list[tuple[str, str | None]]) -> int: """Read HTML CSS and WordprocessingML paragraph indentation declarations.""" - width = 4 if tag in {"blockquote", "ul", "ol"} else 0 + width = 4 if tag in {"blockquote", "ul", "ol", "oi"} else 0 style = next((value or "" for name, value in attrs if name == "style"), "") for match in re.finditer( r"(?:^|;)\s*(?:margin-left|padding-left|padding-inline-start|text-indent)\s*:\s*([^;]+)", @@ -301,12 +305,22 @@ def __init__(self) -> None: super().__init__() self._stack: list[tuple[str, list[str], str | None, int]] = [] self._unscoped_buffer: list[str] = [] + self._superscript_buffers: set[int] = set() # Each entry is ("text", str, tag_name, style) or # ("image", (mime_type, bytes), "", None) -- a single sequence in # true document order, so an image's index among its siblings # reflects where it actually sat. self._finished: list[tuple[str, object, str, str | None, int]] = [] + def _declared_stack_width(self) -> int: + """Combine list depth with explicit width without double counting.""" + list_depth = sum(entry[0] in _LIST_CONTAINER_TAGS for entry in self._stack) + explicit_width = sum( + max(0, entry[3] - 4) if entry[0] in _LIST_CONTAINER_TAGS else entry[3] + for entry in self._stack + ) + return max(explicit_width, list_depth * 4) + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: """Collect relevant text state when an HTML start tag is encountered.""" if tag == "img": @@ -316,6 +330,10 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None if decoded is not None: self._finished.append(("image", decoded, "", None, 0)) return + if tag == "sup": + if self._stack and not "".join(self._stack[-1][1]).strip(): + self._superscript_buffers.add(id(self._stack[-1][1])) + return if tag in {"br", "w:br"} and self._stack: self._stack[-1][1].append("\n") return @@ -328,6 +346,23 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None indent_width + _declared_indent_width(tag, attrs), ) return + if tag in _LIST_CONTAINER_TAGS: + # Emit a parent list item before entering its nested list. Closing + # tags otherwise make the child appear before the parent in the + # finished list, which destroys the source order buyers use to + # read a hierarchy. + if self._stack and self._stack[-1][0] == "li" and self._stack[-1][1]: + tag_name, buffer, style, indent_width = self._stack[-1] + self._stack[-1] = (tag_name, [], style, indent_width) + self._finish_block( + tag_name, + buffer, + style, + self._declared_stack_width(), + ) + style = next((value for name, value in attrs if name == "style" and value), None) + self._stack.append((tag, [], style, _declared_indent_width(tag, attrs))) + return if tag in _TABLE_CELL_TAGS: if self._stack and self._stack[-1][0] in _TABLE_ROW_TAGS and self._stack[-1][1]: self._stack[-1][1].append(" | ") @@ -350,7 +385,7 @@ def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> N def handle_endtag(self, tag: str) -> None: """Close the relevant text state when an HTML end tag is encountered.""" if tag in _DOM_BLOCK_TAGS and self._stack and self._stack[-1][0] == tag: - declared_width = sum(entry[3] for entry in self._stack) + declared_width = self._declared_stack_width() tag_name, buffer, style, _ = self._stack.pop() self._finish_block(tag_name, buffer, style, declared_width) @@ -359,14 +394,20 @@ def _finish_block( ) -> None: """Emit one block buffer, including a block closed only at EOF.""" raw_text = "".join(buffer) + superscript_marker = id(buffer) in self._superscript_buffers for raw_unit, source_indent in _split_dom_units(raw_text): text = normalize_semantic_text(raw_unit) if text: indent_width = declared_width + source_indent - label = "footnote" if _FOOTNOTE_START.match(text) else tag_name + label = ( + "footnote" + if superscript_marker or _FOOTNOTE_START.match(text) + else tag_name + ) self._finished.append( ("text", text, label, style, indent_width) ) + self._superscript_buffers.discard(id(buffer)) def handle_data(self, data: str) -> None: """Collect character data from the current HTML text region.""" @@ -386,7 +427,7 @@ def handle_data(self, data: str) -> None: def finished(self) -> list[tuple[str, object, str, str | None, int]]: """Return the normalized records collected from the HTML fragment.""" while self._stack: - declared_width = sum(entry[3] for entry in self._stack) + declared_width = self._declared_stack_width() tag_name, buffer, style, _ = self._stack.pop() self._finish_block(tag_name, buffer, style, declared_width) if not self._finished: @@ -419,6 +460,62 @@ def flush() -> None: return units +_MARKDOWN_SEPARATOR_CELL = re.compile(r"^:?-{3,}:?$") + + +def _markdown_cells(line: str) -> list[str] | None: + """Return Markdown table cells, or ``None`` for a non-table line.""" + if "|" not in line: + return None + value = line.strip() + if value.startswith("|"): + value = value[1:] + if value.endswith("|") and not value.endswith("\\|"): + value = value[:-1] + cells = [cell.strip().replace(r"\|", "|") for cell in re.split(r"(?= 2 and all(cells) else None + + +def _markdown_table_entries(text: str) -> list[tuple[str, str]]: + """Extract table rows while retaining non-table prose around the table.""" + lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n") + entries: list[tuple[str, str]] = [] + pending: list[str] = [] + found_table = False + + def flush_pending() -> None: + if pending: + value = normalize_semantic_text("\n".join(pending)) + if value: + entries.append(("", value)) + pending.clear() + + index = 0 + while index < len(lines): + header = _markdown_cells(lines[index]) + separator = _markdown_cells(lines[index + 1]) if index + 1 < len(lines) else None + if header is None or separator is None or not all( + _MARKDOWN_SEPARATOR_CELL.fullmatch(cell) for cell in separator + ): + pending.append(lines[index]) + index += 1 + continue + + found_table = True + flush_pending() + entries.append(("markdown_tr", " | ".join(header))) + index += 2 + while index < len(lines) and lines[index].strip(): + cells = _markdown_cells(lines[index]) + if cells is None: + break + entries.append(("markdown_tr", " | ".join(cells))) + index += 1 + + flush_pending() + return entries if found_table else [] + + def chunk_by_dom(html: str) -> list[Chunk]: """Split HTML/MHTML content at sectioning/flow block-element boundaries, plus one ``"image"`` chunk per embedded base64 ````, all in a @@ -433,6 +530,19 @@ def chunk_by_dom(html: str) -> list[Chunk]: is what lets the image be placed back where it actually was relative to the surrounding text chunks. """ + if "<" not in html: + markdown_entries = _markdown_table_entries(html) + if markdown_entries: + return [ + Chunk( + text=text, + unit_type="plain_text", + index=index, + label=label, + ) + for index, (label, text) in enumerate(markdown_entries) + ] + parser = _BlockTextExtractor() parser.feed(html) entries = parser.finished() diff --git a/lineageweave/post_content_normalization.py b/lineageweave/post_content_normalization.py index ac2b4b89b..a723ee5ab 100644 --- a/lineageweave/post_content_normalization.py +++ b/lineageweave/post_content_normalization.py @@ -42,8 +42,8 @@ # what chunk_by_dom already splits on, plus the inline/replaced tags # that carry images or wrap rich-text fragments. _HTML_OPEN_TAG = re.compile( - r"<\s*/?\s*(?:article|section|nav|aside|header|footer|div|p|li|td|th|tr|" - r"table|blockquote|h[1-6]|img|br|hr|ul|ol|span|strong|em|b|i|u|a|" + r"<\s*/?\s*(?:article|section|nav|aside|header|footer|div|p|li|td|th|tr|sup|" + r"table|blockquote|h[1-6]|img|br|hr|ul|ol|oi|span|strong|em|b|i|u|a|" r"html|body|head|style|script|font|center|pre)\b", re.IGNORECASE, ) @@ -253,6 +253,11 @@ def normalize_post_body( vision_client = NullImageContentClient() if not _looks_like_html(body): + markdown_chunks = chunk_by_dom(body) + if any(chunk.label == "markdown_tr" for chunk in markdown_chunks): + return NormalizedPostContent( + text="\n\n".join(chunk.text for chunk in markdown_chunks if chunk.text) + ) return NormalizedPostContent(text=normalize_semantic_text(body)) chunks: list[Chunk] = chunk_by_dom(body) diff --git a/tests/test_chunking.py b/tests/test_chunking.py index f76b11e5a..a851c0c6a 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -111,6 +111,54 @@ def test_chunk_by_dom_labels_markerless_footnotes() -> None: ] +def test_chunk_by_dom_labels_numeric_superscript_footnotes() -> None: + chunks = chunk_by_dom("

    1 Source note attached to the record.

    ") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("footnote", "1 Source note attached to the record."), + ] + + +def test_chunk_by_dom_preserves_nested_list_order_and_depth() -> None: + chunks = chunk_by_dom( + "
    1. Parent item
      • Child item
    " + ) + + assert [chunk.text for chunk in chunks] == ["Parent item", "Child item"] + assert [chunk.indent_width for chunk in chunks] == [4, 8] + + +def test_chunk_by_dom_accepts_exporter_oi_list_container() -> None: + chunks = chunk_by_dom("
  • First item
  • Second item
    • ") + + assert [chunk.text for chunk in chunks] == ["First item", "Second item"] + assert [chunk.indent_width for chunk in chunks] == [4, 4] + + +def test_chunk_by_dom_keeps_markdown_table_rows_as_searchable_units() -> None: + chunks = chunk_by_dom( + "| Project | Status |\n| :--- | ---: |\n| Alpha | Ready |" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("markdown_tr", "Project | Status"), + ("markdown_tr", "Alpha | Ready"), + ] + + +def test_chunk_by_dom_keeps_prose_around_markdown_table_rows() -> None: + chunks = chunk_by_dom( + "Intro.\n\n| Project | Status |\n| --- | --- |\n| Alpha | Ready |\n\nNext action." + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("", "Intro."), + ("markdown_tr", "Project | Status"), + ("markdown_tr", "Alpha | Ready"), + ("", "Next action."), + ] + + def test_chunk_by_dom_word_table_rows_also_group_cells() -> None: html = "1Acme Corp" chunks = chunk_by_dom(html) @@ -154,6 +202,7 @@ def test_chunk_by_dom_reads_the_css_margin_shorthand_not_just_margin_left() -> N assert [chunk.text for chunk in chunks] == ["Outer item", "Nested item"] outer, nested = chunks + assert [outer.indent_width, nested.indent_width] == [7, 10] assert outer.indent_width < nested.indent_width assert outer.indent_width > 0 diff --git a/tests/test_person_mention_projection.py b/tests/test_person_mention_projection.py index 87f2dbbf0..ecd9ab945 100644 --- a/tests/test_person_mention_projection.py +++ b/tests/test_person_mention_projection.py @@ -82,6 +82,9 @@ _SOURCE_ORG_NAMED_HINTS_MIGRATION = ( Path(__file__).resolve().parents[1] / "migrations" / "0039_source_org_named_hints.sql" ) +_MAJOR_EVENT_ACTION_MIGRATION = ( + Path(__file__).resolve().parents[1] / "migrations" / "0100_major_event_action.sql" +) def _postgres_available() -> bool: @@ -146,6 +149,7 @@ def projection_database() -> str: cursor.execute(_SOURCE_ORG_NAMED_HINTS_MIGRATION.read_text(encoding="utf-8")) cursor.execute(_POST_SUMMARY_CONTRACT_MIGRATION.read_text(encoding="utf-8")) cursor.execute(_SUMMARY_FIVE_W1H_MIGRATION.read_text(encoding="utf-8")) + cursor.execute(_MAJOR_EVENT_ACTION_MIGRATION.read_text(encoding="utf-8")) cursor.execute( """ insert into common_lookup_value diff --git a/tests/test_post_content_normalization.py b/tests/test_post_content_normalization.py index 2342e6ad4..9500429a6 100644 --- a/tests/test_post_content_normalization.py +++ b/tests/test_post_content_normalization.py @@ -72,6 +72,20 @@ def test_plain_text_visual_continuation_breaks_are_normalized_for_embeddings() - assert result.text == "- 요청 사항 후속 설명은 같은 항목에 속한다.\n· 다음 항목" +def test_markdown_table_rows_remain_separate_in_normalized_evidence() -> None: + result = normalize_post_body( + "| Project | Status |\n| --- | --- |\n| Alpha | Ready |" + ) + + assert result.text == "Project | Status\n\nAlpha | Ready" + + +def test_exporter_oi_lists_are_normalized_as_html_evidence() -> None: + result = normalize_post_body("
  • Parent
    • Child
    • ") + + assert result.text == "Parent\n\nChild" + + def test_html_tags_never_appear_in_the_normalized_text() -> None: html = '

    Confirm delivery by Friday.

    ' result = normalize_post_body(html) From d3bc76f59bcdf62b923efc82001d196fecbb4394 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 17:53:04 +0900 Subject: [PATCH 2/9] fix: classify numeric superscript footnotes after prose --- lineageweave/chunking.py | 19 ++++++++++++++----- tests/test_chunking.py | 16 ++++++++++++++++ 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/lineageweave/chunking.py b/lineageweave/chunking.py index 3adb84690..e2f013a80 100644 --- a/lineageweave/chunking.py +++ b/lineageweave/chunking.py @@ -305,7 +305,8 @@ def __init__(self) -> None: super().__init__() self._stack: list[tuple[str, list[str], str | None, int]] = [] self._unscoped_buffer: list[str] = [] - self._superscript_buffers: set[int] = set() + self._active_superscripts: list[tuple[int, list[str]]] = [] + self._numeric_superscript_buffers: set[int] = set() # Each entry is ("text", str, tag_name, style) or # ("image", (mime_type, bytes), "", None) -- a single sequence in # true document order, so an image's index among its siblings @@ -331,8 +332,8 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None self._finished.append(("image", decoded, "", None, 0)) return if tag == "sup": - if self._stack and not "".join(self._stack[-1][1]).strip(): - self._superscript_buffers.add(id(self._stack[-1][1])) + if self._stack: + self._active_superscripts.append((id(self._stack[-1][1]), [])) return if tag in {"br", "w:br"} and self._stack: self._stack[-1][1].append("\n") @@ -384,6 +385,12 @@ def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> N def handle_endtag(self, tag: str) -> None: """Close the relevant text state when an HTML end tag is encountered.""" + if tag == "sup": + if self._active_superscripts: + buffer_id, content = self._active_superscripts.pop() + if re.fullmatch(r"\s*\d{1,3}\s*", "".join(content)): + self._numeric_superscript_buffers.add(buffer_id) + return if tag in _DOM_BLOCK_TAGS and self._stack and self._stack[-1][0] == tag: declared_width = self._declared_stack_width() tag_name, buffer, style, _ = self._stack.pop() @@ -394,7 +401,7 @@ def _finish_block( ) -> None: """Emit one block buffer, including a block closed only at EOF.""" raw_text = "".join(buffer) - superscript_marker = id(buffer) in self._superscript_buffers + superscript_marker = id(buffer) in self._numeric_superscript_buffers for raw_unit, source_indent in _split_dom_units(raw_text): text = normalize_semantic_text(raw_unit) if text: @@ -407,7 +414,7 @@ def _finish_block( self._finished.append( ("text", text, label, style, indent_width) ) - self._superscript_buffers.discard(id(buffer)) + self._numeric_superscript_buffers.discard(id(buffer)) def handle_data(self, data: str) -> None: """Collect character data from the current HTML text region.""" @@ -417,6 +424,8 @@ def handle_data(self, data: str) -> None: if decoded == text: break text = decoded + if self._active_superscripts: + self._active_superscripts[-1][1].append(text) had_nbsp = "\xa0" in text text = text.replace("\xa0", " ") if self._stack and (text.strip() or had_nbsp): diff --git a/tests/test_chunking.py b/tests/test_chunking.py index a851c0c6a..a8a6075a5 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -119,6 +119,22 @@ def test_chunk_by_dom_labels_numeric_superscript_footnotes() -> None: ] +def test_chunk_by_dom_labels_numeric_superscript_after_body_text() -> None: + chunks = chunk_by_dom("

    Body claim1 source note.

    ") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("footnote", "Body claim1 source note."), + ] + + +def test_chunk_by_dom_does_not_treat_non_numeric_superscript_as_footnote() -> None: + chunks = chunk_by_dom("

    Formula xn remains prose.

    ") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("p", "Formula xn remains prose."), + ] + + def test_chunk_by_dom_preserves_nested_list_order_and_depth() -> None: chunks = chunk_by_dom( "
    1. Parent item
      • Child item
    " From 790d6442701a0da1feaeb7b094b8259be957df74 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 07:14:51 +0900 Subject: [PATCH 3/9] fix: scope numeric footnotes by paragraph --- frontend/src/postBodyDisplay.test.ts | 14 ++++++++++++++ frontend/src/postBodyDisplay.ts | 10 +++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts index 2b82ab068..94a789eab 100644 --- a/frontend/src/postBodyDisplay.test.ts +++ b/frontend/src/postBodyDisplay.test.ts @@ -67,6 +67,20 @@ describe("splitPostBody", () => { ]); }); + it("limits numeric superscript footnote roles to their source paragraph", () => { + expect( + splitPostBody( + "

    Evidence remains attached to the source.

    " + + "

    1 Source note.

    " + + "

    Continue with the next source action.

    ", + ), + ).toEqual([ + { kind: "text", text: "Evidence remains attached to the source." }, + { kind: "text", text: "1 Source note.", role: "footnote" }, + { kind: "text", text: "Continue with the next source action." }, + ]); + }); + it("keeps exporter list containers in the visible nesting hierarchy", () => { expect(splitPostBody("
  • Parent
    • Child
    • ")).toEqual([ { kind: "text", text: "Parent", indentLevel: 1 }, diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts index e7d57c101..6e2bc8a14 100644 --- a/frontend/src/postBodyDisplay.ts +++ b/frontend/src/postBodyDisplay.ts @@ -28,6 +28,8 @@ const FOOTNOTE_START = /^\s*[*†‡]+(?=\S)/; const INDENT_MARKER = "\u0001lw-indent:"; const INDENT_MARKER_END = "\u0002"; const INDENT_MARKER_PATTERN = /lw-indent:(\d+)/g; +const NUMERIC_FOOTNOTE_MARKER = "\u0003lw-numeric-footnote\u0004"; +const NUMERIC_SUPERSCRIPT = /]*>\s*(\d{1,3})\s*<\/sup>/gi; function stripIndentMarkers(value: string): string { return value @@ -215,11 +217,13 @@ function isDecodableBase64(raw: string): boolean { } function pushText(segments: PostBodySegment[], raw: string, indentUnit: number): void { - const hasNumericSuperscriptMarker = /]*>\s*\d{1,3}\s*<\/sup>/i.test(raw); - const text = stripHtmlTags(raw); + const text = stripHtmlTags( + raw.replace(NUMERIC_SUPERSCRIPT, `${NUMERIC_FOOTNOTE_MARKER}$1`), + ); for (const paragraph of splitSemanticParagraphs(text)) { + const hasNumericSuperscriptMarker = paragraph.includes(NUMERIC_FOOTNOTE_MARKER); const indentLevel = indentationLevel(paragraph, indentUnit); - const normalized = stripIndentMarkers(paragraph) + const normalized = stripIndentMarkers(paragraph.replaceAll(NUMERIC_FOOTNOTE_MARKER, "")) .replace(/^[ \t]+/, "") .replace(/[ \t]+$/gm, ""); if (normalized.trim()) { From a111d8c1c7bc411fc94ffc002cd48f0655bdbb0d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 07:31:07 +0900 Subject: [PATCH 4/9] fix: preserve independent list boundaries --- frontend/src/postBodyDisplay.test.ts | 14 ++++++++++++++ frontend/src/postBodyDisplay.ts | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts index 94a789eab..6e76f52b4 100644 --- a/frontend/src/postBodyDisplay.test.ts +++ b/frontend/src/postBodyDisplay.test.ts @@ -88,6 +88,20 @@ describe("splitPostBody", () => { ]); }); + it("resets indentation between separate top-level lists", () => { + expect(splitPostBody("
    • First
    • Second
    ")).toEqual([ + { kind: "text", text: "First", indentLevel: 1 }, + { kind: "text", text: "Second", indentLevel: 1 }, + ]); + }); + + it("keeps list items separate when optional closing tags are omitted", () => { + expect(splitPostBody("
    • First
    • Second
    ")).toEqual([ + { kind: "text", text: "First", indentLevel: 1 }, + { kind: "text", text: "Second", indentLevel: 1 }, + ]); + }); + it("preserves prose around the supported Markdown table shape", () => { expect( splitMarkdownTableBody("Intro.\n\n| Project | Status |\n| --- | --- |\n| Alpha | Ready |\n\nNext action."), diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts index 6e2bc8a14..63f9f6a12 100644 --- a/frontend/src/postBodyDisplay.ts +++ b/frontend/src/postBodyDisplay.ts @@ -101,7 +101,7 @@ function stripHtmlTags(text: string): string { .replace(BREAK_TAG, "\n") .replace(BLOCK_TAG, (tag) => { const closing = /^<\//.test(tag); - const listContainer = /^<\s*(?:ul|ol|oi)\b/i.test(tag); + const listContainer = /^<\s*\/?\s*(?:ul|ol|oi)\b/i.test(tag); if (closing) { if (listContainer) listDepth = Math.max(0, listDepth - 1); return "\n\n"; @@ -111,7 +111,7 @@ function stripHtmlTags(text: string): string { return "\n\n"; } if (/^<\s*li\b/i.test(tag)) { - return indentMarker(Math.max(listDepth * 4, declaredIndentWidth(tag))); + return `\n\n${indentMarker(Math.max(listDepth * 4, declaredIndentWidth(tag)))}`; } return `\n\n${indentMarker(declaredIndentWidth(tag))}`; }) From b5f86b9d0ea2224258c585f9d45087a600e122a3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 07:41:56 +0900 Subject: [PATCH 5/9] fix: keep table-cell lists in row chunks --- lineageweave/chunking.py | 19 ++++++++++--------- tests/test_chunking.py | 12 ++++++++++++ 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/lineageweave/chunking.py b/lineageweave/chunking.py index fad0922a3..3e38cc93b 100644 --- a/lineageweave/chunking.py +++ b/lineageweave/chunking.py @@ -347,6 +347,16 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None indent_width + _declared_indent_width(tag, attrs), ) return + if tag in _TABLE_CELL_TAGS: + if self._stack and self._stack[-1][0] in _TABLE_ROW_TAGS and self._stack[-1][1]: + self._stack[-1][1].append(" | ") + return + # Nested blocks belong to the open table row. Keep list-item text + # readable without letting a nested list become a separate chunk. + if any(entry[0] in _TABLE_ROW_TAGS for entry in self._stack): + if tag == "li" and self._stack[-1][1]: + self._stack[-1][1].append(" ") + return if tag in _LIST_CONTAINER_TAGS: # Emit a parent list item before entering its nested list. Closing # tags otherwise make the child appear before the parent in the @@ -364,15 +374,6 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None style = next((value for name, value in attrs if name == "style" and value), None) self._stack.append((tag, [], style, _declared_indent_width(tag, attrs))) return - if tag in _TABLE_CELL_TAGS: - if self._stack and self._stack[-1][0] in _TABLE_ROW_TAGS and self._stack[-1][1]: - self._stack[-1][1].append(" | ") - return - # A rich-text editor commonly wraps a table cell in a nested

    or - #

    . Keep that content in the open row; otherwise the nested block - # closes first and destroys the row/column boundary. - if any(entry[0] in _TABLE_ROW_TAGS for entry in self._stack): - return if tag in _DOM_BLOCK_TAGS: if self._stack and self._stack[-1][1]: tag_name, buffer, style, _ = self._stack[-1] diff --git a/tests/test_chunking.py b/tests/test_chunking.py index 07a0e2917..6cdbf656f 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -104,6 +104,18 @@ def test_chunk_by_dom_keeps_nested_table_cell_blocks_in_their_row() -> None: assert [(chunk.label, chunk.text) for chunk in chunks] == [("tr", "No. | Company")] +def test_chunk_by_dom_keeps_cell_lists_inside_their_table_row() -> None: + """A list inside a cell stays readable and grouped with its row.""" + chunks = chunk_by_dom( + "" + "
    Items:
    • A
    • B
    		Owner
    " + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("tr", "Items: A B | Owner") + ] + + def test_chunk_by_dom_labels_markerless_footnotes() -> None: chunks = chunk_by_dom("

    Body text

    *Tier 2: follow-up note

    ") assert [(chunk.label, chunk.text) for chunk in chunks] == [ From 8ed7eac5dc0e827cfdcfb7143eaa6db7c9c92189 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 07:50:06 +0900 Subject: [PATCH 6/9] test: close semantic chunking coverage gaps --- lineageweave/chunking.py | 27 ++++---- tests/test_chunking.py | 132 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 145 insertions(+), 14 deletions(-) diff --git a/lineageweave/chunking.py b/lineageweave/chunking.py index 3e38cc93b..2bc024d49 100644 --- a/lineageweave/chunking.py +++ b/lineageweave/chunking.py @@ -302,6 +302,7 @@ class _BlockTextExtractor(HTMLParser): """ def __init__(self) -> None: + """Initialize parser buffers for ordered text, image, and footnote units.""" super().__init__() self._stack: list[tuple[str, list[str], str | None, int]] = [] self._unscoped_buffer: list[str] = [] @@ -410,16 +411,13 @@ def _finish_block( superscript_marker = id(buffer) in self._numeric_superscript_buffers for raw_unit, source_indent in _split_dom_units(raw_text): text = normalize_semantic_text(raw_unit) - if text: - indent_width = declared_width + source_indent - label = ( - "footnote" - if superscript_marker or _FOOTNOTE_START.match(text) - else tag_name - ) - self._finished.append( - ("text", text, label, style, indent_width) - ) + indent_width = declared_width + source_indent + label = ( + "footnote" + if superscript_marker or _FOOTNOTE_START.match(text) + else tag_name + ) + self._finished.append(("text", text, label, style, indent_width)) self._numeric_superscript_buffers.discard(id(buffer)) def handle_data(self, data: str) -> None: @@ -458,10 +456,10 @@ def _split_dom_units(raw_text: str) -> list[tuple[str, int]]: current: list[str] = [] def flush() -> None: + """Move the current non-empty DOM unit into the result list.""" if current: raw_unit = "\n".join(current) - if raw_unit.strip(): - units.append((raw_unit, _source_indent_width(raw_unit))) + units.append((raw_unit, _source_indent_width(raw_unit))) current.clear() for line in raw_text.replace("\r\n", "\n").replace("\r", "\n").split("\n"): @@ -499,6 +497,7 @@ def _markdown_table_entries(text: str) -> list[tuple[str, str]]: found_table = False def flush_pending() -> None: + """Emit prose accumulated outside a recognized Markdown table.""" if pending: value = normalize_semantic_text("\n".join(pending)) if value: @@ -554,11 +553,11 @@ def _split_plain_text_units(text: str) -> list[tuple[str, int, str]]: current: list[str] = [] def flush() -> None: + """Emit the current authored plain-text semantic unit.""" if current: raw_unit = "\n".join(current) normalized = normalize_semantic_text(raw_unit) - if normalized: - units.append((normalized, _source_indent_width(raw_unit), "")) + units.append((normalized, _source_indent_width(raw_unit), "")) current.clear() index = 0 diff --git a/tests/test_chunking.py b/tests/test_chunking.py index 6cdbf656f..39e7517a0 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -2,6 +2,8 @@ from lineageweave.chunking import ( ConversationTurn, + _length_to_indent_units, + _shorthand_left_value, chunk_by_conversation_turn, chunk_by_dom, chunk_by_source_body, @@ -12,6 +14,7 @@ def test_chunk_by_paragraph_splits_on_blank_lines() -> None: + """Blank lines delimit ordered paragraph chunks.""" text = "First paragraph about budgets.\n\nSecond paragraph about logistics.\n\nThird." chunks = chunk_by_paragraph(text) @@ -25,17 +28,20 @@ def test_chunk_by_paragraph_splits_on_blank_lines() -> None: def test_chunk_by_paragraph_ignores_extra_blank_lines_and_whitespace() -> None: + """Extra blank lines and outer whitespace do not create chunks.""" text = " A. \n\n\n\n B. " chunks = chunk_by_paragraph(text) assert [c.text for c in chunks] == ["A.", "B."] def test_chunk_by_paragraph_empty_text_yields_no_chunks() -> None: + """Empty paragraph input produces no semantic units.""" assert chunk_by_paragraph("") == [] assert chunk_by_paragraph(" \n\n ") == [] def test_chunk_by_sentence_splits_on_sentence_boundaries() -> None: + """Sentence punctuation followed by a new sentence creates a boundary.""" text = "This is one sentence. This is another! Is this a third?" chunks = chunk_by_sentence(text) @@ -47,7 +53,22 @@ def test_chunk_by_sentence_splits_on_sentence_boundaries() -> None: assert all(c.unit_type == "sentence" for c in chunks) +def test_chunk_by_sentence_empty_text_yields_no_chunks() -> None: + """Whitespace alone has no sentence-level semantic unit.""" + assert chunk_by_sentence(" \n ") == [] + + +def test_indent_helpers_cover_invalid_lengths_and_css_shorthand_shapes() -> None: + """Invalid and non-positive lengths stay flat; shorthand picks the left side.""" + assert _length_to_indent_units("auto") == 0 + assert _length_to_indent_units("-8px") == 0 + assert _shorthand_left_value("") == "" + assert _shorthand_left_value("8px") == "8px" + assert _shorthand_left_value("8px 16px") == "16px" + + def test_chunk_by_dom_splits_on_block_element_boundaries() -> None: + """Sibling DOM blocks remain separate semantic units.""" html = ( "

    First block of text.

    Second block of text.

    " "" @@ -62,6 +83,7 @@ def test_chunk_by_dom_splits_on_block_element_boundaries() -> None: def test_chunk_by_dom_nested_blocks_do_not_duplicate_text() -> None: + """The innermost block owns text without duplicating its ancestor.""" html = "

    Nested paragraph text.

    " chunks = chunk_by_dom(html) @@ -72,6 +94,16 @@ def test_chunk_by_dom_nested_blocks_do_not_duplicate_text() -> None: assert chunks[0].label == "p" +def test_chunk_by_dom_flushes_parent_text_before_nested_block() -> None: + """Direct parent text remains before a later child block.""" + chunks = chunk_by_dom("
    Parent text

    Child text

    ") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("div", "Parent text"), + ("p", "Child text"), + ] + + def test_chunk_by_dom_groups_table_cells_by_row_instead_of_flattening() -> None: """Live bug (2026-08-19): each used to push its own independent chunk with no row grouping, so a real table (headers + N data rows) @@ -98,6 +130,7 @@ def test_chunk_by_dom_groups_table_cells_by_row_instead_of_flattening() -> None: def test_chunk_by_dom_keeps_nested_table_cell_blocks_in_their_row() -> None: + """Nested cell blocks retain their table-row grouping.""" chunks = chunk_by_dom( "

    No.

    Company
    " ) @@ -115,8 +148,15 @@ def test_chunk_by_dom_keeps_cell_lists_inside_their_table_row() -> None: ("tr", "Items: A B | Owner") ] + leading_list = chunk_by_dom( + "" + "
    • A
    • B
    		Owner
    " + ) + assert [chunk.text for chunk in leading_list] == ["A B | Owner"] + def test_chunk_by_dom_labels_markerless_footnotes() -> None: + """A leading footnote marker assigns the footnote label.""" chunks = chunk_by_dom("

    Body text

    *Tier 2: follow-up note

    ") assert [(chunk.label, chunk.text) for chunk in chunks] == [ ("p", "Body text"), @@ -125,6 +165,7 @@ def test_chunk_by_dom_labels_markerless_footnotes() -> None: def test_chunk_by_dom_labels_numeric_superscript_footnotes() -> None: + """A leading numeric superscript assigns the footnote label.""" chunks = chunk_by_dom("

    1 Source note attached to the record.

    ") assert [(chunk.label, chunk.text) for chunk in chunks] == [ @@ -133,6 +174,7 @@ def test_chunk_by_dom_labels_numeric_superscript_footnotes() -> None: def test_chunk_by_dom_labels_numeric_superscript_after_body_text() -> None: + """A numeric superscript anywhere in a paragraph marks its evidence role.""" chunks = chunk_by_dom("

    Body claim1 source note.

    ") assert [(chunk.label, chunk.text) for chunk in chunks] == [ @@ -141,6 +183,7 @@ def test_chunk_by_dom_labels_numeric_superscript_after_body_text() -> None: def test_chunk_by_dom_does_not_treat_non_numeric_superscript_as_footnote() -> None: + """A formula superscript remains ordinary prose.""" chunks = chunk_by_dom("

    Formula xn remains prose.

    ") assert [(chunk.label, chunk.text) for chunk in chunks] == [ @@ -149,6 +192,7 @@ def test_chunk_by_dom_does_not_treat_non_numeric_superscript_as_footnote() -> No def test_chunk_by_dom_preserves_nested_list_order_and_depth() -> None: + """Nested list items retain source order and increasing depth.""" chunks = chunk_by_dom( "
    1. Parent item
      • Child item
    " ) @@ -158,6 +202,7 @@ def test_chunk_by_dom_preserves_nested_list_order_and_depth() -> None: def test_chunk_by_dom_accepts_exporter_oi_list_container() -> None: + """The exporter-specific oi tag behaves as an ordered-list container.""" chunks = chunk_by_dom("
  • First item
  • Second item
    • ") assert [chunk.text for chunk in chunks] == ["First item", "Second item"] @@ -165,6 +210,7 @@ def test_chunk_by_dom_accepts_exporter_oi_list_container() -> None: def test_chunk_by_dom_keeps_markdown_table_rows_as_searchable_units() -> None: + """Markdown rows become independently searchable row units.""" chunks = chunk_by_dom( "| Project | Status |\n| :--- | ---: |\n| Alpha | Ready |" ) @@ -176,6 +222,7 @@ def test_chunk_by_dom_keeps_markdown_table_rows_as_searchable_units() -> None: def test_chunk_by_dom_keeps_prose_around_markdown_table_rows() -> None: + """Prose surrounding a Markdown table stays in document order.""" chunks = chunk_by_dom( "Intro.\n\n| Project | Status |\n| --- | --- |\n| Alpha | Ready |\n\nNext action." ) @@ -188,7 +235,31 @@ def test_chunk_by_dom_keeps_prose_around_markdown_table_rows() -> None: ] +def test_chunk_by_dom_accepts_markdown_tables_without_outer_pipes() -> None: + """Outer pipes are optional while columns remain row-scoped evidence.""" + chunks = chunk_by_dom("Project | Status\n--- | ---\nAlpha | Ready") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("markdown_tr", "Project | Status"), + ("markdown_tr", "Alpha | Ready"), + ] + + +def test_chunk_by_dom_keeps_non_table_text_after_a_markdown_table() -> None: + """A malformed next row ends the table and remains ordinary prose.""" + chunks = chunk_by_dom( + "Project | Status\n--- | ---\nAlpha | Ready\nNext action without cells" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("markdown_tr", "Project | Status"), + ("markdown_tr", "Alpha | Ready"), + ("", "Next action without cells"), + ] + + def test_chunk_by_dom_word_table_rows_also_group_cells() -> None: + """WordprocessingML table cells group by their source row.""" html = "1Acme Corp" chunks = chunk_by_dom(html) @@ -197,6 +268,7 @@ def test_chunk_by_dom_word_table_rows_also_group_cells() -> None: def test_chunk_by_dom_keeps_indentation_as_metadata_not_embedding_text() -> None: + """Non-breaking-space indentation stays metadata, not semantic text.""" html = "

      Level one

        Level two

    " chunks = chunk_by_dom(html) @@ -205,6 +277,7 @@ def test_chunk_by_dom_keeps_indentation_as_metadata_not_embedding_text() -> None def test_chunk_by_dom_reads_html_and_word_indentation_declarations() -> None: + """HTML and Word indentation declarations map to comparable units.""" html = ( '

    HTML

    ' '' @@ -216,6 +289,15 @@ def test_chunk_by_dom_reads_html_and_word_indentation_declarations() -> None: assert [chunk.indent_width for chunk in chunks] == [4, 4] +def test_chunk_by_dom_ignores_invalid_word_indentation() -> None: + """Malformed Word indentation metadata cannot create a false hierarchy.""" + chunks = chunk_by_dom( + 'Word' + ) + + assert [(chunk.text, chunk.indent_width) for chunk in chunks] == [("Word", 0)] + + def test_chunk_by_dom_reads_the_css_margin_shorthand_not_just_margin_left() -> None: """Live bug (2026-08-19): a real editor (Word paste, Outlook compose) declares indentation with the box-model shorthand @@ -237,6 +319,7 @@ def test_chunk_by_dom_reads_the_css_margin_shorthand_not_just_margin_left() -> N def test_chunk_by_dom_uses_list_container_depth_as_explicit_indentation() -> None: + """Nested list-container depth contributes explicit indentation.""" html = "
    1. Outer
      1. Nested
    " chunks = chunk_by_dom(html) @@ -246,6 +329,7 @@ def test_chunk_by_dom_uses_list_container_depth_as_explicit_indentation() -> Non def test_chunk_by_source_body_splits_plain_lists_and_markdown_tables() -> None: + """Plain authored lists and tables split into semantic source units.""" body = """1. Background continuation stays with the first item. 2. Decision @@ -265,7 +349,22 @@ def test_chunk_by_source_body_splits_plain_lists_and_markdown_tables() -> None: ] +def test_chunk_by_source_body_keeps_a_single_pipe_row_as_plain_text() -> None: + """One pipe-delimited row alone is not enough evidence of a table.""" + chunks = chunk_by_source_body("Only | one row") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [("", "Only | one row")] + + +def test_chunk_by_source_body_delegates_html_to_dom_chunking() -> None: + """HTML input retains its DOM label instead of entering the plain-text splitter.""" + chunks = chunk_by_source_body("

    HTML evidence

    ") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [("p", "HTML evidence")] + + def test_chunk_by_dom_joins_visual_continuation_lines_but_keeps_list_items() -> None: + """Visual wraps join while authored list starts retain boundaries.""" html = ( '

    1. 배경
    ' " 1) 기존 대차는 이전이 필요함
    " @@ -284,6 +383,7 @@ def test_chunk_by_dom_joins_visual_continuation_lines_but_keeps_list_items() -> def test_normalize_semantic_text_removes_visual_hanging_indent_breaks() -> None: + """Hanging-indent line wraps normalize without flattening list items.""" text = ( "1. 배경\n\n" " 1) 기존 대차는 이전이 필요함\n" @@ -299,16 +399,19 @@ def test_normalize_semantic_text_removes_visual_hanging_indent_breaks() -> None: def test_normalize_semantic_text_preserves_blank_paragraph_boundaries() -> None: + """Blank lines continue to separate authored paragraphs.""" assert normalize_semantic_text("첫 문단\n\n둘째 문단") == "첫 문단\n\n둘째 문단" def test_normalize_semantic_text_does_not_embed_visual_indentation_markers() -> None: + """Presentation-only non-breaking spaces do not enter semantic text.""" assert normalize_semantic_text("\xa0\xa0계속되는 문장\n\xa0\xa0\xa0\xa0다음 줄") == ( "계속되는 문장 다음 줄" ) def test_chunk_by_dom_does_not_infer_marker_depth_without_source_whitespace() -> None: + """Marker shape alone cannot invent indentation depth.""" chunks = chunk_by_dom("

    1. Root
    1) Child
    - Detail

    ") assert [chunk.text for chunk in chunks] == ["1. Root", "1) Child", "- Detail"] @@ -316,11 +419,26 @@ def test_chunk_by_dom_does_not_infer_marker_depth_without_source_whitespace() -> def test_chunk_by_dom_empty_html_yields_no_chunks() -> None: + """Empty block, self-closing block, and empty input yield no chunks.""" assert chunk_by_dom("
    ") == [] + assert chunk_by_dom("

    ") == [] assert chunk_by_dom("") == [] +def test_chunk_by_dom_keeps_unscoped_superscript_as_plain_text() -> None: + """Orphan formatting remains plain text while empty markup adds nothing.""" + chunks = chunk_by_dom("1

    ") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [("", "1")] + + +def test_chunk_by_dom_decodes_deeply_escaped_entities_with_a_bounded_loop() -> None: + """Nested HTML entity escapes decode without an unbounded parser loop.""" + assert [chunk.text for chunk in chunk_by_dom("

    &amp;amp;amp;

    ")] == ["&"] + + def test_chunk_by_dom_falls_back_for_inline_only_markup() -> None: + """Inline-only markup falls back to one plain-text unit.""" chunks = chunk_by_dom("First inline block.Second inline block.") assert len(chunks) == 1 @@ -329,6 +447,7 @@ def test_chunk_by_dom_falls_back_for_inline_only_markup() -> None: def test_chunk_by_dom_flushes_unclosed_block_at_end_of_document() -> None: + """EOF flushes content from an unclosed source block.""" chunks = chunk_by_dom("
    Unclosed source fragment.") assert len(chunks) == 1 @@ -337,6 +456,7 @@ def test_chunk_by_dom_flushes_unclosed_block_at_end_of_document() -> None: def test_chunk_by_dom_interleaves_images_with_text_in_document_order() -> None: + """Embedded images retain their exact position between text blocks.""" tiny_png_b64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=" html = ( f"

    Before the picture.

    " @@ -354,6 +474,7 @@ def test_chunk_by_dom_interleaves_images_with_text_in_document_order() -> None: def test_chunk_by_dom_labels_text_chunks_with_their_tag_name() -> None: + """DOM text chunks expose their source tag as the unit label.""" html = "

    A paragraph.

    " chunks = chunk_by_dom(html) @@ -363,12 +484,20 @@ def test_chunk_by_dom_labels_text_chunks_with_their_tag_name() -> None: def test_chunk_by_dom_skips_malformed_image_data() -> None: + """Malformed base64 cannot create an image chunk.""" html = '

    Text.

    ' chunks = chunk_by_dom(html) assert [c.unit_type for c in chunks] == ["dom"] +def test_chunk_by_dom_skips_images_without_embedded_base64_data() -> None: + """Missing, external, and non-base64 image sources are not embedded images.""" + html = '' + assert chunk_by_dom(html) == [] + + def test_chunk_by_conversation_turn_labels_each_chunk_with_its_sender() -> None: + """Conversation units retain their sender labels and order.""" turns = [ ConversationTurn(sender="alice@example.com", text="Can we move the meeting?"), ConversationTurn(sender="bob@example.com", text="Sure, how about Thursday?"), @@ -381,6 +510,7 @@ def test_chunk_by_conversation_turn_labels_each_chunk_with_its_sender() -> None: def test_chunk_by_conversation_turn_skips_empty_turns() -> None: + """Empty turns are removed before contiguous chunk indexing.""" turns = [ ConversationTurn(sender="alice@example.com", text="Hello."), ConversationTurn(sender="bob@example.com", text=" "), @@ -423,11 +553,13 @@ def test_chunk_by_dom_captures_style_as_separate_metadata_not_embedded_text() -> def test_chunk_by_dom_style_is_none_when_element_has_no_style_attribute() -> None: + """A missing style attribute remains distinct from an empty style value.""" chunks = chunk_by_dom("

    Plain paragraph.

    ") assert chunks[0].style is None def test_chunk_by_dom_splits_on_heading_boundaries_and_labels_the_level() -> None: + """Heading boundaries preserve their source level label.""" html = "

    Quarterly Review

    Body text follows.

    " chunks = chunk_by_dom(html) From 192373bb56e90cdfd91a0e5cd43b84f36e71cab5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 09:57:30 +0900 Subject: [PATCH 7/9] docs: pin current customer tree evidence --- docs/product-technical-gap-baseline.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 626ba7a3b..96499ef80 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -66,7 +66,7 @@ claims that an unmerged PR or historical runtime observation is live behavior. | FR-10 | Standard provenance uses normalized PROV-O relations; qualified influence implies its unqualified relation and KG edges remain a navigation projection. | ADR 0011, 0065 | PROV-O implementation matrices, ontology, CI contract | | FR-11 | Post summaries expose evidence-bearing events and R&R. Requester/processor actions are nullable and may only name actors already bound to the same post summary. | ADR 0052, ADR 0100 | Commit `15e1a378` is on PR #258; one authorized target refresh stored three action rows, while corpus-wide buyer-data acceptance remains unproven | | FR-12 | A hierarchy-enrichment timeout leaves the source-grounded summary readable and the actor unbound; it never creates a guessed catalog identity. | ADR 0101, ADR 0010, ADR 0026 | Commit `1c260f20` contains the boundary, ADR, and focused test; independent review, protected-main merge, and fresh runtime evidence remain pending | -| FR-13 | Customer Master projects authorized corporate entities as a Group → Company → Plant tree. Missing-parent, self-parent, and cyclic edges remain visible as unresolved roots; the UI supports WAI-ARIA tree keyboard navigation and opens source-backed posts independently from hierarchy disclosure. | ADR 0124, ADR 0004, ADR 0010 | `customerMasterTree.ts`, `CustomerMasterTree.tsx`, pure/component tests, and Storybook on code commit `__TREE_CODE_SHA__` | +| FR-13 | Customer Master projects authorized corporate entities as a Group → Company → Plant tree. Missing-parent, self-parent, and cyclic edges remain visible as unresolved roots; the UI supports WAI-ARIA tree keyboard navigation and opens source-backed posts independently from hierarchy disclosure. | ADR 0124, ADR 0004, ADR 0010 | `customerMasterTree.ts`, `CustomerMasterTree.tsx`, pure/component tests, and Storybook on code commit `4228c48fd8795586ed6859ad4df047508a12f86f` | ## TRD From 10c8d2a7b6c2963a8178f17f5c57b20a8cb2484f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 17:39:28 +0900 Subject: [PATCH 8/9] fix: preserve structured footnote blocks in body fallback --- frontend/src/postBodyDisplay.test.ts | 16 ++++++++++++++++ frontend/src/postBodyDisplay.ts | 21 +++++++++++++++++---- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts index 6e76f52b4..666d4474e 100644 --- a/frontend/src/postBodyDisplay.test.ts +++ b/frontend/src/postBodyDisplay.test.ts @@ -67,6 +67,22 @@ describe("splitPostBody", () => { ]); }); + it("preserves HTML and Word footnote blocks as footnote paragraphs", () => { + expect( + splitPostBody( + "

    Body evidence.

    " + + "HTML note." + + "Word note." + + "

    Next action.

    ", + ), + ).toEqual([ + { kind: "text", text: "Body evidence." }, + { kind: "text", text: "HTML note.", role: "footnote" }, + { kind: "text", text: "Word note.", role: "footnote" }, + { kind: "text", text: "Next action." }, + ]); + }); + it("limits numeric superscript footnote roles to their source paragraph", () => { expect( splitPostBody( diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts index 565b12c7a..b08b4803e 100644 --- a/frontend/src/postBodyDisplay.ts +++ b/frontend/src/postBodyDisplay.ts @@ -21,7 +21,7 @@ const DATA_URI_IMG = const HTML_TAG = /<\/?[a-zA-Z][^>]*>/g; const BREAK_TAG = /]*>/gi; const BLOCK_TAG = - /<\/?(?:article|blockquote|div|h[1-6]|li|oi|ol|p|section|table|tbody|td|tfoot|th|thead|tr|ul|w:p|w:tbl|w:tr|w:tc)\b[^>]*>/gi; + /<\/?(?:article|blockquote|div|endnote|footnote|h[1-6]|li|oi|ol|p|section|table|tbody|td|tfoot|th|thead|tr|ul|w:endnote|w:footnote|w:p|w:tbl|w:tr|w:tc)\b[^>]*>/gi; const WORD_INDENT_TAG = /]*\/?\s*>/gi; const LIST_ITEM_START = /^\s*(?:[-*•·]\s+|[*†‡](?=\S)|(?:\d{1,3}|[A-Za-z가-힣])[.)]\s+|[①-⑳]\s+)/; const FOOTNOTE_START = /^\s*[*†‡]+(?=\S)/; @@ -29,6 +29,8 @@ const INDENT_MARKER = "\u0001lw-indent:"; const INDENT_MARKER_END = "\u0002"; const INDENT_MARKER_PATTERN = /lw-indent:(\d+)/g; const NUMERIC_FOOTNOTE_MARKER = "\u0003lw-numeric-footnote\u0004"; +const FOOTNOTE_BLOCK_MARKER = "\u0005lw-footnote-block\u0006"; +const FOOTNOTE_BLOCK_OPEN = /<\s*(?:footnote|endnote|w:footnote|w:endnote)\b[^>]*>/gi; const NUMERIC_SUPERSCRIPT = /]*>\s*(\d{1,3})\s*<\/sup>/gi; function stripIndentMarkers(value: string): string { @@ -111,6 +113,9 @@ function stripHtmlTags(text: string): string { listDepth += 1; return "\n\n"; } + if (/^<\s*\/?\s*(?:footnote|endnote|w:footnote|w:endnote)\b/i.test(tag)) { + return closing ? "\n\n" : `\n\n${FOOTNOTE_BLOCK_MARKER}`; + } if (/^<\s*li\b/i.test(tag)) { return `\n\n${indentMarker(Math.max(listDepth * 4, declaredIndentWidth(tag)))}`; } @@ -219,12 +224,19 @@ function isDecodableBase64(raw: string): boolean { function pushText(segments: PostBodySegment[], raw: string, indentUnit: number): void { const text = stripHtmlTags( - raw.replace(NUMERIC_SUPERSCRIPT, `${NUMERIC_FOOTNOTE_MARKER}$1`), + raw + .replace(FOOTNOTE_BLOCK_OPEN, FOOTNOTE_BLOCK_MARKER) + .replace(NUMERIC_SUPERSCRIPT, `${NUMERIC_FOOTNOTE_MARKER}$1`), ); + let pendingFootnoteBlock = false; for (const paragraph of splitSemanticParagraphs(text)) { const hasNumericSuperscriptMarker = paragraph.includes(NUMERIC_FOOTNOTE_MARKER); + const hasFootnoteBlockMarker = paragraph.includes(FOOTNOTE_BLOCK_MARKER); + pendingFootnoteBlock ||= hasFootnoteBlockMarker; const indentLevel = indentationLevel(paragraph, indentUnit); - const normalized = stripIndentMarkers(paragraph.replaceAll(NUMERIC_FOOTNOTE_MARKER, "")) + const normalized = stripIndentMarkers( + paragraph.replaceAll(NUMERIC_FOOTNOTE_MARKER, "").replaceAll(FOOTNOTE_BLOCK_MARKER, ""), + ) .replace(/^[ \t]+/, "") .replace(/[ \t]+$/gm, ""); if (normalized.trim()) { @@ -232,10 +244,11 @@ function pushText(segments: PostBodySegment[], raw: string, indentUnit: number): kind: "text", text: normalized, ...(indentLevel > 0 ? { indentLevel } : {}), - ...(hasNumericSuperscriptMarker || FOOTNOTE_START.test(normalized) + ...(hasNumericSuperscriptMarker || pendingFootnoteBlock || FOOTNOTE_START.test(normalized) ? { role: "footnote" as const } : {}), }); + pendingFootnoteBlock = false; } } } From ef907e40c21b2320aae0180f3e740f8ed49efd21 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 03:23:14 -0700 Subject: [PATCH 9/9] fix: preserve semantic image evidence tables (#303) * fix: preserve semantic image evidence tables * fix: allow deep vision evidence completion * fix: prevent image OCR evidence regression * test: cover Markdown table delimiter edges * test: cover escaped Markdown table cells * fix: normalize quoted gateway environment values * fix: preserve vision work across database restarts * fix: preserve colon-containing image OCR * fix: continue backfill after protected OCR retry * fix: preserve multiline vision captions * fix: preserve escaped image table cells * feat: preserve metric superscript and subscript semantics (#344) * feat: preserve metric script semantics * chore: satisfy chunking lint checks * fix: preserve plain metric script semantics * fix: normalize metric scripts in markdown cells --- CHANGELOG.md | 5 + docker/contextual-orchestrator/start.py | 12 +- ...103-semantic-document-evidence-contract.md | 19 ++- ...hematical-script-semantic-normalization.md | 50 +++++++ frontend/src/PostBody.stories.tsx | 19 +++ frontend/src/PostBody.test.tsx | 59 ++++++++ frontend/src/PostBody.tsx | 23 ++- frontend/src/postBodyDisplay.test.ts | 27 ++++ frontend/src/postBodyDisplay.ts | 25 +++- lineageweave/chunking.py | 46 +++++- lineageweave/image_content.py | 33 +++-- lineageweave/post_content_persistence.py | 41 +++++- scripts/backfill_post_content.py | 41 ++++-- tests/test_backfill_post_content.py | 135 ++++++++++++++++++ tests/test_chunking.py | 54 +++++++ tests/test_contextual_orchestrator_start.py | 18 ++- tests/test_image_content.py | 48 +++++++ tests/test_post_content_persistence_edges.py | 52 ++++++- 18 files changed, 659 insertions(+), 48 deletions(-) create mode 100644 docs/adr/0105-mathematical-script-semantic-normalization.md create mode 100644 tests/test_backfill_post_content.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 93c3e51c0..33d3a0eeb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,11 @@ All notable changes to this project are documented here. Format follows semantic-unit parser and buyer body renderer. See the [product and technical gap baseline](docs/product-technical-gap-baseline.md) and [ADR 0103](docs/adr/0103-semantic-document-evidence-contract.md). +- Preserve multiline VISION table rows, render parent and region OCR tables + accessibly, and request source-visible entity, relationship, layout, and + document-purpose evidence instead of a generic image caption. VISION calls + now share the structure channel's 600-second deep-agent runtime boundary; + an empty same-image retry can no longer erase previously observed OCR. - `make smoke` and `make seed` now run through the locked project `uv` environment, so local OIDC and synthetic-data workflows resolve the same diff --git a/docker/contextual-orchestrator/start.py b/docker/contextual-orchestrator/start.py index e35c9a436..9ceef0ad2 100644 --- a/docker/contextual-orchestrator/start.py +++ b/docker/contextual-orchestrator/start.py @@ -14,9 +14,11 @@ def _pop_first_env(*names: str) -> str: - """Read the first configured alias without leaving credentials in the environment.""" + """Read the first alias, removing quotes preserved by Docker env files.""" for name in names: value = os.environ.pop(name, "").strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}: + value = value[1:-1] if value: return value return "" @@ -27,7 +29,7 @@ def main() -> None: provider_key = _pop_first_env("LLM_GATEWAY_API_KEY", "LLM_API_KEY", "NVIDIA_NIM_API_KEY") if not provider_key: raise SystemExit("LLM_GATEWAY_API_KEY or LLM_API_KEY is required to start the real LLM service") - auth_token = os.environ.get("CONTEXTUAL_ORCHESTRATOR_TOKEN", "").strip() + auth_token = _pop_first_env("CONTEXTUAL_ORCHESTRATOR_TOKEN") if not auth_token: raise SystemExit("CONTEXTUAL_ORCHESTRATOR_TOKEN is required to start the authenticated LLM service") @@ -36,14 +38,14 @@ def main() -> None: raise SystemExit("LLM_GATEWAY_API_URL or LLM_GATEWAY_URL is required to start the gateway") if not provider_url.rstrip("/").endswith("/v1"): provider_url = provider_url.rstrip("/") + "/v1" - raw_limit = os.environ.pop("LLM_GATEWAY_MAX_OUTPUT_TOKENS", "4096").strip() + raw_limit = _pop_first_env("LLM_GATEWAY_MAX_OUTPUT_TOKENS") or "4096" try: max_output_tokens = int(raw_limit) except ValueError as exc: raise SystemExit("LLM_GATEWAY_MAX_OUTPUT_TOKENS must be an integer") from exc if not 64 <= max_output_tokens <= 4096: raise SystemExit("LLM_GATEWAY_MAX_OUTPUT_TOKENS must be between 64 and 4096") - raw_body_limit = os.environ.pop("CONTEXTUAL_ORCHESTRATOR_MAX_BODY_BYTES", str(8 * 1024 * 1024)).strip() + raw_body_limit = _pop_first_env("CONTEXTUAL_ORCHESTRATOR_MAX_BODY_BYTES") or str(8 * 1024 * 1024) try: max_body_bytes = int(raw_body_limit) except ValueError as exc: @@ -56,7 +58,7 @@ def main() -> None: agent["base_url"] = provider_url agent["credential_key"] = "LLM_GATEWAY_API_KEY" agent.setdefault("provider_protocol", "auto") - embedding_model = os.environ.get("LLM_GATEWAY_EMBEDDING_MODEL", "").strip() + embedding_model = _pop_first_env("LLM_GATEWAY_EMBEDDING_MODEL") if embedding_model: embedding_agents = [ agent diff --git a/docs/adr/0103-semantic-document-evidence-contract.md b/docs/adr/0103-semantic-document-evidence-contract.md index faca30d23..ea65bc111 100644 --- a/docs/adr/0103-semantic-document-evidence-contract.md +++ b/docs/adr/0103-semantic-document-evidence-contract.md @@ -36,6 +36,18 @@ recognizable header/separator/data shape and otherwise preserves plain text. 5. Keep the frontend's raw-source fallback aligned with the persisted unit labels. Persisted row units render as accessible tables; unresolved structure remains visibly unresolved and actionable. +6. Apply the same narrow Markdown-table renderer to persisted image OCR. + VISION output may use multiple `TEXT` lines so row boundaries survive; its + caption names only visible entities, relationships, layout, and document + purpose rather than offering a generic one-sentence description. The + client allows 600 seconds for deep orchestrator work; a 180-second local + cutoff already terminated a valid live response before delivery. +7. Serialize replacement per source post and reject a same-image retry when + its content hash matches non-empty persisted OCR but the retry returns no + OCR. Provider completion is transport evidence, not permission to erase a + stronger prior observation. During an operator backfill, this typed + preservation failure skips only the affected post, records it in the + aggregate result, and allows the remaining selected posts to continue. ## Rejected alternatives @@ -54,6 +66,10 @@ recognizable header/separator/data shape and otherwise preserves plain text. markup or image base64. - The database keeps the existing normalized unit tables; this decision adds no denormalized JSON field or new service. +- A weaker same-image VISION retry fails before replacement, leaving the + prior committed evidence available for a later orchestrator retry. +- A protected retry does not abort an entire operator batch; the skipped-post + count is visible to the operator without exposing raw post content. - Markdown dialects outside the narrow recognized shape remain plain text and are reported as a future parser extension rather than guessed. @@ -62,4 +78,5 @@ recognizable header/separator/data shape and otherwise preserves plain text. The baseline's synthetic tests cover numeric superscript footnotes, marker footnotes, nested `ol`/`ul`/`oi` order and depth, HTML/OOXML rows, Markdown rows, React table rendering, and unresolved indentation. Full CI remains the -release gate. +release gate. A persistence regression test proves that an empty same-hash +VISION retry cannot delete previously observed OCR. diff --git a/docs/adr/0105-mathematical-script-semantic-normalization.md b/docs/adr/0105-mathematical-script-semantic-normalization.md new file mode 100644 index 000000000..304fc52f8 --- /dev/null +++ b/docs/adr/0105-mathematical-script-semantic-normalization.md @@ -0,0 +1,50 @@ +# ADR 0105: Preserve explicit metric scripts in semantic text + +**Status:** Accepted on this PR; not protected-main truth +**Date:** 2026-08-21 +**Owners:** LineageWeave ingestion and buyer-surface maintainers + +## Context + +Source posts commonly encode a unit such as `m3`, `m3`, +`m^3`, or `m_3` with HTML or plain-text notation. Dropping the markup changes +the searchable meaning to `m3`, while treating every numeric `sup` element as +mathematics would break the existing numeric-footnote contract. Full MathML +parsing is not yet justified by the current product surface, but the loss of +explicit unit scripts is a buyer-visible defect. + +MathML 4 defines `msup`, `msub`, and `msubsup` as structural script elements; +HTML `sup`/`sub` are a permitted lighter-weight notation when detailed +mathematical markup is not required. This decision therefore adds a bounded +normalization boundary and keeps the source representation unchanged. + +## Decision + +1. Preserve the immutable source body exactly as imported. +2. In derived semantic text only, normalize an explicitly bounded metric base + (`m`, `cm`, `mm`, `km`, or `kg`, optionally preceded by a number) followed + by numeric `sup`/`sub` markup or plain-text `^`/`_` notation into Unicode + superscript/subscript digits. For example, `5m3` and `5m^3` + become `5m³`, while `m3` and `m_3` become `m₃`. +3. Keep ordinary numeric superscripts and caret expressions on prose under the existing footnote + role contract. Do not infer a mathematical formula from an arbitrary word. +4. Apply the same bounded normalization in backend semantic chunks and the + React buyer display so search text and visible text agree. +5. Defer full MathML/LaTeX parsing, expression trees, and ontology term + creation until an authorized fixture demonstrates a need beyond metric + scripts. Any such change requires a new ADR and parser contract. + +## Consequences + +- Search and the buyer popup retain the visible distinction between `m³` and + `m3` without exposing source HTML to the embedding model. +- Existing numeric-footnote tests remain unchanged because the bounded metric + pattern is the only new conversion. +- The current implementation does not claim to understand arbitrary equations; + unsupported script markup remains ordinary source text and must not be + presented as a parsed ontology expression. + +## References (APA 7th) + +World Wide Web Consortium. (2026). *Mathematical Markup Language (MathML) +Version 4.0* (W3C Recommendation). https://www.w3.org/TR/mathml4/ diff --git a/frontend/src/PostBody.stories.tsx b/frontend/src/PostBody.stories.tsx index 43260ec2d..358758cb0 100644 --- a/frontend/src/PostBody.stories.tsx +++ b/frontend/src/PostBody.stories.tsx @@ -10,6 +10,9 @@ export default meta; type Story = StoryObj; +const TINY_PNG = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="; + export const MarkdownTableEvidence: Story = { args: { body: "| Workstream | State |\n| --- | --- |\n| Alpha | Ready |", @@ -44,6 +47,22 @@ export const MarkdownTableFallback: Story = { }, }; +export const ImageOcrTableEvidence: Story = { + args: { + body: ``, + imageContent: [ + { + unit_index: 0, + mime_type: "image/png", + status_code: "completed", + extracted_text: "| Workstream | State |\n| --- | --- |\n| Alpha | Ready |", + caption: "A synthetic workstream status table.", + tags: ["table"], + }, + ], + }, +}; + export const NumericFootnote: Story = { args: { body: "

    Evidence remains attached to the source.

    1 Source note.

    ", diff --git a/frontend/src/PostBody.test.tsx b/frontend/src/PostBody.test.tsx index b92083608..1e4c915af 100644 --- a/frontend/src/PostBody.test.tsx +++ b/frontend/src/PostBody.test.tsx @@ -279,6 +279,43 @@ describe("PostBody", () => { expect(screen.getAllByRole("row")).toHaveLength(2); }); + it("renders table-shaped image OCR as accessible evidence", () => { + render( + '} + imageContent={[ + { + unit_index: 0, + mime_type: "image/png", + status_code: "completed", + extracted_text: "| Project | Status |\n| --- | --- |\n| Alpha | Ready |", + caption: "A synthetic project status table.", + tags: ["table"], + regions: [ + { + region_index: 0, + x_ratio: 0, + y_ratio: 0, + width_ratio: 1, + height_ratio: 1, + status_code: "completed", + extracted_text: "| Owner | Action |\n| --- | --- |\n| Team A | Review |", + caption: "The table region assigns an action to a team.", + tags: ["assignment"], + }, + ], + }, + ]} + />, + ); + + expect(screen.getAllByRole("table")).toHaveLength(2); + expect(screen.getByRole("columnheader", { name: "Project" })).toBeInTheDocument(); + expect(screen.getByText("Ready")).toBeInTheDocument(); + expect(screen.getByText("The table region assigns an action to a team.")).toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: "Owner" })).toBeInTheDocument(); + }); + it("marks persisted footnotes as footnote evidence", () => { render( { expect(screen.getByText("Panel")).toBeInTheDocument(); }); + it("keeps escaped pipe characters inside image OCR table cells", () => { + render( + '} + imageContent={[ + { + unit_index: 0, + mime_type: "image/png", + status_code: "described", + extracted_text: "| Item | State |\n| --- | --- |\n| Review \\| approve | Ready |", + caption: "A table image with an escaped separator.", + tags: [], + }, + ]} + />, + ); + + expect(screen.getByRole("table")).toBeInTheDocument(); + expect(screen.getByText("Review | approve")).toBeInTheDocument(); + expect(screen.getByText("Ready")).toBeInTheDocument(); + }); + it("keeps source-image placement while showing persisted OCR and caption evidence", () => { render( { - const cells = row.split("|").map((cell) => cell.trim()); + const cells = row.split(/(? cell.trim().replace(/\\\|/g, "|")); if (cells[0] === "") cells.shift(); if (cells[cells.length - 1] === "") cells.pop(); return cells; @@ -27,10 +27,20 @@ function parsePipeDelimitedTable(text: string): string[][] | null { function renderImageText(text: string) { const rows = parsePipeDelimitedTable(text); if (!rows) return

    {text}

    ; + const [header, ...bodyRows] = rows; return ( + + + {header.map((cell, cellIndex) => ( + + ))} + + - {rows.map((row, rowIndex) => ( + {bodyRows.map((row, rowIndex) => ( {row.map((cell, cellIndex) => ( @@ -77,7 +87,14 @@ function renderImageEvidence(
      {imageContent.regions.map((region) => (
    1. - {region.caption || region.extracted_text || t("Unknown")} + {region.caption ?

      {region.caption}

      : null} + {region.extracted_text ? ( +
      + {renderImageText(region.extracted_text)} +
      + ) : region.caption ? null : ( + t("Unknown") + )} {region.tags.length ? ( {t("Image tags")}: {region.tags.join(", ")} diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts index 666d4474e..e7e47cf90 100644 --- a/frontend/src/postBodyDisplay.test.ts +++ b/frontend/src/postBodyDisplay.test.ts @@ -67,6 +67,18 @@ describe("splitPostBody", () => { ]); }); + it("preserves explicit metric superscripts and subscripts", () => { + expect(splitPostBody("

      Volume: 5m3, index m3.

      ")).toEqual([ + { kind: "text", text: "Volume: 5m³, index m₃." }, + ]); + }); + + it("normalizes plain-text metric superscripts and subscripts", () => { + expect(splitPostBody("

      Volume: 5m^3, index m_3, braced m^{2}.

      ")).toEqual([ + { kind: "text", text: "Volume: 5m³, index m₃, braced m²." }, + ]); + }); + it("preserves HTML and Word footnote blocks as footnote paragraphs", () => { expect( splitPostBody( @@ -128,6 +140,21 @@ describe("splitPostBody", () => { ]); }); + it("normalizes metric scripts inside Markdown table cells", () => { + expect( + splitMarkdownTableBody("| Metric | Index |\n| --- | --- |\n| 5m^3 | m3 |"), + ).toEqual([{ kind: "table", rows: [["Metric", "Index"], ["5m³", "m₃"]] }]); + }); + + it("unescapes pipe characters inside Markdown cells without accepting a short delimiter", () => { + expect( + splitMarkdownTableBody( + "| Project | Notes |\n| :--- | ---: |\n| Alpha | Ready \\| review |", + ), + ).toEqual([{ kind: "table", rows: [["Project", "Notes"], ["Alpha", "Ready | review"]] }]); + expect(splitMarkdownTableBody("| Project | Status |\n| -- | -- |\n| Alpha | Ready |")).toBeNull(); + }); + it("leaves a plain-text post unchanged so existing popups keep their wording", () => { expect(splitPostBody("The full body text.")).toEqual([ { kind: "text", text: "The full body text." }, diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts index b08b4803e..588901483 100644 --- a/frontend/src/postBodyDisplay.ts +++ b/frontend/src/postBodyDisplay.ts @@ -32,6 +32,27 @@ const NUMERIC_FOOTNOTE_MARKER = "\u0003lw-numeric-footnote\u0004"; const FOOTNOTE_BLOCK_MARKER = "\u0005lw-footnote-block\u0006"; const FOOTNOTE_BLOCK_OPEN = /<\s*(?:footnote|endnote|w:footnote|w:endnote)\b[^>]*>/gi; const NUMERIC_SUPERSCRIPT = /]*>\s*(\d{1,3})\s*<\/sup>/gi; +const SUPERSCRIPT_DIGITS = "⁰¹²³⁴⁵⁶⁷⁸⁹"; +const SUBSCRIPT_DIGITS = "₀₁₂₃₄₅₆₇₈₉"; +const METRIC_MARKUP = + /((?]*>\s*(\d{1,3})\s*<\/\2>/gi; +const METRIC_PLAIN_SCRIPT = + /((? { + const table = kind.toLowerCase() === "sup" ? SUPERSCRIPT_DIGITS : SUBSCRIPT_DIGITS; + return `${base}${[...digits].map((digit) => table[Number(digit)]).join("")}`; + }) + .replace( + METRIC_PLAIN_SCRIPT, + (_match, base: string, kind: string, bracedDigits: string, digits: string) => { + const table = kind === "^" ? SUPERSCRIPT_DIGITS : SUBSCRIPT_DIGITS; + return `${base}${[...(bracedDigits || digits)].map((digit) => table[Number(digit)]).join("")}`; + }, + ); +} function stripIndentMarkers(value: string): string { return value @@ -224,7 +245,7 @@ function isDecodableBase64(raw: string): boolean { function pushText(segments: PostBodySegment[], raw: string, indentUnit: number): void { const text = stripHtmlTags( - raw + normalizeMetricMarkup(raw) .replace(FOOTNOTE_BLOCK_OPEN, FOOTNOTE_BLOCK_MARKER) .replace(NUMERIC_SUPERSCRIPT, `${NUMERIC_FOOTNOTE_MARKER}$1`), ); @@ -257,7 +278,7 @@ function markdownCells(line: string): string[] | null { const value = line.trim().replace(/^\|/, "").replace(/(? cell.trim().replace(/\\\|/g, "|")); - return cells.length >= 2 && cells.every(Boolean) ? cells : null; + return cells.length >= 2 && cells.every(Boolean) ? cells.map(normalizeMetricMarkup) : null; } function isMarkdownSeparatorRow(cells: string[] | null): boolean { diff --git a/lineageweave/chunking.py b/lineageweave/chunking.py index 541d18deb..7a8bbc28b 100644 --- a/lineageweave/chunking.py +++ b/lineageweave/chunking.py @@ -134,6 +134,7 @@ def _is_footnote_reference(attrs: list[tuple[str, str | None]]) -> bool: def normalize_semantic_text(text: str) -> str: """Remove visual hanging-indent breaks without changing source content.""" + text = _normalize_metric_markup(_normalize_plain_metric_scripts(text)) lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n") normalized: list[str] = [] for line in lines: @@ -296,6 +297,37 @@ def chunk_by_paragraph(text: str) -> list[Chunk]: _SENTENCE_BOUNDARY = re.compile(r"(?<=[.!?])\s+(?=[A-Z0-9가-힣])") +_SUPERSCRIPT_DIGITS = str.maketrans("0123456789", "⁰¹²³⁴⁵⁶⁷⁸⁹") +_SUBSCRIPT_DIGITS = str.maketrans("0123456789", "₀₁₂₃₄₅₆₇₈₉") +_METRIC_MARKUP = re.compile( + r"(?P(?sup|sub)\b[^>]*>\s*(?P\d{1,3})\s*", + re.IGNORECASE, +) +_METRIC_PLAIN_SCRIPT = re.compile( + r"(?P(?\^|_)\s*(?:\{(?P\d{1,3})\}|(?P\d{1,3}))", + re.IGNORECASE, +) + + +def _normalize_plain_metric_scripts(text: str) -> str: + """Normalize bounded plain-text metric exponents and indices.""" + def replace(match: re.Match[str]) -> str: + table = _SUPERSCRIPT_DIGITS if match.group("kind") == "^" else _SUBSCRIPT_DIGITS + digits = match.group("braced_digits") or match.group("digits") or "" + return f"{match.group('base')}{digits.translate(table)}" + + return _METRIC_PLAIN_SCRIPT.sub(replace, text) + + +def _normalize_metric_markup(html: str) -> str: + """Keep explicit metric superscript/subscript digits in semantic text.""" + def replace(match: re.Match[str]) -> str: + table = _SUPERSCRIPT_DIGITS if match.group("kind").lower() == "sup" else _SUBSCRIPT_DIGITS + return f"{match.group('base')}{match.group('digits').translate(table)}" + + return _METRIC_MARKUP.sub(replace, html) def chunk_by_sentence(text: str) -> list[Chunk]: @@ -554,8 +586,7 @@ def _markdown_cells(line: str) -> list[str] | None: if "|" not in line: return None value = line.strip() - if value.startswith("|"): - value = value[1:] + value = value.removeprefix("|") if value.endswith("|") and not value.endswith("\\|"): value = value[:-1] cells = [cell.strip().replace(r"\|", "|") for cell in re.split(r"(? None: found_table = True flush_pending() - entries.append(("markdown_tr", " | ".join(header))) + entries.append(("markdown_tr", " | ".join(normalize_semantic_text(cell) for cell in header))) index += 2 while index < len(lines) and lines[index].strip(): cells = _markdown_cells(lines[index]) if cells is None: break - entries.append(("markdown_tr", " | ".join(cells))) + entries.append(("markdown_tr", " | ".join(normalize_semantic_text(cell) for cell in cells))) index += 1 flush_pending() @@ -616,7 +647,10 @@ def _is_markdown_table_row(line: str) -> bool: def _render_markdown_table_row(line: str) -> str: """Keep Markdown table columns as searchable row evidence.""" - return " | ".join(cell.strip() for cell in line.strip().strip("|").split("|")) + return " | ".join( + normalize_semantic_text(cell.strip()) + for cell in line.strip().strip("|").split("|") + ) def _split_plain_text_units(text: str) -> list[tuple[str, int, str]]: @@ -695,7 +729,7 @@ def chunk_by_dom(html: str) -> list[Chunk]: ] parser = _BlockTextExtractor() - parser.feed(html) + parser.feed(_normalize_metric_markup(html)) entries = parser.finished() chunks: list[Chunk] = [] for index, ( diff --git a/lineageweave/image_content.py b/lineageweave/image_content.py index f6500ffd3..a0d298a91 100644 --- a/lineageweave/image_content.py +++ b/lineageweave/image_content.py @@ -141,7 +141,8 @@ class ImageDescription: Attributes: extracted_text: OCR result -- every piece of legible text found in the image, empty string if none. - caption: one-sentence description of what the image shows. + caption: factual description of the visible entities, relationships, + and layout that make the image useful as semantic evidence. tags: short tags for the main objects/subjects, for independent keyword search separate from the free-text caption. """ @@ -177,15 +178,18 @@ def describe(self, image_bytes: bytes, mime_type: str) -> ImageDescription: # p _RESPONSE_FORMAT = ( - "Examine this image. Reply with EXACTLY three lines, no extra commentary:\n" + "Examine this image. Reply with exactly the three labeled sections below and no " + "extra commentary. TEXT may span multiple lines; CAPTION and TAGS stay on their " + "labeled lines.\n" "TEXT: \n" + "If the image contains a table, preserve its row/column structure as a Markdown " + "pipe table: one row per line, with a separator row immediately after the visible " + "header. Never flatten a table into an unstructured word list or invent a header " + "that is not visible.>\n" "CAPTION: <2-4 concise, evidence-grounded sentences describing the visible layout, " "objects, relationships, directions, measurements, and labels; do not guess " - "anything that is not visible>\n" - "TAGS: " + "anything that is not visible. Omit anything the pixels do not support.>\n" + "TAGS: " ) _REGION_RESPONSE_FORMAT = ( "Find distinct meaningful visual regions in this image for separate OCR and description. " @@ -251,14 +255,15 @@ def _parse_description(content: str) -> ImageDescription: remainder = _strip_outer_markdown_emphasis(match.group(2)) if remainder: fields[label].append(remainder) - multiline_field = "TEXT" if label == "TEXT" else None + multiline_field = label if label in {"TEXT", "CAPTION"} else None continue - if re.match(r"^\s*[*_`>#\-\s]*[A-Za-z][A-Za-z0-9 _-]*\s*:", line): - multiline_field = None - continue - if multiline_field == "TEXT" and line.strip(): - fields["TEXT"].append(_strip_outer_markdown_emphasis(line)) + if multiline_field in {"TEXT", "CAPTION"} and line.strip(): + # A colon is common inside OCR (for example ``Date: 2026-08-21``). + # Only the known response labels above end the active section; + # treating every colon as a provider label loses real image text + # or the continuation of a detailed caption. + fields[multiline_field].append(_strip_outer_markdown_emphasis(line)) if not fields["TEXT"] and not fields["CAPTION"]: raise ImageDescriptionParseError("vision response had no usable TEXT or CAPTION content") @@ -289,7 +294,7 @@ def __init__( api_key: str, model: str | None = None, *, - timeout: float = 180.0, + timeout: float = 600.0, allow_insecure_http: bool = False, ) -> None: parsed = urlparse(base_url) diff --git a/lineageweave/post_content_persistence.py b/lineageweave/post_content_persistence.py index 86dcfd4de..44d7da8bf 100644 --- a/lineageweave/post_content_persistence.py +++ b/lineageweave/post_content_persistence.py @@ -30,6 +30,10 @@ _LOGGER = logging.getLogger(__name__) +class ImageOcrPreservationError(RuntimeError): + """A retry would erase stronger OCR already persisted for the same image.""" + + def _bounded_unit_batches( # noqa: UP047 - retain Python 3.10 compatibility. units: list[tuple[_BatchKey, str]], ) -> list[list[tuple[_BatchKey, str]]]: @@ -84,12 +88,24 @@ async def persist_post_content( Provider calls happen before the short database transaction. A failed or unavailable embedding call writes no vector row; it never writes a zero or - guessed vector. The raw body remains in ``source_post`` for future retry. + guessed vector. A same-image retry cannot replace non-empty persisted OCR + with an empty result. The raw body remains in ``source_post`` for future + retry. """ normalized = normalized_result or normalize_post_body(body, vision_client) chunks = chunk_by_source_body(body) image_results = {result.chunk_index: result for result in normalized.image_results} formatting = {hint.chunk_index: hint.style for hint in normalized.formatting_hints} + image_ocr_by_sha256: dict[str, bool] = {} + for chunk in chunks: + if chunk.unit_type != "image" or chunk.image_data is None: + continue + result = image_results.get(chunk.index) + description = result.description if result else None + content_sha256 = hashlib.sha256(chunk.image_data).hexdigest() + image_ocr_by_sha256[content_sha256] = image_ocr_by_sha256.get( + content_sha256, False + ) or bool(description and description.extracted_text.strip()) prepared: list[tuple[Chunk, str, str | None]] = [] for chunk in chunks: @@ -226,6 +242,29 @@ async def persist_post_content( ) async with conn.transaction(): + if image_ocr_by_sha256: + await conn.fetchval( + "select post_id from source_post where post_id = $1 for update", + post_id, + ) + previous_images = await conn.fetch( + """ + select image.content_sha256, image.extracted_text + from post_content_unit unit + join post_content_image image using (post_content_unit_id) + where unit.post_id = $1 + and nullif(btrim(image.extracted_text), '') is not null + """, + post_id, + ) + if any( + row["content_sha256"] in image_ocr_by_sha256 + and not image_ocr_by_sha256[row["content_sha256"]] + for row in previous_images + ): + raise ImageOcrPreservationError( + "refusing to replace non-empty image OCR with an empty retry result" + ) await conn.execute("delete from post_content_unit where post_id = $1", post_id) unit_ids: dict[int, str] = {} for chunk, unit_text, style in prepared: diff --git a/scripts/backfill_post_content.py b/scripts/backfill_post_content.py index ef54c91a7..1263f25b8 100644 --- a/scripts/backfill_post_content.py +++ b/scripts/backfill_post_content.py @@ -27,7 +27,10 @@ from lineageweave.image_content import NullImageContentClient, orchestrator_vision_client from lineageweave.llm_context import build_post_llm_metadata, use_llm_metadata from lineageweave.post_content_normalization import normalize_post_body -from lineageweave.post_content_persistence import persist_post_content +from lineageweave.post_content_persistence import ( + ImageOcrPreservationError, + persist_post_content, +) from lineageweave.post_structure import ContextualOrchestratorPostStructureClient, NullPostStructureClient @@ -55,6 +58,15 @@ def _parser() -> argparse.ArgumentParser: return parser +async def _ensure_open_connection( + conn: asyncpg.Connection, target_dsn: str +) -> asyncpg.Connection: + """Reconnect after a database restart without repeating VISION work.""" + if not conn.is_closed(): + return conn + return await asyncpg.connect(target_dsn) + + async def backfill_post_content( target_dsn: str, raw_post_ids: list[str] | None, @@ -217,17 +229,22 @@ async def backfill_post_content( if described_images == 0 and not normalized.text.strip(): result["skipped_posts"] += 1 continue - await persist_post_content( - conn, - str(row["post_id"]), - row["post_body"], - vision_client=vision_client, - embedding_client=embedding_client, - embedding_model_code=embedding_model or None, - normalized_result=normalized, - structure_client=structure_client, - post_title=row["post_title"], - ) + conn = await _ensure_open_connection(conn, target_dsn) + try: + await persist_post_content( + conn, + str(row["post_id"]), + row["post_body"], + vision_client=vision_client, + embedding_client=embedding_client, + embedding_model_code=embedding_model or None, + normalized_result=normalized, + structure_client=structure_client, + post_title=row["post_title"], + ) + except ImageOcrPreservationError: + result["skipped_posts"] += 1 + continue async with conn.transaction(): await record_post_content_backfill_success( conn, diff --git a/tests/test_backfill_post_content.py b/tests/test_backfill_post_content.py new file mode 100644 index 000000000..d5f034297 --- /dev/null +++ b/tests/test_backfill_post_content.py @@ -0,0 +1,135 @@ +"""Operator backfill connection recovery contracts.""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace + +from scripts import backfill_post_content +from lineageweave.post_content_persistence import ImageOcrPreservationError + + +def test_reconnects_only_after_database_connection_closes(monkeypatch) -> None: + replacement_connection = object() + connected_dsns: list[str] = [] + + class Connection: + def __init__(self, closed: bool) -> None: + self._closed = closed + + def is_closed(self) -> bool: + return self._closed + + async def connect(dsn: str): + connected_dsns.append(dsn) + return replacement_connection + + monkeypatch.setattr(backfill_post_content.asyncpg, "connect", connect) + + current_connection = Connection(False) + assert ( + asyncio.run(backfill_post_content._ensure_open_connection(current_connection, "dsn")) + is current_connection + ) + assert asyncio.run(backfill_post_content._ensure_open_connection(Connection(True), "dsn")) is replacement_connection + assert connected_dsns == ["dsn"] + + +def test_backfill_skips_ocr_protected_post_and_continues(monkeypatch) -> None: + post_ids = [ + "00505695-0000-1fd1-8000-000000000001", + "00505695-0000-1fd1-8000-000000000002", + ] + + class Transaction: + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, traceback): + return False + + class Connection: + def is_closed(self) -> bool: + return False + + async def fetch(self, query, *args): + return [{"post_id": post_id} for post_id in post_ids] + + async def fetchrow(self, query, post_id): + return { + "post_id": post_id, + "post_title": "Synthetic title", + "post_body": "Synthetic body", + "author_account_id": None, + "source_process_unit_code": None, + "source_author_code": None, + "source_company_code": None, + "source_customer_code": None, + "source_project_code": None, + "source_sales_pool_code": None, + "corporate_entity_code": None, + } + + def transaction(self): + return Transaction() + + async def fetchval(self, query, *args): + return 0 + + async def close(self): + return None + + connection = Connection() + persisted: list[str] = [] + + async def connect(_dsn): + return connection + + async def persist(conn, post_id, body, **kwargs): + persisted.append(post_id) + if post_id == post_ids[0]: + raise ImageOcrPreservationError("protected") + return 1 + + async def record_success(conn, post_id, body): + return None + + class MetadataContext: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, traceback): + return False + + monkeypatch.setattr(backfill_post_content.asyncpg, "connect", connect) + monkeypatch.setattr(backfill_post_content, "persist_post_content", persist) + monkeypatch.setattr( + backfill_post_content, + "record_post_content_backfill_success", + record_success, + ) + monkeypatch.setattr( + backfill_post_content, + "normalize_post_body", + lambda body, vision_client: SimpleNamespace(image_results=(), text="text"), + ) + monkeypatch.setattr( + backfill_post_content, + "build_post_llm_metadata", + lambda post_id, row: {}, + ) + monkeypatch.setattr( + backfill_post_content, + "use_llm_metadata", + lambda metadata: MetadataContext(), + ) + + result = asyncio.run( + backfill_post_content.backfill_post_content( + "dsn", post_ids, limit=None, normalize_only=True + ) + ) + + assert persisted == post_ids + assert result["processed_posts"] == 1 + assert result["skipped_posts"] == 1 diff --git a/tests/test_chunking.py b/tests/test_chunking.py index 7525746c8..258bc9ad4 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -191,6 +191,45 @@ def test_chunk_by_dom_does_not_treat_non_numeric_superscript_as_footnote() -> No ] +def test_chunk_by_dom_preserves_explicit_metric_superscripts() -> None: + """A unit exponent remains searchable mathematical evidence.""" + chunks = chunk_by_dom("

      Volume: 5m3.

      ") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("p", "Volume: 5m³."), + ] + + +def test_chunk_by_dom_preserves_explicit_metric_subscripts() -> None: + """A unit subscript is retained without changing ordinary footnotes.""" + chunks = chunk_by_dom("

      Index m3 is measured.

      ") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("p", "Index m₃ is measured."), + ] + + +def test_chunk_by_source_body_normalizes_plain_metric_scripts() -> None: + """Plain-text metric scripts retain searchable exponent/index semantics.""" + chunks = chunk_by_source_body("Volume: 5m^3; index m_3; braced m^{2}.") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("", "Volume: 5m³; index m₃; braced m²."), + ] + + +def test_chunk_by_source_body_normalizes_metric_scripts_in_markdown_table_cells() -> None: + """Markdown table cells retain the same searchable metric semantics as prose.""" + chunks = chunk_by_source_body( + "| Metric | Index |\n| --- | --- |\n| 5m^3 | m_3 |" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("tr", "Metric | Index"), + ("tr", "5m³ | m₃"), + ] + + def test_chunk_by_dom_preserves_nested_list_order_and_depth() -> None: """Nested list items retain source order and increasing depth.""" chunks = chunk_by_dom( @@ -221,6 +260,21 @@ def test_chunk_by_dom_keeps_markdown_table_rows_as_searchable_units() -> None: ] +def test_chunk_by_dom_preserves_escaped_markdown_pipes_and_rejects_short_delimiters() -> None: + escaped = chunk_by_dom( + "| Field | Notes |\n| --- | --- |\n| Owner | Ready \\| review |" + ) + assert [(chunk.label, chunk.text) for chunk in escaped] == [ + ("markdown_tr", "Field | Notes"), + ("markdown_tr", "Owner | Ready | review"), + ] + + short_delimiter = chunk_by_dom( + "| Field | Value |\n| -- | -- |\n| Owner | Buyer |" + ) + assert all(chunk.label != "markdown_tr" for chunk in short_delimiter) + + def test_chunk_by_dom_keeps_prose_around_markdown_table_rows() -> None: """Prose surrounding a Markdown table stays in document order.""" chunks = chunk_by_dom( diff --git a/tests/test_contextual_orchestrator_start.py b/tests/test_contextual_orchestrator_start.py index e3618aa82..a4f6bcfad 100644 --- a/tests/test_contextual_orchestrator_start.py +++ b/tests/test_contextual_orchestrator_start.py @@ -52,6 +52,15 @@ def test_gateway_api_key_accepts_local_compatibility_alias(monkeypatch) -> None: assert module._pop_first_env("LLM_GATEWAY_API_KEY", "LLM_API_KEY") == "compatibility-key" +def test_env_file_quotes_are_not_part_of_transport_values(monkeypatch) -> None: + module = _load_start_module() + monkeypatch.setenv("LLM_GATEWAY_API_KEY", "'provider-key'") + monkeypatch.setenv("LLM_GATEWAY_API_URL", '"https://gateway.example/v1"') + + assert module._pop_first_env("LLM_GATEWAY_API_KEY") == "provider-key" + assert module._pop_first_env("LLM_GATEWAY_API_URL") == "https://gateway.example/v1" + + def test_bootstrap_registers_embedding_agent_before_deleting_secrets(monkeypatch) -> None: module = _load_start_module() captured: dict[str, object] = {} @@ -90,9 +99,11 @@ def serve() -> None: monkeypatch.setattr(module, "Path", FakePath) monkeypatch.setattr(sys, "argv", ["start.py"]) monkeypatch.setenv("LLM_GATEWAY_API_KEY", "provider-key") - monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_TOKEN", "orchestrator-token") + monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_TOKEN", "'orchestrator-token'") monkeypatch.setenv("LLM_GATEWAY_API_URL", "https://gateway.example") - monkeypatch.setenv("LLM_GATEWAY_EMBEDDING_MODEL", "embedding-model") + monkeypatch.setenv("LLM_GATEWAY_EMBEDDING_MODEL", "'embedding-model'") + monkeypatch.setenv("LLM_GATEWAY_MAX_OUTPUT_TOKENS", "'2048'") + monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_MAX_BODY_BYTES", '"65536"') module.main() @@ -100,6 +111,9 @@ def serve() -> None: assert isinstance(argv, list) assert "--embedding-provider-url" not in argv assert "--embedding-model" not in argv + assert argv[argv.index("--auth-token") + 1] == "orchestrator-token" + assert argv[argv.index("--max-output-tokens") + 1] == "2048" + assert argv[argv.index("--max-body-bytes") + 1] == "65536" assert captured["credentials"] == [ ("NVIDIA_NIM_API_KEY", "provider-key"), ("LLM_GATEWAY_API_KEY", "provider-key"), diff --git a/tests/test_image_content.py b/tests/test_image_content.py index de7fc49b5..148a979c5 100644 --- a/tests/test_image_content.py +++ b/tests/test_image_content.py @@ -85,6 +85,35 @@ def test_parse_description_preserves_multiline_ocr_text() -> None: assert description.caption == "A scanned page." +def test_parse_description_preserves_multiline_caption_evidence() -> None: + """Detailed VISION captions remain complete when providers wrap lines.""" + content = ( + "CAPTION: A project status table for the customer meeting.\n" + "The left column lists workstreams and the right column lists owners.\n" + "TEXT: Workstream | Owner\nAlpha | Team A\nTAGS: table, assignment" + ) + + description = _parse_description(content) + + assert description.caption == ( + "A project status table for the customer meeting.\n" + "The left column lists workstreams and the right column lists owners." + ) + assert description.extracted_text == "Workstream | Owner\nAlpha | Team A" + + +def test_parse_description_preserves_ocr_lines_that_contain_colons() -> None: + """A colon in a scanned field is OCR content, not a new response field.""" + content = ( + "TEXT: Invoice\nDate: 2026-08-21\nTotal: 100\n" + "CAPTION: A synthetic invoice.\nTAGS: invoice" + ) + + description = _parse_description(content) + + assert description.extracted_text == "Invoice\nDate: 2026-08-21\nTotal: 100" + + def test_parse_description_preserves_table_row_structure_in_ocr_text() -> None: """Live gap (2026-08-19): an image containing a table used to have its text flattened into an unstructured word list on OCR, the same @@ -202,6 +231,14 @@ def test_orchestrator_vision_client_does_not_double_v1() -> None: assert client._base_url == "https://gateway.example/v1" +def test_orchestrator_vision_client_allows_deep_agent_runtime() -> None: + """A valid VISION result must not be cut off by the former 180s limit.""" + client = orchestrator_vision_client("https://gateway.example", "key") + + assert isinstance(client, OpenAiCompatibleVisionClient) + assert client._timeout == 600.0 + + def test_orchestrator_vision_client_is_null_when_unconfigured() -> None: client = orchestrator_vision_client("", "") assert isinstance(client, NullImageContentClient) @@ -226,6 +263,17 @@ def test_ocr_prompt_asks_for_table_row_structure() -> None: assert "table" in _RESPONSE_FORMAT.lower() +def test_ocr_prompt_allows_multiline_tables_and_requests_semantic_detail() -> None: + """Table rows and ontology-ready captions must fit the response contract.""" + prompt = _RESPONSE_FORMAT.lower() + + assert "text may span multiple lines" in prompt + assert "separator row" in prompt + assert "named entities" in prompt + assert "relationships" in prompt + assert "exactly three lines" not in prompt + + def test_region_prompt_requires_full_image_coverage() -> None: """Live gap (2026-08-19): "distinct meaningful visual regions" alone let the model describe only the most visually striking part of an diff --git a/tests/test_post_content_persistence_edges.py b/tests/test_post_content_persistence_edges.py index 24bec9e6d..4eac1b77e 100644 --- a/tests/test_post_content_persistence_edges.py +++ b/tests/test_post_content_persistence_edges.py @@ -2,6 +2,7 @@ import asyncio from contextlib import asynccontextmanager +import hashlib from types import SimpleNamespace import pytest @@ -27,9 +28,10 @@ def _persist(*args: object, **kwargs: object) -> int: class _Connection: - def __init__(self) -> None: + def __init__(self, previous_images: tuple[dict[str, object], ...] = ()) -> None: self.executed: list[tuple[str, tuple[object, ...]]] = [] self.fetchvals: list[tuple[str, tuple[object, ...]]] = [] + self.previous_images = previous_images self._next_id = 0 @asynccontextmanager @@ -45,6 +47,10 @@ async def fetchval(self, query: str, *args: object) -> str: self._next_id += 1 return f"id-{self._next_id}" + async def fetch(self, query: str, *args: object) -> list[dict[str, object]]: + assert "post_content_image" in query + return list(self.previous_images) + class _EmbedMany: available = True @@ -190,7 +196,18 @@ def test_persists_image_tags_formatting_and_embeddings() -> None: ), ), ) - conn = _Connection() + conn = _Connection( + ( + { + "content_sha256": hashlib.sha256(b"hello").hexdigest(), + "extracted_text": "previous OCR", + }, + { + "content_sha256": hashlib.sha256(b"replaced image").hexdigest(), + "extracted_text": "removed image OCR", + }, + ) + ) embedder = _EmbedMany() count = _persist( @@ -213,6 +230,37 @@ def test_persists_image_tags_formatting_and_embeddings() -> None: assert sum("post_content_embedding_value" in query for query, _args in conn.executed) == 2 * len(chunks) +def test_same_image_retry_cannot_replace_existing_ocr_with_empty_text() -> None: + body = '' + image_index = chunk_by_dom(body)[0].index + normalized = NormalizedPostContent( + text="[image: updated caption]", + image_results=( + ImageContentResult( + image_index, + "image/png", + "described", + SimpleNamespace(caption="updated caption", extracted_text="", tags=()), + ), + ), + ) + conn = _Connection( + ( + { + "content_sha256": hashlib.sha256(b"hello").hexdigest(), + "description_status_code": "described", + "extracted_text": "prior OCR", + "caption": "prior caption", + }, + ) + ) + + with pytest.raises(RuntimeError, match="refusing to replace non-empty image OCR"): + _persist(conn, "post-4", body, normalized_result=normalized) + + assert not any("delete from post_content_unit" in query for query, _args in conn.executed) + + def test_legacy_embed_and_malformed_vectors_never_write_vectors() -> None: conn = _Connection() legacy = _LegacyEmbed([float("nan")])
    + {cell} +
    {cell}