From 0269b9eb2ec3675d958d6221fe45b9975d595347 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 17:51:33 +0900 Subject: [PATCH 01/12] fix: preserve semantic image evidence tables --- CHANGELOG.md | 3 ++ ...102-semantic-document-evidence-contract.md | 4 ++ frontend/src/PostBody.stories.tsx | 19 ++++++++++ frontend/src/PostBody.test.tsx | 37 +++++++++++++++++++ frontend/src/PostBody.tsx | 16 +++++++- lineageweave/image_content.py | 20 ++++++---- tests/test_image_content.py | 11 ++++++ 7 files changed, 101 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e27533f4f..b3ebb44e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,9 @@ All notable changes to this project are documented here. Format follows semantic-unit parser and buyer body renderer. See the [product and technical gap baseline](docs/product-technical-gap-baseline.md) and [ADR 0102](docs/adr/0102-semantic-document-evidence-contract.md). +- Preserve multiline VISION table rows, render parent and region OCR tables + accessibly, and request source-visible entity, relationship, layout, and + document-purpose evidence instead of a generic image caption. ## [2.10.0] - 2026-08-18 diff --git a/docs/adr/0102-semantic-document-evidence-contract.md b/docs/adr/0102-semantic-document-evidence-contract.md index fa20e86eb..2873b8e3a 100644 --- a/docs/adr/0102-semantic-document-evidence-contract.md +++ b/docs/adr/0102-semantic-document-evidence-contract.md @@ -36,6 +36,10 @@ recognizable header/separator/data shape and otherwise preserves plain text. 5. Keep the frontend's raw-source fallback aligned with the persisted unit labels. Persisted row units render as accessible tables; unresolved structure remains visibly unresolved and actionable. +6. Apply the same narrow Markdown-table renderer to persisted image OCR. + VISION output may use multiple `TEXT` lines so row boundaries survive; its + caption names only visible entities, relationships, layout, and document + purpose rather than offering a generic one-sentence description. ## Rejected alternatives diff --git a/frontend/src/PostBody.stories.tsx b/frontend/src/PostBody.stories.tsx index 43260ec2d..358758cb0 100644 --- a/frontend/src/PostBody.stories.tsx +++ b/frontend/src/PostBody.stories.tsx @@ -10,6 +10,9 @@ export default meta; type Story = StoryObj; +const TINY_PNG = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="; + export const MarkdownTableEvidence: Story = { args: { body: "| Workstream | State |\n| --- | --- |\n| Alpha | Ready |", @@ -44,6 +47,22 @@ export const MarkdownTableFallback: Story = { }, }; +export const ImageOcrTableEvidence: Story = { + args: { + body: ``, + imageContent: [ + { + unit_index: 0, + mime_type: "image/png", + status_code: "completed", + extracted_text: "| Workstream | State |\n| --- | --- |\n| Alpha | Ready |", + caption: "A synthetic workstream status table.", + tags: ["table"], + }, + ], + }, +}; + export const NumericFootnote: Story = { args: { body: "

Evidence remains attached to the source.

1 Source note.

", diff --git a/frontend/src/PostBody.test.tsx b/frontend/src/PostBody.test.tsx index cb5cfd064..93baf810b 100644 --- a/frontend/src/PostBody.test.tsx +++ b/frontend/src/PostBody.test.tsx @@ -164,6 +164,43 @@ describe("PostBody", () => { expect(screen.getByText("Next action.")).toBeInTheDocument(); }); + it("renders table-shaped image OCR as accessible evidence", () => { + render( + '} + imageContent={[ + { + unit_index: 0, + mime_type: "image/png", + status_code: "completed", + extracted_text: "| Project | Status |\n| --- | --- |\n| Alpha | Ready |", + caption: "A synthetic project status table.", + tags: ["table"], + regions: [ + { + region_index: 0, + x_ratio: 0, + y_ratio: 0, + width_ratio: 1, + height_ratio: 1, + status_code: "completed", + extracted_text: "| Owner | Action |\n| --- | --- |\n| Team A | Review |", + caption: "The table region assigns an action to a team.", + tags: ["assignment"], + }, + ], + }, + ]} + />, + ); + + expect(screen.getAllByRole("table")).toHaveLength(2); + expect(screen.getByRole("columnheader", { name: "Project" })).toBeInTheDocument(); + expect(screen.getByText("Ready")).toBeInTheDocument(); + expect(screen.getByText("The table region assigns an action to a team.")).toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: "Owner" })).toBeInTheDocument(); + }); + it("marks persisted footnotes as footnote evidence", () => { render( {t("Text detected in image")} -

{imageContent.extracted_text}

+ {renderExtractedText(imageContent.extracted_text)} ) : null} {imageContent?.regions?.length ? ( @@ -46,7 +46,14 @@ function renderSegment(segment: PostBodySegment, index: number, imageContent?: P
    {imageContent.regions.map((region) => (
  1. - {region.caption || region.extracted_text || t("Unknown")} + {region.caption ?

    {region.caption}

    : null} + {region.extracted_text ? ( +
    + {renderExtractedText(region.extracted_text)} +
    + ) : region.caption ? null : ( + t("Unknown") + )}
    2. ))}
@@ -162,6 +169,11 @@ function renderMarkdownBlocks(blocks: MarkdownBodyBlock[]): ReactNode[] { }); } +function renderExtractedText(text: string): ReactNode { + const markdownBlocks = splitMarkdownTableBody(text); + return markdownBlocks ? renderMarkdownBlocks(markdownBlocks) :

{text}

; +} + export function PostBody({ body, imageContent = [], diff --git a/lineageweave/image_content.py b/lineageweave/image_content.py index 3aa2c9038..ef1d40366 100644 --- a/lineageweave/image_content.py +++ b/lineageweave/image_content.py @@ -139,7 +139,8 @@ class ImageDescription: Attributes: extracted_text: OCR result -- every piece of legible text found in the image, empty string if none. - caption: one-sentence description of what the image shows. + caption: factual description of the visible entities, relationships, + and layout that make the image useful as semantic evidence. tags: short tags for the main objects/subjects, for independent keyword search separate from the free-text caption. """ @@ -175,13 +176,18 @@ def describe(self, image_bytes: bytes, mime_type: str) -> ImageDescription: # p _RESPONSE_FORMAT = ( - "Examine this image. Reply with EXACTLY three lines, no extra commentary:\n" + "Examine this image. Reply with exactly the three labeled sections below and no " + "extra commentary. TEXT may span multiple lines; CAPTION and TAGS stay on their " + "labeled lines.\n" "TEXT: \n" - "CAPTION: \n" - "TAGS: " + "If the image contains a table, preserve its row/column structure as a Markdown " + "pipe table: one row per line, with a separator row immediately after the visible " + "header. Never flatten a table into an unstructured word list or invent a header " + "that is not visible.>\n" + "CAPTION: \n" + "TAGS: " ) _REGION_RESPONSE_FORMAT = ( "Find distinct meaningful visual regions in this image for separate OCR and description. " diff --git a/tests/test_image_content.py b/tests/test_image_content.py index f694635a4..23146be4e 100644 --- a/tests/test_image_content.py +++ b/tests/test_image_content.py @@ -226,6 +226,17 @@ def test_ocr_prompt_asks_for_table_row_structure() -> None: assert "table" in _RESPONSE_FORMAT.lower() +def test_ocr_prompt_allows_multiline_tables_and_requests_semantic_detail() -> None: + """Table rows and ontology-ready captions must fit the response contract.""" + prompt = _RESPONSE_FORMAT.lower() + + assert "text may span multiple lines" in prompt + assert "separator row" in prompt + assert "named entities" in prompt + assert "relationships" in prompt + assert "exactly three lines" not in prompt + + def test_region_prompt_requires_full_image_coverage() -> None: """Live gap (2026-08-19): "distinct meaningful visual regions" alone let the model describe only the most visually striking part of an From e055463151ac7ba37abb733dbee103c1f82e9c2e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 18:02:54 +0900 Subject: [PATCH 02/12] fix: allow deep vision evidence completion --- CHANGELOG.md | 3 ++- docs/adr/0103-semantic-document-evidence-contract.md | 4 +++- lineageweave/image_content.py | 2 +- tests/test_image_content.py | 8 ++++++++ 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fcbaead76..4956b8dcb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,8 @@ All notable changes to this project are documented here. Format follows [ADR 0103](docs/adr/0103-semantic-document-evidence-contract.md). - Preserve multiline VISION table rows, render parent and region OCR tables accessibly, and request source-visible entity, relationship, layout, and - document-purpose evidence instead of a generic image caption. + document-purpose evidence instead of a generic image caption. VISION calls + now share the structure channel's 600-second deep-agent runtime boundary. ## [2.10.0] - 2026-08-18 diff --git a/docs/adr/0103-semantic-document-evidence-contract.md b/docs/adr/0103-semantic-document-evidence-contract.md index c7c7c7794..3d252189d 100644 --- a/docs/adr/0103-semantic-document-evidence-contract.md +++ b/docs/adr/0103-semantic-document-evidence-contract.md @@ -39,7 +39,9 @@ recognizable header/separator/data shape and otherwise preserves plain text. 6. Apply the same narrow Markdown-table renderer to persisted image OCR. VISION output may use multiple `TEXT` lines so row boundaries survive; its caption names only visible entities, relationships, layout, and document - purpose rather than offering a generic one-sentence description. + purpose rather than offering a generic one-sentence description. The + client allows 600 seconds for deep orchestrator work; a 180-second local + cutoff already terminated a valid live response before delivery. ## Rejected alternatives diff --git a/lineageweave/image_content.py b/lineageweave/image_content.py index ef1d40366..9b52626bc 100644 --- a/lineageweave/image_content.py +++ b/lineageweave/image_content.py @@ -293,7 +293,7 @@ def __init__( api_key: str, model: str | None = None, *, - timeout: float = 180.0, + timeout: float = 600.0, allow_insecure_http: bool = False, ) -> None: parsed = urlparse(base_url) diff --git a/tests/test_image_content.py b/tests/test_image_content.py index 23146be4e..39c20496b 100644 --- a/tests/test_image_content.py +++ b/tests/test_image_content.py @@ -202,6 +202,14 @@ def test_orchestrator_vision_client_does_not_double_v1() -> None: assert client._base_url == "https://gateway.example/v1" +def test_orchestrator_vision_client_allows_deep_agent_runtime() -> None: + """A valid VISION result must not be cut off by the former 180s limit.""" + client = orchestrator_vision_client("https://gateway.example", "key") + + assert isinstance(client, OpenAiCompatibleVisionClient) + assert client._timeout == 600.0 + + def test_orchestrator_vision_client_is_null_when_unconfigured() -> None: client = orchestrator_vision_client("", "") assert isinstance(client, NullImageContentClient) From f5b234591d0265ca5ab2aed692ce24422de0527a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 18:21:43 +0900 Subject: [PATCH 03/12] fix: prevent image OCR evidence regression --- CHANGELOG.md | 3 +- ...103-semantic-document-evidence-contract.md | 9 +++- lineageweave/post_content_persistence.py | 37 ++++++++++++- tests/test_post_content_persistence_edges.py | 54 ++++++++++++++++++- 4 files changed, 98 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4956b8dcb..4ab4eb530 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,8 @@ All notable changes to this project are documented here. Format follows - Preserve multiline VISION table rows, render parent and region OCR tables accessibly, and request source-visible entity, relationship, layout, and document-purpose evidence instead of a generic image caption. VISION calls - now share the structure channel's 600-second deep-agent runtime boundary. + now share the structure channel's 600-second deep-agent runtime boundary; + an empty same-image retry can no longer erase previously observed OCR. ## [2.10.0] - 2026-08-18 diff --git a/docs/adr/0103-semantic-document-evidence-contract.md b/docs/adr/0103-semantic-document-evidence-contract.md index 3d252189d..c4b8de7b4 100644 --- a/docs/adr/0103-semantic-document-evidence-contract.md +++ b/docs/adr/0103-semantic-document-evidence-contract.md @@ -42,6 +42,10 @@ recognizable header/separator/data shape and otherwise preserves plain text. purpose rather than offering a generic one-sentence description. The client allows 600 seconds for deep orchestrator work; a 180-second local cutoff already terminated a valid live response before delivery. +7. Serialize replacement per source post and reject a same-image retry when + its content hash matches non-empty persisted OCR but the retry returns no + OCR. Provider completion is transport evidence, not permission to erase a + stronger prior observation. ## Rejected alternatives @@ -60,6 +64,8 @@ recognizable header/separator/data shape and otherwise preserves plain text. markup or image base64. - The database keeps the existing normalized unit tables; this decision adds no denormalized JSON field or new service. +- A weaker same-image VISION retry fails before replacement, leaving the + prior committed evidence available for a later orchestrator retry. - Markdown dialects outside the narrow recognized shape remain plain text and are reported as a future parser extension rather than guessed. @@ -68,4 +74,5 @@ recognizable header/separator/data shape and otherwise preserves plain text. The baseline's synthetic tests cover numeric superscript footnotes, marker footnotes, nested `ol`/`ul`/`oi` order and depth, HTML/OOXML rows, Markdown rows, React table rendering, and unresolved indentation. Full CI remains the -release gate. +release gate. A persistence regression test proves that an empty same-hash +VISION retry cannot delete previously observed OCR. diff --git a/lineageweave/post_content_persistence.py b/lineageweave/post_content_persistence.py index 4df7597b0..3abbe0ffc 100644 --- a/lineageweave/post_content_persistence.py +++ b/lineageweave/post_content_persistence.py @@ -76,12 +76,24 @@ async def persist_post_content( Provider calls happen before the short database transaction. A failed or unavailable embedding call writes no vector row; it never writes a zero or - guessed vector. The raw body remains in ``source_post`` for future retry. + guessed vector. A same-image retry cannot replace non-empty persisted OCR + with an empty result. The raw body remains in ``source_post`` for future + retry. """ normalized = normalized_result or normalize_post_body(body, vision_client) chunks = chunk_by_source_body(body) image_results = {result.chunk_index: result for result in normalized.image_results} formatting = {hint.chunk_index: hint.style for hint in normalized.formatting_hints} + image_ocr_by_sha256: dict[str, bool] = {} + for chunk in chunks: + if chunk.unit_type != "image" or chunk.image_data is None: + continue + result = image_results.get(chunk.index) + description = result.description if result else None + content_sha256 = hashlib.sha256(chunk.image_data).hexdigest() + image_ocr_by_sha256[content_sha256] = image_ocr_by_sha256.get( + content_sha256, False + ) or bool(description and description.extracted_text.strip()) prepared: list[tuple[Chunk, str, str | None]] = [] for chunk in chunks: @@ -196,6 +208,29 @@ async def persist_post_content( continue async with conn.transaction(): + if image_ocr_by_sha256: + await conn.fetchval( + "select post_id from source_post where post_id = $1 for update", + post_id, + ) + previous_images = await conn.fetch( + """ + select image.content_sha256, image.extracted_text + from post_content_unit unit + join post_content_image image using (post_content_unit_id) + where unit.post_id = $1 + and nullif(btrim(image.extracted_text), '') is not null + """, + post_id, + ) + if any( + row["content_sha256"] in image_ocr_by_sha256 + and not image_ocr_by_sha256[row["content_sha256"]] + for row in previous_images + ): + raise RuntimeError( + "refusing to replace non-empty image OCR with an empty retry result" + ) await conn.execute("delete from post_content_unit where post_id = $1", post_id) unit_ids: dict[int, str] = {} for chunk, unit_text, style in prepared: diff --git a/tests/test_post_content_persistence_edges.py b/tests/test_post_content_persistence_edges.py index 89bc77708..3f07dea90 100644 --- a/tests/test_post_content_persistence_edges.py +++ b/tests/test_post_content_persistence_edges.py @@ -2,8 +2,11 @@ import asyncio from contextlib import asynccontextmanager +import hashlib from types import SimpleNamespace +import pytest + from lineageweave.chunking import chunk_by_dom from lineageweave.image_content import ImageRegion from lineageweave.post_content_normalization import ( @@ -20,9 +23,10 @@ def _persist(*args: object, **kwargs: object) -> int: class _Connection: - def __init__(self) -> None: + def __init__(self, previous_images: tuple[dict[str, object], ...] = ()) -> None: self.executed: list[tuple[str, tuple[object, ...]]] = [] self.fetchvals: list[tuple[str, tuple[object, ...]]] = [] + self.previous_images = previous_images self._next_id = 0 @asynccontextmanager @@ -38,6 +42,10 @@ async def fetchval(self, query: str, *args: object) -> str: self._next_id += 1 return f"id-{self._next_id}" + async def fetch(self, query: str, *args: object) -> list[dict[str, object]]: + assert "post_content_image" in query + return list(self.previous_images) + class _EmbedMany: available = True @@ -124,7 +132,18 @@ def test_persists_image_tags_formatting_and_embeddings() -> None: ), ), ) - conn = _Connection() + conn = _Connection( + ( + { + "content_sha256": hashlib.sha256(b"hello").hexdigest(), + "extracted_text": "previous OCR", + }, + { + "content_sha256": hashlib.sha256(b"replaced image").hexdigest(), + "extracted_text": "removed image OCR", + }, + ) + ) embedder = _EmbedMany() count = _persist( @@ -147,6 +166,37 @@ def test_persists_image_tags_formatting_and_embeddings() -> None: assert sum("post_content_embedding_value" in query for query, _args in conn.executed) == 2 * len(chunks) +def test_same_image_retry_cannot_replace_existing_ocr_with_empty_text() -> None: + body = '' + image_index = chunk_by_dom(body)[0].index + normalized = NormalizedPostContent( + text="[image: updated caption]", + image_results=( + ImageContentResult( + image_index, + "image/png", + "described", + SimpleNamespace(caption="updated caption", extracted_text="", tags=()), + ), + ), + ) + conn = _Connection( + ( + { + "content_sha256": hashlib.sha256(b"hello").hexdigest(), + "description_status_code": "described", + "extracted_text": "prior OCR", + "caption": "prior caption", + }, + ) + ) + + with pytest.raises(RuntimeError, match="refusing to replace non-empty image OCR"): + _persist(conn, "post-4", body, normalized_result=normalized) + + assert not any("delete from post_content_unit" in query for query, _args in conn.executed) + + def test_legacy_embed_and_malformed_vectors_never_write_vectors() -> None: conn = _Connection() legacy = _LegacyEmbed([float("nan")]) From 2702fd69ab6000fc6c7bc6cf9ed5d3dfdb362970 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 18:45:09 +0900 Subject: [PATCH 04/12] test: cover Markdown table delimiter edges --- frontend/src/postBodyDisplay.test.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts index 2b82ab068..dbdbc9c6c 100644 --- a/frontend/src/postBodyDisplay.test.ts +++ b/frontend/src/postBodyDisplay.test.ts @@ -84,6 +84,15 @@ describe("splitPostBody", () => { ]); }); + it("unescapes pipe characters inside Markdown cells without accepting a short delimiter", () => { + expect( + splitMarkdownTableBody( + "| Project | Notes |\n| :--- | ---: |\n| Alpha | Ready \\| review |", + ), + ).toEqual([{ kind: "table", rows: [["Project", "Notes"], ["Alpha", "Ready | review"]] }]); + expect(splitMarkdownTableBody("| Project | Status |\n| -- | -- |\n| Alpha | Ready |")).toBeNull(); + }); + it("leaves a plain-text post unchanged so existing popups keep their wording", () => { expect(splitPostBody("The full body text.")).toEqual([ { kind: "text", text: "The full body text." }, From fe0a4f260d005ccd5bee2aaacc2fd6f76b6fbe06 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 18:50:19 +0900 Subject: [PATCH 05/12] test: cover escaped Markdown table cells --- tests/test_chunking.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_chunking.py b/tests/test_chunking.py index 07a0e2917..d0e3c34a6 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -163,6 +163,21 @@ def test_chunk_by_dom_keeps_markdown_table_rows_as_searchable_units() -> None: ] +def test_chunk_by_dom_preserves_escaped_markdown_pipes_and_rejects_short_delimiters() -> None: + escaped = chunk_by_dom( + "| Field | Notes |\n| --- | --- |\n| Owner | Ready \\| review |" + ) + assert [(chunk.label, chunk.text) for chunk in escaped] == [ + ("markdown_tr", "Field | Notes"), + ("markdown_tr", "Owner | Ready | review"), + ] + + short_delimiter = chunk_by_dom( + "| Field | Value |\n| -- | -- |\n| Owner | Buyer |" + ) + assert all(chunk.label != "markdown_tr" for chunk in short_delimiter) + + def test_chunk_by_dom_keeps_prose_around_markdown_table_rows() -> None: chunks = chunk_by_dom( "Intro.\n\n| Project | Status |\n| --- | --- |\n| Alpha | Ready |\n\nNext action." From e5d0221f32baf6f6d6e785a977191d981fb6abe9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 19:22:23 +0900 Subject: [PATCH 06/12] fix: normalize quoted gateway environment values --- docker/contextual-orchestrator/start.py | 4 +++- tests/test_contextual_orchestrator_start.py | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/docker/contextual-orchestrator/start.py b/docker/contextual-orchestrator/start.py index e35c9a436..db9c3e93d 100644 --- a/docker/contextual-orchestrator/start.py +++ b/docker/contextual-orchestrator/start.py @@ -14,9 +14,11 @@ def _pop_first_env(*names: str) -> str: - """Read the first configured alias without leaving credentials in the environment.""" + """Read the first alias, removing quotes preserved by Docker env files.""" for name in names: value = os.environ.pop(name, "").strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}: + value = value[1:-1] if value: return value return "" diff --git a/tests/test_contextual_orchestrator_start.py b/tests/test_contextual_orchestrator_start.py index e3618aa82..3ddbc2de7 100644 --- a/tests/test_contextual_orchestrator_start.py +++ b/tests/test_contextual_orchestrator_start.py @@ -52,6 +52,15 @@ def test_gateway_api_key_accepts_local_compatibility_alias(monkeypatch) -> None: assert module._pop_first_env("LLM_GATEWAY_API_KEY", "LLM_API_KEY") == "compatibility-key" +def test_env_file_quotes_are_not_part_of_transport_values(monkeypatch) -> None: + module = _load_start_module() + monkeypatch.setenv("LLM_GATEWAY_API_KEY", "'provider-key'") + monkeypatch.setenv("LLM_GATEWAY_API_URL", '"https://gateway.example/v1"') + + assert module._pop_first_env("LLM_GATEWAY_API_KEY") == "provider-key" + assert module._pop_first_env("LLM_GATEWAY_API_URL") == "https://gateway.example/v1" + + def test_bootstrap_registers_embedding_agent_before_deleting_secrets(monkeypatch) -> None: module = _load_start_module() captured: dict[str, object] = {} From b7e6e82d60c6a939666c745e41ccdc47ce4a4f16 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 19:33:44 +0900 Subject: [PATCH 07/12] fix: preserve vision work across database restarts --- scripts/backfill_post_content.py | 10 +++++++++ tests/test_backfill_post_content.py | 33 +++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 tests/test_backfill_post_content.py diff --git a/scripts/backfill_post_content.py b/scripts/backfill_post_content.py index 11b7c0ab3..518efd27d 100644 --- a/scripts/backfill_post_content.py +++ b/scripts/backfill_post_content.py @@ -54,6 +54,15 @@ def _parser() -> argparse.ArgumentParser: return parser +async def _ensure_open_connection( + conn: asyncpg.Connection, target_dsn: str +) -> asyncpg.Connection: + """Reconnect after a database restart without repeating VISION work.""" + if not conn.is_closed(): + return conn + return await asyncpg.connect(target_dsn) + + async def backfill_post_content( target_dsn: str, raw_post_ids: list[str] | None, @@ -216,6 +225,7 @@ async def backfill_post_content( if described_images == 0 and not normalized.text.strip(): result["skipped_posts"] += 1 continue + conn = await _ensure_open_connection(conn, target_dsn) await persist_post_content( conn, str(row["post_id"]), diff --git a/tests/test_backfill_post_content.py b/tests/test_backfill_post_content.py new file mode 100644 index 000000000..1486a1713 --- /dev/null +++ b/tests/test_backfill_post_content.py @@ -0,0 +1,33 @@ +"""Operator backfill connection recovery contracts.""" + +from __future__ import annotations + +import asyncio + +from scripts import backfill_post_content + + +def test_reconnects_only_after_database_connection_closes(monkeypatch) -> None: + replacement_connection = object() + connected_dsns: list[str] = [] + + class Connection: + def __init__(self, closed: bool) -> None: + self._closed = closed + + def is_closed(self) -> bool: + return self._closed + + async def connect(dsn: str): + connected_dsns.append(dsn) + return replacement_connection + + monkeypatch.setattr(backfill_post_content.asyncpg, "connect", connect) + + current_connection = Connection(False) + assert ( + asyncio.run(backfill_post_content._ensure_open_connection(current_connection, "dsn")) + is current_connection + ) + assert asyncio.run(backfill_post_content._ensure_open_connection(Connection(True), "dsn")) is replacement_connection + assert connected_dsns == ["dsn"] From ba71bc16a1ec796dd6b8cd22236b9185589a4328 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 10:23:28 +0900 Subject: [PATCH 08/12] fix: preserve colon-containing image OCR --- lineageweave/image_content.py | 6 +++--- tests/test_image_content.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/lineageweave/image_content.py b/lineageweave/image_content.py index 9b52626bc..9762e5772 100644 --- a/lineageweave/image_content.py +++ b/lineageweave/image_content.py @@ -256,10 +256,10 @@ def _parse_description(content: str) -> ImageDescription: multiline_field = "TEXT" if label == "TEXT" else None continue - if re.match(r"^\s*[*_`>#\-\s]*[A-Za-z][A-Za-z0-9 _-]*\s*:", line): - multiline_field = None - continue if multiline_field == "TEXT" and line.strip(): + # A colon is common inside OCR (for example ``Date: 2026-08-21``). + # Only the known response labels above end the TEXT section; + # treating every colon as a provider label loses real image text. fields["TEXT"].append(_strip_outer_markdown_emphasis(line)) if not fields["TEXT"] and not fields["CAPTION"]: diff --git a/tests/test_image_content.py b/tests/test_image_content.py index 39c20496b..9d1bf0be8 100644 --- a/tests/test_image_content.py +++ b/tests/test_image_content.py @@ -85,6 +85,18 @@ def test_parse_description_preserves_multiline_ocr_text() -> None: assert description.caption == "A scanned page." +def test_parse_description_preserves_ocr_lines_that_contain_colons() -> None: + """A colon in a scanned field is OCR content, not a new response field.""" + content = ( + "TEXT: Invoice\nDate: 2026-08-21\nTotal: 100\n" + "CAPTION: A synthetic invoice.\nTAGS: invoice" + ) + + description = _parse_description(content) + + assert description.extracted_text == "Invoice\nDate: 2026-08-21\nTotal: 100" + + def test_parse_description_preserves_table_row_structure_in_ocr_text() -> None: """Live gap (2026-08-19): an image containing a table used to have its text flattened into an unstructured word list on OCR, the same From a4a6008080d7620825bb57284b034d97df9b46c4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 15:05:13 +0900 Subject: [PATCH 09/12] fix: continue backfill after protected OCR retry --- docker/contextual-orchestrator/start.py | 8 +- ...103-semantic-document-evidence-contract.md | 6 +- lineageweave/post_content_persistence.py | 6 +- scripts/backfill_post_content.py | 31 +++--- tests/test_backfill_post_content.py | 102 ++++++++++++++++++ tests/test_contextual_orchestrator_start.py | 9 +- 6 files changed, 142 insertions(+), 20 deletions(-) diff --git a/docker/contextual-orchestrator/start.py b/docker/contextual-orchestrator/start.py index db9c3e93d..9ceef0ad2 100644 --- a/docker/contextual-orchestrator/start.py +++ b/docker/contextual-orchestrator/start.py @@ -29,7 +29,7 @@ def main() -> None: provider_key = _pop_first_env("LLM_GATEWAY_API_KEY", "LLM_API_KEY", "NVIDIA_NIM_API_KEY") if not provider_key: raise SystemExit("LLM_GATEWAY_API_KEY or LLM_API_KEY is required to start the real LLM service") - auth_token = os.environ.get("CONTEXTUAL_ORCHESTRATOR_TOKEN", "").strip() + auth_token = _pop_first_env("CONTEXTUAL_ORCHESTRATOR_TOKEN") if not auth_token: raise SystemExit("CONTEXTUAL_ORCHESTRATOR_TOKEN is required to start the authenticated LLM service") @@ -38,14 +38,14 @@ def main() -> None: raise SystemExit("LLM_GATEWAY_API_URL or LLM_GATEWAY_URL is required to start the gateway") if not provider_url.rstrip("/").endswith("/v1"): provider_url = provider_url.rstrip("/") + "/v1" - raw_limit = os.environ.pop("LLM_GATEWAY_MAX_OUTPUT_TOKENS", "4096").strip() + raw_limit = _pop_first_env("LLM_GATEWAY_MAX_OUTPUT_TOKENS") or "4096" try: max_output_tokens = int(raw_limit) except ValueError as exc: raise SystemExit("LLM_GATEWAY_MAX_OUTPUT_TOKENS must be an integer") from exc if not 64 <= max_output_tokens <= 4096: raise SystemExit("LLM_GATEWAY_MAX_OUTPUT_TOKENS must be between 64 and 4096") - raw_body_limit = os.environ.pop("CONTEXTUAL_ORCHESTRATOR_MAX_BODY_BYTES", str(8 * 1024 * 1024)).strip() + raw_body_limit = _pop_first_env("CONTEXTUAL_ORCHESTRATOR_MAX_BODY_BYTES") or str(8 * 1024 * 1024) try: max_body_bytes = int(raw_body_limit) except ValueError as exc: @@ -58,7 +58,7 @@ def main() -> None: agent["base_url"] = provider_url agent["credential_key"] = "LLM_GATEWAY_API_KEY" agent.setdefault("provider_protocol", "auto") - embedding_model = os.environ.get("LLM_GATEWAY_EMBEDDING_MODEL", "").strip() + embedding_model = _pop_first_env("LLM_GATEWAY_EMBEDDING_MODEL") if embedding_model: embedding_agents = [ agent diff --git a/docs/adr/0103-semantic-document-evidence-contract.md b/docs/adr/0103-semantic-document-evidence-contract.md index c4b8de7b4..ea65bc111 100644 --- a/docs/adr/0103-semantic-document-evidence-contract.md +++ b/docs/adr/0103-semantic-document-evidence-contract.md @@ -45,7 +45,9 @@ recognizable header/separator/data shape and otherwise preserves plain text. 7. Serialize replacement per source post and reject a same-image retry when its content hash matches non-empty persisted OCR but the retry returns no OCR. Provider completion is transport evidence, not permission to erase a - stronger prior observation. + stronger prior observation. During an operator backfill, this typed + preservation failure skips only the affected post, records it in the + aggregate result, and allows the remaining selected posts to continue. ## Rejected alternatives @@ -66,6 +68,8 @@ recognizable header/separator/data shape and otherwise preserves plain text. no denormalized JSON field or new service. - A weaker same-image VISION retry fails before replacement, leaving the prior committed evidence available for a later orchestrator retry. +- A protected retry does not abort an entire operator batch; the skipped-post + count is visible to the operator without exposing raw post content. - Markdown dialects outside the narrow recognized shape remain plain text and are reported as a future parser extension rather than guessed. diff --git a/lineageweave/post_content_persistence.py b/lineageweave/post_content_persistence.py index 860a08990..44d7da8bf 100644 --- a/lineageweave/post_content_persistence.py +++ b/lineageweave/post_content_persistence.py @@ -30,6 +30,10 @@ _LOGGER = logging.getLogger(__name__) +class ImageOcrPreservationError(RuntimeError): + """A retry would erase stronger OCR already persisted for the same image.""" + + def _bounded_unit_batches( # noqa: UP047 - retain Python 3.10 compatibility. units: list[tuple[_BatchKey, str]], ) -> list[list[tuple[_BatchKey, str]]]: @@ -258,7 +262,7 @@ async def persist_post_content( and not image_ocr_by_sha256[row["content_sha256"]] for row in previous_images ): - raise RuntimeError( + raise ImageOcrPreservationError( "refusing to replace non-empty image OCR with an empty retry result" ) await conn.execute("delete from post_content_unit where post_id = $1", post_id) diff --git a/scripts/backfill_post_content.py b/scripts/backfill_post_content.py index 2dad1d500..1263f25b8 100644 --- a/scripts/backfill_post_content.py +++ b/scripts/backfill_post_content.py @@ -27,7 +27,10 @@ from lineageweave.image_content import NullImageContentClient, orchestrator_vision_client from lineageweave.llm_context import build_post_llm_metadata, use_llm_metadata from lineageweave.post_content_normalization import normalize_post_body -from lineageweave.post_content_persistence import persist_post_content +from lineageweave.post_content_persistence import ( + ImageOcrPreservationError, + persist_post_content, +) from lineageweave.post_structure import ContextualOrchestratorPostStructureClient, NullPostStructureClient @@ -227,17 +230,21 @@ async def backfill_post_content( result["skipped_posts"] += 1 continue conn = await _ensure_open_connection(conn, target_dsn) - await persist_post_content( - conn, - str(row["post_id"]), - row["post_body"], - vision_client=vision_client, - embedding_client=embedding_client, - embedding_model_code=embedding_model or None, - normalized_result=normalized, - structure_client=structure_client, - post_title=row["post_title"], - ) + try: + await persist_post_content( + conn, + str(row["post_id"]), + row["post_body"], + vision_client=vision_client, + embedding_client=embedding_client, + embedding_model_code=embedding_model or None, + normalized_result=normalized, + structure_client=structure_client, + post_title=row["post_title"], + ) + except ImageOcrPreservationError: + result["skipped_posts"] += 1 + continue async with conn.transaction(): await record_post_content_backfill_success( conn, diff --git a/tests/test_backfill_post_content.py b/tests/test_backfill_post_content.py index 1486a1713..d5f034297 100644 --- a/tests/test_backfill_post_content.py +++ b/tests/test_backfill_post_content.py @@ -3,8 +3,10 @@ from __future__ import annotations import asyncio +from types import SimpleNamespace from scripts import backfill_post_content +from lineageweave.post_content_persistence import ImageOcrPreservationError def test_reconnects_only_after_database_connection_closes(monkeypatch) -> None: @@ -31,3 +33,103 @@ async def connect(dsn: str): ) assert asyncio.run(backfill_post_content._ensure_open_connection(Connection(True), "dsn")) is replacement_connection assert connected_dsns == ["dsn"] + + +def test_backfill_skips_ocr_protected_post_and_continues(monkeypatch) -> None: + post_ids = [ + "00505695-0000-1fd1-8000-000000000001", + "00505695-0000-1fd1-8000-000000000002", + ] + + class Transaction: + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, traceback): + return False + + class Connection: + def is_closed(self) -> bool: + return False + + async def fetch(self, query, *args): + return [{"post_id": post_id} for post_id in post_ids] + + async def fetchrow(self, query, post_id): + return { + "post_id": post_id, + "post_title": "Synthetic title", + "post_body": "Synthetic body", + "author_account_id": None, + "source_process_unit_code": None, + "source_author_code": None, + "source_company_code": None, + "source_customer_code": None, + "source_project_code": None, + "source_sales_pool_code": None, + "corporate_entity_code": None, + } + + def transaction(self): + return Transaction() + + async def fetchval(self, query, *args): + return 0 + + async def close(self): + return None + + connection = Connection() + persisted: list[str] = [] + + async def connect(_dsn): + return connection + + async def persist(conn, post_id, body, **kwargs): + persisted.append(post_id) + if post_id == post_ids[0]: + raise ImageOcrPreservationError("protected") + return 1 + + async def record_success(conn, post_id, body): + return None + + class MetadataContext: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, traceback): + return False + + monkeypatch.setattr(backfill_post_content.asyncpg, "connect", connect) + monkeypatch.setattr(backfill_post_content, "persist_post_content", persist) + monkeypatch.setattr( + backfill_post_content, + "record_post_content_backfill_success", + record_success, + ) + monkeypatch.setattr( + backfill_post_content, + "normalize_post_body", + lambda body, vision_client: SimpleNamespace(image_results=(), text="text"), + ) + monkeypatch.setattr( + backfill_post_content, + "build_post_llm_metadata", + lambda post_id, row: {}, + ) + monkeypatch.setattr( + backfill_post_content, + "use_llm_metadata", + lambda metadata: MetadataContext(), + ) + + result = asyncio.run( + backfill_post_content.backfill_post_content( + "dsn", post_ids, limit=None, normalize_only=True + ) + ) + + assert persisted == post_ids + assert result["processed_posts"] == 1 + assert result["skipped_posts"] == 1 diff --git a/tests/test_contextual_orchestrator_start.py b/tests/test_contextual_orchestrator_start.py index 3ddbc2de7..a4f6bcfad 100644 --- a/tests/test_contextual_orchestrator_start.py +++ b/tests/test_contextual_orchestrator_start.py @@ -99,9 +99,11 @@ def serve() -> None: monkeypatch.setattr(module, "Path", FakePath) monkeypatch.setattr(sys, "argv", ["start.py"]) monkeypatch.setenv("LLM_GATEWAY_API_KEY", "provider-key") - monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_TOKEN", "orchestrator-token") + monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_TOKEN", "'orchestrator-token'") monkeypatch.setenv("LLM_GATEWAY_API_URL", "https://gateway.example") - monkeypatch.setenv("LLM_GATEWAY_EMBEDDING_MODEL", "embedding-model") + monkeypatch.setenv("LLM_GATEWAY_EMBEDDING_MODEL", "'embedding-model'") + monkeypatch.setenv("LLM_GATEWAY_MAX_OUTPUT_TOKENS", "'2048'") + monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_MAX_BODY_BYTES", '"65536"') module.main() @@ -109,6 +111,9 @@ def serve() -> None: assert isinstance(argv, list) assert "--embedding-provider-url" not in argv assert "--embedding-model" not in argv + assert argv[argv.index("--auth-token") + 1] == "orchestrator-token" + assert argv[argv.index("--max-output-tokens") + 1] == "2048" + assert argv[argv.index("--max-body-bytes") + 1] == "65536" assert captured["credentials"] == [ ("NVIDIA_NIM_API_KEY", "provider-key"), ("LLM_GATEWAY_API_KEY", "provider-key"), From 1c861f43da49bee1076d87c047ba63fd280ea75f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 17:17:38 +0900 Subject: [PATCH 10/12] fix: preserve multiline vision captions --- lineageweave/image_content.py | 11 ++++++----- tests/test_image_content.py | 17 +++++++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/lineageweave/image_content.py b/lineageweave/image_content.py index aba174c99..89b64a2f6 100644 --- a/lineageweave/image_content.py +++ b/lineageweave/image_content.py @@ -255,14 +255,15 @@ def _parse_description(content: str) -> ImageDescription: remainder = _strip_outer_markdown_emphasis(match.group(2)) if remainder: fields[label].append(remainder) - multiline_field = "TEXT" if label == "TEXT" else None + multiline_field = label if label in {"TEXT", "CAPTION"} else None continue - if multiline_field == "TEXT" and line.strip(): + if multiline_field in {"TEXT", "CAPTION"} and line.strip(): # A colon is common inside OCR (for example ``Date: 2026-08-21``). - # Only the known response labels above end the TEXT section; - # treating every colon as a provider label loses real image text. - fields["TEXT"].append(_strip_outer_markdown_emphasis(line)) + # Only the known response labels above end the active section; + # treating every colon as a provider label loses real image text + # or the continuation of a detailed caption. + fields[multiline_field].append(_strip_outer_markdown_emphasis(line)) if not fields["TEXT"] and not fields["CAPTION"]: raise ImageDescriptionParseError("vision response had no usable TEXT or CAPTION content") diff --git a/tests/test_image_content.py b/tests/test_image_content.py index e65dab4ef..148a979c5 100644 --- a/tests/test_image_content.py +++ b/tests/test_image_content.py @@ -85,6 +85,23 @@ def test_parse_description_preserves_multiline_ocr_text() -> None: assert description.caption == "A scanned page." +def test_parse_description_preserves_multiline_caption_evidence() -> None: + """Detailed VISION captions remain complete when providers wrap lines.""" + content = ( + "CAPTION: A project status table for the customer meeting.\n" + "The left column lists workstreams and the right column lists owners.\n" + "TEXT: Workstream | Owner\nAlpha | Team A\nTAGS: table, assignment" + ) + + description = _parse_description(content) + + assert description.caption == ( + "A project status table for the customer meeting.\n" + "The left column lists workstreams and the right column lists owners." + ) + assert description.extracted_text == "Workstream | Owner\nAlpha | Team A" + + def test_parse_description_preserves_ocr_lines_that_contain_colons() -> None: """A colon in a scanned field is OCR content, not a new response field.""" content = ( From 7727cb1f5b5be4ef9205c25d59a7ab9818a4bb65 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 17:47:13 +0900 Subject: [PATCH 11/12] fix: preserve escaped image table cells --- frontend/src/PostBody.test.tsx | 22 ++++++++++++++++++++++ frontend/src/PostBody.tsx | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/frontend/src/PostBody.test.tsx b/frontend/src/PostBody.test.tsx index bbb03e682..1e4c915af 100644 --- a/frontend/src/PostBody.test.tsx +++ b/frontend/src/PostBody.test.tsx @@ -408,6 +408,28 @@ describe("PostBody", () => { expect(screen.getByText("Panel")).toBeInTheDocument(); }); + it("keeps escaped pipe characters inside image OCR table cells", () => { + render( + '} + imageContent={[ + { + unit_index: 0, + mime_type: "image/png", + status_code: "described", + extracted_text: "| Item | State |\n| --- | --- |\n| Review \\| approve | Ready |", + caption: "A table image with an escaped separator.", + tags: [], + }, + ]} + />, + ); + + expect(screen.getByRole("table")).toBeInTheDocument(); + expect(screen.getByText("Review | approve")).toBeInTheDocument(); + expect(screen.getByText("Ready")).toBeInTheDocument(); + }); + it("keeps source-image placement while showing persisted OCR and caption evidence", () => { render( { - const cells = row.split("|").map((cell) => cell.trim()); + const cells = row.split(/(? cell.trim().replace(/\\\|/g, "|")); if (cells[0] === "") cells.shift(); if (cells[cells.length - 1] === "") cells.pop(); return cells; From 497ac120c2ea22f97ef2e4a4bcd15fc2a3610046 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 03:23:02 -0700 Subject: [PATCH 12/12] feat: preserve metric superscript and subscript semantics (#344) * feat: preserve metric script semantics * chore: satisfy chunking lint checks * fix: preserve plain metric script semantics * fix: normalize metric scripts in markdown cells --- ...hematical-script-semantic-normalization.md | 50 +++++++++++++++++++ frontend/src/postBodyDisplay.test.ts | 18 +++++++ frontend/src/postBodyDisplay.ts | 25 +++++++++- lineageweave/chunking.py | 46 ++++++++++++++--- tests/test_chunking.py | 39 +++++++++++++++ 5 files changed, 170 insertions(+), 8 deletions(-) create mode 100644 docs/adr/0105-mathematical-script-semantic-normalization.md diff --git a/docs/adr/0105-mathematical-script-semantic-normalization.md b/docs/adr/0105-mathematical-script-semantic-normalization.md new file mode 100644 index 000000000..304fc52f8 --- /dev/null +++ b/docs/adr/0105-mathematical-script-semantic-normalization.md @@ -0,0 +1,50 @@ +# ADR 0105: Preserve explicit metric scripts in semantic text + +**Status:** Accepted on this PR; not protected-main truth +**Date:** 2026-08-21 +**Owners:** LineageWeave ingestion and buyer-surface maintainers + +## Context + +Source posts commonly encode a unit such as `m3`, `m3`, +`m^3`, or `m_3` with HTML or plain-text notation. Dropping the markup changes +the searchable meaning to `m3`, while treating every numeric `sup` element as +mathematics would break the existing numeric-footnote contract. Full MathML +parsing is not yet justified by the current product surface, but the loss of +explicit unit scripts is a buyer-visible defect. + +MathML 4 defines `msup`, `msub`, and `msubsup` as structural script elements; +HTML `sup`/`sub` are a permitted lighter-weight notation when detailed +mathematical markup is not required. This decision therefore adds a bounded +normalization boundary and keeps the source representation unchanged. + +## Decision + +1. Preserve the immutable source body exactly as imported. +2. In derived semantic text only, normalize an explicitly bounded metric base + (`m`, `cm`, `mm`, `km`, or `kg`, optionally preceded by a number) followed + by numeric `sup`/`sub` markup or plain-text `^`/`_` notation into Unicode + superscript/subscript digits. For example, `5m3` and `5m^3` + become `5m³`, while `m3` and `m_3` become `m₃`. +3. Keep ordinary numeric superscripts and caret expressions on prose under the existing footnote + role contract. Do not infer a mathematical formula from an arbitrary word. +4. Apply the same bounded normalization in backend semantic chunks and the + React buyer display so search text and visible text agree. +5. Defer full MathML/LaTeX parsing, expression trees, and ontology term + creation until an authorized fixture demonstrates a need beyond metric + scripts. Any such change requires a new ADR and parser contract. + +## Consequences + +- Search and the buyer popup retain the visible distinction between `m³` and + `m3` without exposing source HTML to the embedding model. +- Existing numeric-footnote tests remain unchanged because the bounded metric + pattern is the only new conversion. +- The current implementation does not claim to understand arbitrary equations; + unsupported script markup remains ordinary source text and must not be + presented as a parsed ontology expression. + +## References (APA 7th) + +World Wide Web Consortium. (2026). *Mathematical Markup Language (MathML) +Version 4.0* (W3C Recommendation). https://www.w3.org/TR/mathml4/ diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts index 0e85f6025..e7e47cf90 100644 --- a/frontend/src/postBodyDisplay.test.ts +++ b/frontend/src/postBodyDisplay.test.ts @@ -67,6 +67,18 @@ describe("splitPostBody", () => { ]); }); + it("preserves explicit metric superscripts and subscripts", () => { + expect(splitPostBody("

Volume: 5m3, index m3.

")).toEqual([ + { kind: "text", text: "Volume: 5m³, index m₃." }, + ]); + }); + + it("normalizes plain-text metric superscripts and subscripts", () => { + expect(splitPostBody("

Volume: 5m^3, index m_3, braced m^{2}.

")).toEqual([ + { kind: "text", text: "Volume: 5m³, index m₃, braced m²." }, + ]); + }); + it("preserves HTML and Word footnote blocks as footnote paragraphs", () => { expect( splitPostBody( @@ -128,6 +140,12 @@ describe("splitPostBody", () => { ]); }); + it("normalizes metric scripts inside Markdown table cells", () => { + expect( + splitMarkdownTableBody("| Metric | Index |\n| --- | --- |\n| 5m^3 | m3 |"), + ).toEqual([{ kind: "table", rows: [["Metric", "Index"], ["5m³", "m₃"]] }]); + }); + it("unescapes pipe characters inside Markdown cells without accepting a short delimiter", () => { expect( splitMarkdownTableBody( diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts index b08b4803e..588901483 100644 --- a/frontend/src/postBodyDisplay.ts +++ b/frontend/src/postBodyDisplay.ts @@ -32,6 +32,27 @@ const NUMERIC_FOOTNOTE_MARKER = "\u0003lw-numeric-footnote\u0004"; const FOOTNOTE_BLOCK_MARKER = "\u0005lw-footnote-block\u0006"; const FOOTNOTE_BLOCK_OPEN = /<\s*(?:footnote|endnote|w:footnote|w:endnote)\b[^>]*>/gi; const NUMERIC_SUPERSCRIPT = /]*>\s*(\d{1,3})\s*<\/sup>/gi; +const SUPERSCRIPT_DIGITS = "⁰¹²³⁴⁵⁶⁷⁸⁹"; +const SUBSCRIPT_DIGITS = "₀₁₂₃₄₅₆₇₈₉"; +const METRIC_MARKUP = + /((?]*>\s*(\d{1,3})\s*<\/\2>/gi; +const METRIC_PLAIN_SCRIPT = + /((? { + const table = kind.toLowerCase() === "sup" ? SUPERSCRIPT_DIGITS : SUBSCRIPT_DIGITS; + return `${base}${[...digits].map((digit) => table[Number(digit)]).join("")}`; + }) + .replace( + METRIC_PLAIN_SCRIPT, + (_match, base: string, kind: string, bracedDigits: string, digits: string) => { + const table = kind === "^" ? SUPERSCRIPT_DIGITS : SUBSCRIPT_DIGITS; + return `${base}${[...(bracedDigits || digits)].map((digit) => table[Number(digit)]).join("")}`; + }, + ); +} function stripIndentMarkers(value: string): string { return value @@ -224,7 +245,7 @@ function isDecodableBase64(raw: string): boolean { function pushText(segments: PostBodySegment[], raw: string, indentUnit: number): void { const text = stripHtmlTags( - raw + normalizeMetricMarkup(raw) .replace(FOOTNOTE_BLOCK_OPEN, FOOTNOTE_BLOCK_MARKER) .replace(NUMERIC_SUPERSCRIPT, `${NUMERIC_FOOTNOTE_MARKER}$1`), ); @@ -257,7 +278,7 @@ function markdownCells(line: string): string[] | null { const value = line.trim().replace(/^\|/, "").replace(/(? cell.trim().replace(/\\\|/g, "|")); - return cells.length >= 2 && cells.every(Boolean) ? cells : null; + return cells.length >= 2 && cells.every(Boolean) ? cells.map(normalizeMetricMarkup) : null; } function isMarkdownSeparatorRow(cells: string[] | null): boolean { diff --git a/lineageweave/chunking.py b/lineageweave/chunking.py index 541d18deb..7a8bbc28b 100644 --- a/lineageweave/chunking.py +++ b/lineageweave/chunking.py @@ -134,6 +134,7 @@ def _is_footnote_reference(attrs: list[tuple[str, str | None]]) -> bool: def normalize_semantic_text(text: str) -> str: """Remove visual hanging-indent breaks without changing source content.""" + text = _normalize_metric_markup(_normalize_plain_metric_scripts(text)) lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n") normalized: list[str] = [] for line in lines: @@ -296,6 +297,37 @@ def chunk_by_paragraph(text: str) -> list[Chunk]: _SENTENCE_BOUNDARY = re.compile(r"(?<=[.!?])\s+(?=[A-Z0-9가-힣])") +_SUPERSCRIPT_DIGITS = str.maketrans("0123456789", "⁰¹²³⁴⁵⁶⁷⁸⁹") +_SUBSCRIPT_DIGITS = str.maketrans("0123456789", "₀₁₂₃₄₅₆₇₈₉") +_METRIC_MARKUP = re.compile( + r"(?P(?sup|sub)\b[^>]*>\s*(?P\d{1,3})\s*", + re.IGNORECASE, +) +_METRIC_PLAIN_SCRIPT = re.compile( + r"(?P(?\^|_)\s*(?:\{(?P\d{1,3})\}|(?P\d{1,3}))", + re.IGNORECASE, +) + + +def _normalize_plain_metric_scripts(text: str) -> str: + """Normalize bounded plain-text metric exponents and indices.""" + def replace(match: re.Match[str]) -> str: + table = _SUPERSCRIPT_DIGITS if match.group("kind") == "^" else _SUBSCRIPT_DIGITS + digits = match.group("braced_digits") or match.group("digits") or "" + return f"{match.group('base')}{digits.translate(table)}" + + return _METRIC_PLAIN_SCRIPT.sub(replace, text) + + +def _normalize_metric_markup(html: str) -> str: + """Keep explicit metric superscript/subscript digits in semantic text.""" + def replace(match: re.Match[str]) -> str: + table = _SUPERSCRIPT_DIGITS if match.group("kind").lower() == "sup" else _SUBSCRIPT_DIGITS + return f"{match.group('base')}{match.group('digits').translate(table)}" + + return _METRIC_MARKUP.sub(replace, html) def chunk_by_sentence(text: str) -> list[Chunk]: @@ -554,8 +586,7 @@ def _markdown_cells(line: str) -> list[str] | None: if "|" not in line: return None value = line.strip() - if value.startswith("|"): - value = value[1:] + value = value.removeprefix("|") if value.endswith("|") and not value.endswith("\\|"): value = value[:-1] cells = [cell.strip().replace(r"\|", "|") for cell in re.split(r"(? None: found_table = True flush_pending() - entries.append(("markdown_tr", " | ".join(header))) + entries.append(("markdown_tr", " | ".join(normalize_semantic_text(cell) for cell in header))) index += 2 while index < len(lines) and lines[index].strip(): cells = _markdown_cells(lines[index]) if cells is None: break - entries.append(("markdown_tr", " | ".join(cells))) + entries.append(("markdown_tr", " | ".join(normalize_semantic_text(cell) for cell in cells))) index += 1 flush_pending() @@ -616,7 +647,10 @@ def _is_markdown_table_row(line: str) -> bool: def _render_markdown_table_row(line: str) -> str: """Keep Markdown table columns as searchable row evidence.""" - return " | ".join(cell.strip() for cell in line.strip().strip("|").split("|")) + return " | ".join( + normalize_semantic_text(cell.strip()) + for cell in line.strip().strip("|").split("|") + ) def _split_plain_text_units(text: str) -> list[tuple[str, int, str]]: @@ -695,7 +729,7 @@ def chunk_by_dom(html: str) -> list[Chunk]: ] parser = _BlockTextExtractor() - parser.feed(html) + parser.feed(_normalize_metric_markup(html)) entries = parser.finished() chunks: list[Chunk] = [] for index, ( diff --git a/tests/test_chunking.py b/tests/test_chunking.py index 55c3839eb..258bc9ad4 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -191,6 +191,45 @@ def test_chunk_by_dom_does_not_treat_non_numeric_superscript_as_footnote() -> No ] +def test_chunk_by_dom_preserves_explicit_metric_superscripts() -> None: + """A unit exponent remains searchable mathematical evidence.""" + chunks = chunk_by_dom("

Volume: 5m3.

") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("p", "Volume: 5m³."), + ] + + +def test_chunk_by_dom_preserves_explicit_metric_subscripts() -> None: + """A unit subscript is retained without changing ordinary footnotes.""" + chunks = chunk_by_dom("

Index m3 is measured.

") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("p", "Index m₃ is measured."), + ] + + +def test_chunk_by_source_body_normalizes_plain_metric_scripts() -> None: + """Plain-text metric scripts retain searchable exponent/index semantics.""" + chunks = chunk_by_source_body("Volume: 5m^3; index m_3; braced m^{2}.") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("", "Volume: 5m³; index m₃; braced m²."), + ] + + +def test_chunk_by_source_body_normalizes_metric_scripts_in_markdown_table_cells() -> None: + """Markdown table cells retain the same searchable metric semantics as prose.""" + chunks = chunk_by_source_body( + "| Metric | Index |\n| --- | --- |\n| 5m^3 | m_3 |" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("tr", "Metric | Index"), + ("tr", "5m³ | m₃"), + ] + + def test_chunk_by_dom_preserves_nested_list_order_and_depth() -> None: """Nested list items retain source order and increasing depth.""" chunks = chunk_by_dom(