diff --git a/CHANGELOG.md b/CHANGELOG.md index 93c3e51c0..33d3a0eeb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,11 @@ All notable changes to this project are documented here. Format follows semantic-unit parser and buyer body renderer. See the [product and technical gap baseline](docs/product-technical-gap-baseline.md) and [ADR 0103](docs/adr/0103-semantic-document-evidence-contract.md). +- Preserve multiline VISION table rows, render parent and region OCR tables + accessibly, and request source-visible entity, relationship, layout, and + document-purpose evidence instead of a generic image caption. VISION calls + now share the structure channel's 600-second deep-agent runtime boundary; + an empty same-image retry can no longer erase previously observed OCR. - `make smoke` and `make seed` now run through the locked project `uv` environment, so local OIDC and synthetic-data workflows resolve the same diff --git a/docker/contextual-orchestrator/start.py b/docker/contextual-orchestrator/start.py index e35c9a436..9ceef0ad2 100644 --- a/docker/contextual-orchestrator/start.py +++ b/docker/contextual-orchestrator/start.py @@ -14,9 +14,11 @@ def _pop_first_env(*names: str) -> str: - """Read the first configured alias without leaving credentials in the environment.""" + """Read the first alias, removing quotes preserved by Docker env files.""" for name in names: value = os.environ.pop(name, "").strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}: + value = value[1:-1] if value: return value return "" @@ -27,7 +29,7 @@ def main() -> None: provider_key = _pop_first_env("LLM_GATEWAY_API_KEY", "LLM_API_KEY", "NVIDIA_NIM_API_KEY") if not provider_key: raise SystemExit("LLM_GATEWAY_API_KEY or LLM_API_KEY is required to start the real LLM service") - auth_token = os.environ.get("CONTEXTUAL_ORCHESTRATOR_TOKEN", "").strip() + auth_token = _pop_first_env("CONTEXTUAL_ORCHESTRATOR_TOKEN") if not auth_token: raise SystemExit("CONTEXTUAL_ORCHESTRATOR_TOKEN is required to start the authenticated LLM service") @@ -36,14 +38,14 @@ def main() -> None: raise SystemExit("LLM_GATEWAY_API_URL or LLM_GATEWAY_URL is required to start the gateway") if not provider_url.rstrip("/").endswith("/v1"): provider_url = provider_url.rstrip("/") + "/v1" - raw_limit = os.environ.pop("LLM_GATEWAY_MAX_OUTPUT_TOKENS", "4096").strip() + raw_limit = _pop_first_env("LLM_GATEWAY_MAX_OUTPUT_TOKENS") or "4096" try: max_output_tokens = int(raw_limit) except ValueError as exc: raise SystemExit("LLM_GATEWAY_MAX_OUTPUT_TOKENS must be an integer") from exc if not 64 <= max_output_tokens <= 4096: raise SystemExit("LLM_GATEWAY_MAX_OUTPUT_TOKENS must be between 64 and 4096") - raw_body_limit = os.environ.pop("CONTEXTUAL_ORCHESTRATOR_MAX_BODY_BYTES", str(8 * 1024 * 1024)).strip() + raw_body_limit = _pop_first_env("CONTEXTUAL_ORCHESTRATOR_MAX_BODY_BYTES") or str(8 * 1024 * 1024) try: max_body_bytes = int(raw_body_limit) except ValueError as exc: @@ -56,7 +58,7 @@ def main() -> None: agent["base_url"] = provider_url agent["credential_key"] = "LLM_GATEWAY_API_KEY" agent.setdefault("provider_protocol", "auto") - embedding_model = os.environ.get("LLM_GATEWAY_EMBEDDING_MODEL", "").strip() + embedding_model = _pop_first_env("LLM_GATEWAY_EMBEDDING_MODEL") if embedding_model: embedding_agents = [ agent diff --git a/docs/adr/0103-semantic-document-evidence-contract.md b/docs/adr/0103-semantic-document-evidence-contract.md index faca30d23..ea65bc111 100644 --- a/docs/adr/0103-semantic-document-evidence-contract.md +++ b/docs/adr/0103-semantic-document-evidence-contract.md @@ -36,6 +36,18 @@ recognizable header/separator/data shape and otherwise preserves plain text. 5. Keep the frontend's raw-source fallback aligned with the persisted unit labels. Persisted row units render as accessible tables; unresolved structure remains visibly unresolved and actionable. +6. Apply the same narrow Markdown-table renderer to persisted image OCR. + VISION output may use multiple `TEXT` lines so row boundaries survive; its + caption names only visible entities, relationships, layout, and document + purpose rather than offering a generic one-sentence description. The + client allows 600 seconds for deep orchestrator work; a 180-second local + cutoff already terminated a valid live response before delivery. +7. Serialize replacement per source post and reject a same-image retry when + its content hash matches non-empty persisted OCR but the retry returns no + OCR. Provider completion is transport evidence, not permission to erase a + stronger prior observation. During an operator backfill, this typed + preservation failure skips only the affected post, records it in the + aggregate result, and allows the remaining selected posts to continue. ## Rejected alternatives @@ -54,6 +66,10 @@ recognizable header/separator/data shape and otherwise preserves plain text. markup or image base64. - The database keeps the existing normalized unit tables; this decision adds no denormalized JSON field or new service. +- A weaker same-image VISION retry fails before replacement, leaving the + prior committed evidence available for a later orchestrator retry. +- A protected retry does not abort an entire operator batch; the skipped-post + count is visible to the operator without exposing raw post content. - Markdown dialects outside the narrow recognized shape remain plain text and are reported as a future parser extension rather than guessed. @@ -62,4 +78,5 @@ recognizable header/separator/data shape and otherwise preserves plain text. The baseline's synthetic tests cover numeric superscript footnotes, marker footnotes, nested `ol`/`ul`/`oi` order and depth, HTML/OOXML rows, Markdown rows, React table rendering, and unresolved indentation. Full CI remains the -release gate. +release gate. A persistence regression test proves that an empty same-hash +VISION retry cannot delete previously observed OCR. diff --git a/docs/adr/0105-mathematical-script-semantic-normalization.md b/docs/adr/0105-mathematical-script-semantic-normalization.md new file mode 100644 index 000000000..304fc52f8 --- /dev/null +++ b/docs/adr/0105-mathematical-script-semantic-normalization.md @@ -0,0 +1,50 @@ +# ADR 0105: Preserve explicit metric scripts in semantic text + +**Status:** Accepted on this PR; not protected-main truth +**Date:** 2026-08-21 +**Owners:** LineageWeave ingestion and buyer-surface maintainers + +## Context + +Source posts commonly encode a unit such as `m3`, `m3`, +`m^3`, or `m_3` with HTML or plain-text notation. Dropping the markup changes +the searchable meaning to `m3`, while treating every numeric `sup` element as +mathematics would break the existing numeric-footnote contract. Full MathML +parsing is not yet justified by the current product surface, but the loss of +explicit unit scripts is a buyer-visible defect. + +MathML 4 defines `msup`, `msub`, and `msubsup` as structural script elements; +HTML `sup`/`sub` are a permitted lighter-weight notation when detailed +mathematical markup is not required. This decision therefore adds a bounded +normalization boundary and keeps the source representation unchanged. + +## Decision + +1. Preserve the immutable source body exactly as imported. +2. In derived semantic text only, normalize an explicitly bounded metric base + (`m`, `cm`, `mm`, `km`, or `kg`, optionally preceded by a number) followed + by numeric `sup`/`sub` markup or plain-text `^`/`_` notation into Unicode + superscript/subscript digits. For example, `5m3` and `5m^3` + become `5m³`, while `m3` and `m_3` become `m₃`. +3. Keep ordinary numeric superscripts and caret expressions on prose under the existing footnote + role contract. Do not infer a mathematical formula from an arbitrary word. +4. Apply the same bounded normalization in backend semantic chunks and the + React buyer display so search text and visible text agree. +5. Defer full MathML/LaTeX parsing, expression trees, and ontology term + creation until an authorized fixture demonstrates a need beyond metric + scripts. Any such change requires a new ADR and parser contract. + +## Consequences + +- Search and the buyer popup retain the visible distinction between `m³` and + `m3` without exposing source HTML to the embedding model. +- Existing numeric-footnote tests remain unchanged because the bounded metric + pattern is the only new conversion. +- The current implementation does not claim to understand arbitrary equations; + unsupported script markup remains ordinary source text and must not be + presented as a parsed ontology expression. + +## References (APA 7th) + +World Wide Web Consortium. (2026). *Mathematical Markup Language (MathML) +Version 4.0* (W3C Recommendation). https://www.w3.org/TR/mathml4/ diff --git a/frontend/src/PostBody.stories.tsx b/frontend/src/PostBody.stories.tsx index 43260ec2d..358758cb0 100644 --- a/frontend/src/PostBody.stories.tsx +++ b/frontend/src/PostBody.stories.tsx @@ -10,6 +10,9 @@ export default meta; type Story = StoryObj; +const TINY_PNG = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="; + export const MarkdownTableEvidence: Story = { args: { body: "| Workstream | State |\n| --- | --- |\n| Alpha | Ready |", @@ -44,6 +47,22 @@ export const MarkdownTableFallback: Story = { }, }; +export const ImageOcrTableEvidence: Story = { + args: { + body: ``, + imageContent: [ + { + unit_index: 0, + mime_type: "image/png", + status_code: "completed", + extracted_text: "| Workstream | State |\n| --- | --- |\n| Alpha | Ready |", + caption: "A synthetic workstream status table.", + tags: ["table"], + }, + ], + }, +}; + export const NumericFootnote: Story = { args: { body: "

Evidence remains attached to the source.

1 Source note.

", diff --git a/frontend/src/PostBody.test.tsx b/frontend/src/PostBody.test.tsx index b92083608..1e4c915af 100644 --- a/frontend/src/PostBody.test.tsx +++ b/frontend/src/PostBody.test.tsx @@ -279,6 +279,43 @@ describe("PostBody", () => { expect(screen.getAllByRole("row")).toHaveLength(2); }); + it("renders table-shaped image OCR as accessible evidence", () => { + render( + '} + imageContent={[ + { + unit_index: 0, + mime_type: "image/png", + status_code: "completed", + extracted_text: "| Project | Status |\n| --- | --- |\n| Alpha | Ready |", + caption: "A synthetic project status table.", + tags: ["table"], + regions: [ + { + region_index: 0, + x_ratio: 0, + y_ratio: 0, + width_ratio: 1, + height_ratio: 1, + status_code: "completed", + extracted_text: "| Owner | Action |\n| --- | --- |\n| Team A | Review |", + caption: "The table region assigns an action to a team.", + tags: ["assignment"], + }, + ], + }, + ]} + />, + ); + + expect(screen.getAllByRole("table")).toHaveLength(2); + expect(screen.getByRole("columnheader", { name: "Project" })).toBeInTheDocument(); + expect(screen.getByText("Ready")).toBeInTheDocument(); + expect(screen.getByText("The table region assigns an action to a team.")).toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: "Owner" })).toBeInTheDocument(); + }); + it("marks persisted footnotes as footnote evidence", () => { render( { expect(screen.getByText("Panel")).toBeInTheDocument(); }); + it("keeps escaped pipe characters inside image OCR table cells", () => { + render( + '} + imageContent={[ + { + unit_index: 0, + mime_type: "image/png", + status_code: "described", + extracted_text: "| Item | State |\n| --- | --- |\n| Review \\| approve | Ready |", + caption: "A table image with an escaped separator.", + tags: [], + }, + ]} + />, + ); + + expect(screen.getByRole("table")).toBeInTheDocument(); + expect(screen.getByText("Review | approve")).toBeInTheDocument(); + expect(screen.getByText("Ready")).toBeInTheDocument(); + }); + it("keeps source-image placement while showing persisted OCR and caption evidence", () => { render( { - const cells = row.split("|").map((cell) => cell.trim()); + const cells = row.split(/(? cell.trim().replace(/\\\|/g, "|")); if (cells[0] === "") cells.shift(); if (cells[cells.length - 1] === "") cells.pop(); return cells; @@ -27,10 +27,20 @@ function parsePipeDelimitedTable(text: string): string[][] | null { function renderImageText(text: string) { const rows = parsePipeDelimitedTable(text); if (!rows) return

{text}

; + const [header, ...bodyRows] = rows; return ( + + + {header.map((cell, cellIndex) => ( + + ))} + + - {rows.map((row, rowIndex) => ( + {bodyRows.map((row, rowIndex) => ( {row.map((cell, cellIndex) => ( @@ -77,7 +87,14 @@ function renderImageEvidence(
    {imageContent.regions.map((region) => (
  1. - {region.caption || region.extracted_text || t("Unknown")} + {region.caption ?

    {region.caption}

    : null} + {region.extracted_text ? ( +
    + {renderImageText(region.extracted_text)} +
    + ) : region.caption ? null : ( + t("Unknown") + )} {region.tags.length ? ( {t("Image tags")}: {region.tags.join(", ")} diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts index 666d4474e..e7e47cf90 100644 --- a/frontend/src/postBodyDisplay.test.ts +++ b/frontend/src/postBodyDisplay.test.ts @@ -67,6 +67,18 @@ describe("splitPostBody", () => { ]); }); + it("preserves explicit metric superscripts and subscripts", () => { + expect(splitPostBody("

    Volume: 5m3, index m3.

    ")).toEqual([ + { kind: "text", text: "Volume: 5m³, index m₃." }, + ]); + }); + + it("normalizes plain-text metric superscripts and subscripts", () => { + expect(splitPostBody("

    Volume: 5m^3, index m_3, braced m^{2}.

    ")).toEqual([ + { kind: "text", text: "Volume: 5m³, index m₃, braced m²." }, + ]); + }); + it("preserves HTML and Word footnote blocks as footnote paragraphs", () => { expect( splitPostBody( @@ -128,6 +140,21 @@ describe("splitPostBody", () => { ]); }); + it("normalizes metric scripts inside Markdown table cells", () => { + expect( + splitMarkdownTableBody("| Metric | Index |\n| --- | --- |\n| 5m^3 | m3 |"), + ).toEqual([{ kind: "table", rows: [["Metric", "Index"], ["5m³", "m₃"]] }]); + }); + + it("unescapes pipe characters inside Markdown cells without accepting a short delimiter", () => { + expect( + splitMarkdownTableBody( + "| Project | Notes |\n| :--- | ---: |\n| Alpha | Ready \\| review |", + ), + ).toEqual([{ kind: "table", rows: [["Project", "Notes"], ["Alpha", "Ready | review"]] }]); + expect(splitMarkdownTableBody("| Project | Status |\n| -- | -- |\n| Alpha | Ready |")).toBeNull(); + }); + it("leaves a plain-text post unchanged so existing popups keep their wording", () => { expect(splitPostBody("The full body text.")).toEqual([ { kind: "text", text: "The full body text." }, diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts index b08b4803e..588901483 100644 --- a/frontend/src/postBodyDisplay.ts +++ b/frontend/src/postBodyDisplay.ts @@ -32,6 +32,27 @@ const NUMERIC_FOOTNOTE_MARKER = "\u0003lw-numeric-footnote\u0004"; const FOOTNOTE_BLOCK_MARKER = "\u0005lw-footnote-block\u0006"; const FOOTNOTE_BLOCK_OPEN = /<\s*(?:footnote|endnote|w:footnote|w:endnote)\b[^>]*>/gi; const NUMERIC_SUPERSCRIPT = /]*>\s*(\d{1,3})\s*<\/sup>/gi; +const SUPERSCRIPT_DIGITS = "⁰¹²³⁴⁵⁶⁷⁸⁹"; +const SUBSCRIPT_DIGITS = "₀₁₂₃₄₅₆₇₈₉"; +const METRIC_MARKUP = + /((?]*>\s*(\d{1,3})\s*<\/\2>/gi; +const METRIC_PLAIN_SCRIPT = + /((? { + const table = kind.toLowerCase() === "sup" ? SUPERSCRIPT_DIGITS : SUBSCRIPT_DIGITS; + return `${base}${[...digits].map((digit) => table[Number(digit)]).join("")}`; + }) + .replace( + METRIC_PLAIN_SCRIPT, + (_match, base: string, kind: string, bracedDigits: string, digits: string) => { + const table = kind === "^" ? SUPERSCRIPT_DIGITS : SUBSCRIPT_DIGITS; + return `${base}${[...(bracedDigits || digits)].map((digit) => table[Number(digit)]).join("")}`; + }, + ); +} function stripIndentMarkers(value: string): string { return value @@ -224,7 +245,7 @@ function isDecodableBase64(raw: string): boolean { function pushText(segments: PostBodySegment[], raw: string, indentUnit: number): void { const text = stripHtmlTags( - raw + normalizeMetricMarkup(raw) .replace(FOOTNOTE_BLOCK_OPEN, FOOTNOTE_BLOCK_MARKER) .replace(NUMERIC_SUPERSCRIPT, `${NUMERIC_FOOTNOTE_MARKER}$1`), ); @@ -257,7 +278,7 @@ function markdownCells(line: string): string[] | null { const value = line.trim().replace(/^\|/, "").replace(/(? cell.trim().replace(/\\\|/g, "|")); - return cells.length >= 2 && cells.every(Boolean) ? cells : null; + return cells.length >= 2 && cells.every(Boolean) ? cells.map(normalizeMetricMarkup) : null; } function isMarkdownSeparatorRow(cells: string[] | null): boolean { diff --git a/lineageweave/chunking.py b/lineageweave/chunking.py index 541d18deb..7a8bbc28b 100644 --- a/lineageweave/chunking.py +++ b/lineageweave/chunking.py @@ -134,6 +134,7 @@ def _is_footnote_reference(attrs: list[tuple[str, str | None]]) -> bool: def normalize_semantic_text(text: str) -> str: """Remove visual hanging-indent breaks without changing source content.""" + text = _normalize_metric_markup(_normalize_plain_metric_scripts(text)) lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n") normalized: list[str] = [] for line in lines: @@ -296,6 +297,37 @@ def chunk_by_paragraph(text: str) -> list[Chunk]: _SENTENCE_BOUNDARY = re.compile(r"(?<=[.!?])\s+(?=[A-Z0-9가-힣])") +_SUPERSCRIPT_DIGITS = str.maketrans("0123456789", "⁰¹²³⁴⁵⁶⁷⁸⁹") +_SUBSCRIPT_DIGITS = str.maketrans("0123456789", "₀₁₂₃₄₅₆₇₈₉") +_METRIC_MARKUP = re.compile( + r"(?P(?sup|sub)\b[^>]*>\s*(?P\d{1,3})\s*", + re.IGNORECASE, +) +_METRIC_PLAIN_SCRIPT = re.compile( + r"(?P(?\^|_)\s*(?:\{(?P\d{1,3})\}|(?P\d{1,3}))", + re.IGNORECASE, +) + + +def _normalize_plain_metric_scripts(text: str) -> str: + """Normalize bounded plain-text metric exponents and indices.""" + def replace(match: re.Match[str]) -> str: + table = _SUPERSCRIPT_DIGITS if match.group("kind") == "^" else _SUBSCRIPT_DIGITS + digits = match.group("braced_digits") or match.group("digits") or "" + return f"{match.group('base')}{digits.translate(table)}" + + return _METRIC_PLAIN_SCRIPT.sub(replace, text) + + +def _normalize_metric_markup(html: str) -> str: + """Keep explicit metric superscript/subscript digits in semantic text.""" + def replace(match: re.Match[str]) -> str: + table = _SUPERSCRIPT_DIGITS if match.group("kind").lower() == "sup" else _SUBSCRIPT_DIGITS + return f"{match.group('base')}{match.group('digits').translate(table)}" + + return _METRIC_MARKUP.sub(replace, html) def chunk_by_sentence(text: str) -> list[Chunk]: @@ -554,8 +586,7 @@ def _markdown_cells(line: str) -> list[str] | None: if "|" not in line: return None value = line.strip() - if value.startswith("|"): - value = value[1:] + value = value.removeprefix("|") if value.endswith("|") and not value.endswith("\\|"): value = value[:-1] cells = [cell.strip().replace(r"\|", "|") for cell in re.split(r"(? None: found_table = True flush_pending() - entries.append(("markdown_tr", " | ".join(header))) + entries.append(("markdown_tr", " | ".join(normalize_semantic_text(cell) for cell in header))) index += 2 while index < len(lines) and lines[index].strip(): cells = _markdown_cells(lines[index]) if cells is None: break - entries.append(("markdown_tr", " | ".join(cells))) + entries.append(("markdown_tr", " | ".join(normalize_semantic_text(cell) for cell in cells))) index += 1 flush_pending() @@ -616,7 +647,10 @@ def _is_markdown_table_row(line: str) -> bool: def _render_markdown_table_row(line: str) -> str: """Keep Markdown table columns as searchable row evidence.""" - return " | ".join(cell.strip() for cell in line.strip().strip("|").split("|")) + return " | ".join( + normalize_semantic_text(cell.strip()) + for cell in line.strip().strip("|").split("|") + ) def _split_plain_text_units(text: str) -> list[tuple[str, int, str]]: @@ -695,7 +729,7 @@ def chunk_by_dom(html: str) -> list[Chunk]: ] parser = _BlockTextExtractor() - parser.feed(html) + parser.feed(_normalize_metric_markup(html)) entries = parser.finished() chunks: list[Chunk] = [] for index, ( diff --git a/lineageweave/image_content.py b/lineageweave/image_content.py index f6500ffd3..a0d298a91 100644 --- a/lineageweave/image_content.py +++ b/lineageweave/image_content.py @@ -141,7 +141,8 @@ class ImageDescription: Attributes: extracted_text: OCR result -- every piece of legible text found in the image, empty string if none. - caption: one-sentence description of what the image shows. + caption: factual description of the visible entities, relationships, + and layout that make the image useful as semantic evidence. tags: short tags for the main objects/subjects, for independent keyword search separate from the free-text caption. """ @@ -177,15 +178,18 @@ def describe(self, image_bytes: bytes, mime_type: str) -> ImageDescription: # p _RESPONSE_FORMAT = ( - "Examine this image. Reply with EXACTLY three lines, no extra commentary:\n" + "Examine this image. Reply with exactly the three labeled sections below and no " + "extra commentary. TEXT may span multiple lines; CAPTION and TAGS stay on their " + "labeled lines.\n" "TEXT: \n" + "If the image contains a table, preserve its row/column structure as a Markdown " + "pipe table: one row per line, with a separator row immediately after the visible " + "header. Never flatten a table into an unstructured word list or invent a header " + "that is not visible.>\n" "CAPTION: <2-4 concise, evidence-grounded sentences describing the visible layout, " "objects, relationships, directions, measurements, and labels; do not guess " - "anything that is not visible>\n" - "TAGS: " + "anything that is not visible. Omit anything the pixels do not support.>\n" + "TAGS: " ) _REGION_RESPONSE_FORMAT = ( "Find distinct meaningful visual regions in this image for separate OCR and description. " @@ -251,14 +255,15 @@ def _parse_description(content: str) -> ImageDescription: remainder = _strip_outer_markdown_emphasis(match.group(2)) if remainder: fields[label].append(remainder) - multiline_field = "TEXT" if label == "TEXT" else None + multiline_field = label if label in {"TEXT", "CAPTION"} else None continue - if re.match(r"^\s*[*_`>#\-\s]*[A-Za-z][A-Za-z0-9 _-]*\s*:", line): - multiline_field = None - continue - if multiline_field == "TEXT" and line.strip(): - fields["TEXT"].append(_strip_outer_markdown_emphasis(line)) + if multiline_field in {"TEXT", "CAPTION"} and line.strip(): + # A colon is common inside OCR (for example ``Date: 2026-08-21``). + # Only the known response labels above end the active section; + # treating every colon as a provider label loses real image text + # or the continuation of a detailed caption. + fields[multiline_field].append(_strip_outer_markdown_emphasis(line)) if not fields["TEXT"] and not fields["CAPTION"]: raise ImageDescriptionParseError("vision response had no usable TEXT or CAPTION content") @@ -289,7 +294,7 @@ def __init__( api_key: str, model: str | None = None, *, - timeout: float = 180.0, + timeout: float = 600.0, allow_insecure_http: bool = False, ) -> None: parsed = urlparse(base_url) diff --git a/lineageweave/post_content_persistence.py b/lineageweave/post_content_persistence.py index 86dcfd4de..44d7da8bf 100644 --- a/lineageweave/post_content_persistence.py +++ b/lineageweave/post_content_persistence.py @@ -30,6 +30,10 @@ _LOGGER = logging.getLogger(__name__) +class ImageOcrPreservationError(RuntimeError): + """A retry would erase stronger OCR already persisted for the same image.""" + + def _bounded_unit_batches( # noqa: UP047 - retain Python 3.10 compatibility. units: list[tuple[_BatchKey, str]], ) -> list[list[tuple[_BatchKey, str]]]: @@ -84,12 +88,24 @@ async def persist_post_content( Provider calls happen before the short database transaction. A failed or unavailable embedding call writes no vector row; it never writes a zero or - guessed vector. The raw body remains in ``source_post`` for future retry. + guessed vector. A same-image retry cannot replace non-empty persisted OCR + with an empty result. The raw body remains in ``source_post`` for future + retry. """ normalized = normalized_result or normalize_post_body(body, vision_client) chunks = chunk_by_source_body(body) image_results = {result.chunk_index: result for result in normalized.image_results} formatting = {hint.chunk_index: hint.style for hint in normalized.formatting_hints} + image_ocr_by_sha256: dict[str, bool] = {} + for chunk in chunks: + if chunk.unit_type != "image" or chunk.image_data is None: + continue + result = image_results.get(chunk.index) + description = result.description if result else None + content_sha256 = hashlib.sha256(chunk.image_data).hexdigest() + image_ocr_by_sha256[content_sha256] = image_ocr_by_sha256.get( + content_sha256, False + ) or bool(description and description.extracted_text.strip()) prepared: list[tuple[Chunk, str, str | None]] = [] for chunk in chunks: @@ -226,6 +242,29 @@ async def persist_post_content( ) async with conn.transaction(): + if image_ocr_by_sha256: + await conn.fetchval( + "select post_id from source_post where post_id = $1 for update", + post_id, + ) + previous_images = await conn.fetch( + """ + select image.content_sha256, image.extracted_text + from post_content_unit unit + join post_content_image image using (post_content_unit_id) + where unit.post_id = $1 + and nullif(btrim(image.extracted_text), '') is not null + """, + post_id, + ) + if any( + row["content_sha256"] in image_ocr_by_sha256 + and not image_ocr_by_sha256[row["content_sha256"]] + for row in previous_images + ): + raise ImageOcrPreservationError( + "refusing to replace non-empty image OCR with an empty retry result" + ) await conn.execute("delete from post_content_unit where post_id = $1", post_id) unit_ids: dict[int, str] = {} for chunk, unit_text, style in prepared: diff --git a/scripts/backfill_post_content.py b/scripts/backfill_post_content.py index ef54c91a7..1263f25b8 100644 --- a/scripts/backfill_post_content.py +++ b/scripts/backfill_post_content.py @@ -27,7 +27,10 @@ from lineageweave.image_content import NullImageContentClient, orchestrator_vision_client from lineageweave.llm_context import build_post_llm_metadata, use_llm_metadata from lineageweave.post_content_normalization import normalize_post_body -from lineageweave.post_content_persistence import persist_post_content +from lineageweave.post_content_persistence import ( + ImageOcrPreservationError, + persist_post_content, +) from lineageweave.post_structure import ContextualOrchestratorPostStructureClient, NullPostStructureClient @@ -55,6 +58,15 @@ def _parser() -> argparse.ArgumentParser: return parser +async def _ensure_open_connection( + conn: asyncpg.Connection, target_dsn: str +) -> asyncpg.Connection: + """Reconnect after a database restart without repeating VISION work.""" + if not conn.is_closed(): + return conn + return await asyncpg.connect(target_dsn) + + async def backfill_post_content( target_dsn: str, raw_post_ids: list[str] | None, @@ -217,17 +229,22 @@ async def backfill_post_content( if described_images == 0 and not normalized.text.strip(): result["skipped_posts"] += 1 continue - await persist_post_content( - conn, - str(row["post_id"]), - row["post_body"], - vision_client=vision_client, - embedding_client=embedding_client, - embedding_model_code=embedding_model or None, - normalized_result=normalized, - structure_client=structure_client, - post_title=row["post_title"], - ) + conn = await _ensure_open_connection(conn, target_dsn) + try: + await persist_post_content( + conn, + str(row["post_id"]), + row["post_body"], + vision_client=vision_client, + embedding_client=embedding_client, + embedding_model_code=embedding_model or None, + normalized_result=normalized, + structure_client=structure_client, + post_title=row["post_title"], + ) + except ImageOcrPreservationError: + result["skipped_posts"] += 1 + continue async with conn.transaction(): await record_post_content_backfill_success( conn, diff --git a/tests/test_backfill_post_content.py b/tests/test_backfill_post_content.py new file mode 100644 index 000000000..d5f034297 --- /dev/null +++ b/tests/test_backfill_post_content.py @@ -0,0 +1,135 @@ +"""Operator backfill connection recovery contracts.""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace + +from scripts import backfill_post_content +from lineageweave.post_content_persistence import ImageOcrPreservationError + + +def test_reconnects_only_after_database_connection_closes(monkeypatch) -> None: + replacement_connection = object() + connected_dsns: list[str] = [] + + class Connection: + def __init__(self, closed: bool) -> None: + self._closed = closed + + def is_closed(self) -> bool: + return self._closed + + async def connect(dsn: str): + connected_dsns.append(dsn) + return replacement_connection + + monkeypatch.setattr(backfill_post_content.asyncpg, "connect", connect) + + current_connection = Connection(False) + assert ( + asyncio.run(backfill_post_content._ensure_open_connection(current_connection, "dsn")) + is current_connection + ) + assert asyncio.run(backfill_post_content._ensure_open_connection(Connection(True), "dsn")) is replacement_connection + assert connected_dsns == ["dsn"] + + +def test_backfill_skips_ocr_protected_post_and_continues(monkeypatch) -> None: + post_ids = [ + "00505695-0000-1fd1-8000-000000000001", + "00505695-0000-1fd1-8000-000000000002", + ] + + class Transaction: + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, traceback): + return False + + class Connection: + def is_closed(self) -> bool: + return False + + async def fetch(self, query, *args): + return [{"post_id": post_id} for post_id in post_ids] + + async def fetchrow(self, query, post_id): + return { + "post_id": post_id, + "post_title": "Synthetic title", + "post_body": "Synthetic body", + "author_account_id": None, + "source_process_unit_code": None, + "source_author_code": None, + "source_company_code": None, + "source_customer_code": None, + "source_project_code": None, + "source_sales_pool_code": None, + "corporate_entity_code": None, + } + + def transaction(self): + return Transaction() + + async def fetchval(self, query, *args): + return 0 + + async def close(self): + return None + + connection = Connection() + persisted: list[str] = [] + + async def connect(_dsn): + return connection + + async def persist(conn, post_id, body, **kwargs): + persisted.append(post_id) + if post_id == post_ids[0]: + raise ImageOcrPreservationError("protected") + return 1 + + async def record_success(conn, post_id, body): + return None + + class MetadataContext: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, traceback): + return False + + monkeypatch.setattr(backfill_post_content.asyncpg, "connect", connect) + monkeypatch.setattr(backfill_post_content, "persist_post_content", persist) + monkeypatch.setattr( + backfill_post_content, + "record_post_content_backfill_success", + record_success, + ) + monkeypatch.setattr( + backfill_post_content, + "normalize_post_body", + lambda body, vision_client: SimpleNamespace(image_results=(), text="text"), + ) + monkeypatch.setattr( + backfill_post_content, + "build_post_llm_metadata", + lambda post_id, row: {}, + ) + monkeypatch.setattr( + backfill_post_content, + "use_llm_metadata", + lambda metadata: MetadataContext(), + ) + + result = asyncio.run( + backfill_post_content.backfill_post_content( + "dsn", post_ids, limit=None, normalize_only=True + ) + ) + + assert persisted == post_ids + assert result["processed_posts"] == 1 + assert result["skipped_posts"] == 1 diff --git a/tests/test_chunking.py b/tests/test_chunking.py index 7525746c8..258bc9ad4 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -191,6 +191,45 @@ def test_chunk_by_dom_does_not_treat_non_numeric_superscript_as_footnote() -> No ] +def test_chunk_by_dom_preserves_explicit_metric_superscripts() -> None: + """A unit exponent remains searchable mathematical evidence.""" + chunks = chunk_by_dom("

    Volume: 5m3.

    ") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("p", "Volume: 5m³."), + ] + + +def test_chunk_by_dom_preserves_explicit_metric_subscripts() -> None: + """A unit subscript is retained without changing ordinary footnotes.""" + chunks = chunk_by_dom("

    Index m3 is measured.

    ") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("p", "Index m₃ is measured."), + ] + + +def test_chunk_by_source_body_normalizes_plain_metric_scripts() -> None: + """Plain-text metric scripts retain searchable exponent/index semantics.""" + chunks = chunk_by_source_body("Volume: 5m^3; index m_3; braced m^{2}.") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("", "Volume: 5m³; index m₃; braced m²."), + ] + + +def test_chunk_by_source_body_normalizes_metric_scripts_in_markdown_table_cells() -> None: + """Markdown table cells retain the same searchable metric semantics as prose.""" + chunks = chunk_by_source_body( + "| Metric | Index |\n| --- | --- |\n| 5m^3 | m_3 |" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("tr", "Metric | Index"), + ("tr", "5m³ | m₃"), + ] + + def test_chunk_by_dom_preserves_nested_list_order_and_depth() -> None: """Nested list items retain source order and increasing depth.""" chunks = chunk_by_dom( @@ -221,6 +260,21 @@ def test_chunk_by_dom_keeps_markdown_table_rows_as_searchable_units() -> None: ] +def test_chunk_by_dom_preserves_escaped_markdown_pipes_and_rejects_short_delimiters() -> None: + escaped = chunk_by_dom( + "| Field | Notes |\n| --- | --- |\n| Owner | Ready \\| review |" + ) + assert [(chunk.label, chunk.text) for chunk in escaped] == [ + ("markdown_tr", "Field | Notes"), + ("markdown_tr", "Owner | Ready | review"), + ] + + short_delimiter = chunk_by_dom( + "| Field | Value |\n| -- | -- |\n| Owner | Buyer |" + ) + assert all(chunk.label != "markdown_tr" for chunk in short_delimiter) + + def test_chunk_by_dom_keeps_prose_around_markdown_table_rows() -> None: """Prose surrounding a Markdown table stays in document order.""" chunks = chunk_by_dom( diff --git a/tests/test_contextual_orchestrator_start.py b/tests/test_contextual_orchestrator_start.py index e3618aa82..a4f6bcfad 100644 --- a/tests/test_contextual_orchestrator_start.py +++ b/tests/test_contextual_orchestrator_start.py @@ -52,6 +52,15 @@ def test_gateway_api_key_accepts_local_compatibility_alias(monkeypatch) -> None: assert module._pop_first_env("LLM_GATEWAY_API_KEY", "LLM_API_KEY") == "compatibility-key" +def test_env_file_quotes_are_not_part_of_transport_values(monkeypatch) -> None: + module = _load_start_module() + monkeypatch.setenv("LLM_GATEWAY_API_KEY", "'provider-key'") + monkeypatch.setenv("LLM_GATEWAY_API_URL", '"https://gateway.example/v1"') + + assert module._pop_first_env("LLM_GATEWAY_API_KEY") == "provider-key" + assert module._pop_first_env("LLM_GATEWAY_API_URL") == "https://gateway.example/v1" + + def test_bootstrap_registers_embedding_agent_before_deleting_secrets(monkeypatch) -> None: module = _load_start_module() captured: dict[str, object] = {} @@ -90,9 +99,11 @@ def serve() -> None: monkeypatch.setattr(module, "Path", FakePath) monkeypatch.setattr(sys, "argv", ["start.py"]) monkeypatch.setenv("LLM_GATEWAY_API_KEY", "provider-key") - monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_TOKEN", "orchestrator-token") + monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_TOKEN", "'orchestrator-token'") monkeypatch.setenv("LLM_GATEWAY_API_URL", "https://gateway.example") - monkeypatch.setenv("LLM_GATEWAY_EMBEDDING_MODEL", "embedding-model") + monkeypatch.setenv("LLM_GATEWAY_EMBEDDING_MODEL", "'embedding-model'") + monkeypatch.setenv("LLM_GATEWAY_MAX_OUTPUT_TOKENS", "'2048'") + monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_MAX_BODY_BYTES", '"65536"') module.main() @@ -100,6 +111,9 @@ def serve() -> None: assert isinstance(argv, list) assert "--embedding-provider-url" not in argv assert "--embedding-model" not in argv + assert argv[argv.index("--auth-token") + 1] == "orchestrator-token" + assert argv[argv.index("--max-output-tokens") + 1] == "2048" + assert argv[argv.index("--max-body-bytes") + 1] == "65536" assert captured["credentials"] == [ ("NVIDIA_NIM_API_KEY", "provider-key"), ("LLM_GATEWAY_API_KEY", "provider-key"), diff --git a/tests/test_image_content.py b/tests/test_image_content.py index de7fc49b5..148a979c5 100644 --- a/tests/test_image_content.py +++ b/tests/test_image_content.py @@ -85,6 +85,35 @@ def test_parse_description_preserves_multiline_ocr_text() -> None: assert description.caption == "A scanned page." +def test_parse_description_preserves_multiline_caption_evidence() -> None: + """Detailed VISION captions remain complete when providers wrap lines.""" + content = ( + "CAPTION: A project status table for the customer meeting.\n" + "The left column lists workstreams and the right column lists owners.\n" + "TEXT: Workstream | Owner\nAlpha | Team A\nTAGS: table, assignment" + ) + + description = _parse_description(content) + + assert description.caption == ( + "A project status table for the customer meeting.\n" + "The left column lists workstreams and the right column lists owners." + ) + assert description.extracted_text == "Workstream | Owner\nAlpha | Team A" + + +def test_parse_description_preserves_ocr_lines_that_contain_colons() -> None: + """A colon in a scanned field is OCR content, not a new response field.""" + content = ( + "TEXT: Invoice\nDate: 2026-08-21\nTotal: 100\n" + "CAPTION: A synthetic invoice.\nTAGS: invoice" + ) + + description = _parse_description(content) + + assert description.extracted_text == "Invoice\nDate: 2026-08-21\nTotal: 100" + + def test_parse_description_preserves_table_row_structure_in_ocr_text() -> None: """Live gap (2026-08-19): an image containing a table used to have its text flattened into an unstructured word list on OCR, the same @@ -202,6 +231,14 @@ def test_orchestrator_vision_client_does_not_double_v1() -> None: assert client._base_url == "https://gateway.example/v1" +def test_orchestrator_vision_client_allows_deep_agent_runtime() -> None: + """A valid VISION result must not be cut off by the former 180s limit.""" + client = orchestrator_vision_client("https://gateway.example", "key") + + assert isinstance(client, OpenAiCompatibleVisionClient) + assert client._timeout == 600.0 + + def test_orchestrator_vision_client_is_null_when_unconfigured() -> None: client = orchestrator_vision_client("", "") assert isinstance(client, NullImageContentClient) @@ -226,6 +263,17 @@ def test_ocr_prompt_asks_for_table_row_structure() -> None: assert "table" in _RESPONSE_FORMAT.lower() +def test_ocr_prompt_allows_multiline_tables_and_requests_semantic_detail() -> None: + """Table rows and ontology-ready captions must fit the response contract.""" + prompt = _RESPONSE_FORMAT.lower() + + assert "text may span multiple lines" in prompt + assert "separator row" in prompt + assert "named entities" in prompt + assert "relationships" in prompt + assert "exactly three lines" not in prompt + + def test_region_prompt_requires_full_image_coverage() -> None: """Live gap (2026-08-19): "distinct meaningful visual regions" alone let the model describe only the most visually striking part of an diff --git a/tests/test_post_content_persistence_edges.py b/tests/test_post_content_persistence_edges.py index 24bec9e6d..4eac1b77e 100644 --- a/tests/test_post_content_persistence_edges.py +++ b/tests/test_post_content_persistence_edges.py @@ -2,6 +2,7 @@ import asyncio from contextlib import asynccontextmanager +import hashlib from types import SimpleNamespace import pytest @@ -27,9 +28,10 @@ def _persist(*args: object, **kwargs: object) -> int: class _Connection: - def __init__(self) -> None: + def __init__(self, previous_images: tuple[dict[str, object], ...] = ()) -> None: self.executed: list[tuple[str, tuple[object, ...]]] = [] self.fetchvals: list[tuple[str, tuple[object, ...]]] = [] + self.previous_images = previous_images self._next_id = 0 @asynccontextmanager @@ -45,6 +47,10 @@ async def fetchval(self, query: str, *args: object) -> str: self._next_id += 1 return f"id-{self._next_id}" + async def fetch(self, query: str, *args: object) -> list[dict[str, object]]: + assert "post_content_image" in query + return list(self.previous_images) + class _EmbedMany: available = True @@ -190,7 +196,18 @@ def test_persists_image_tags_formatting_and_embeddings() -> None: ), ), ) - conn = _Connection() + conn = _Connection( + ( + { + "content_sha256": hashlib.sha256(b"hello").hexdigest(), + "extracted_text": "previous OCR", + }, + { + "content_sha256": hashlib.sha256(b"replaced image").hexdigest(), + "extracted_text": "removed image OCR", + }, + ) + ) embedder = _EmbedMany() count = _persist( @@ -213,6 +230,37 @@ def test_persists_image_tags_formatting_and_embeddings() -> None: assert sum("post_content_embedding_value" in query for query, _args in conn.executed) == 2 * len(chunks) +def test_same_image_retry_cannot_replace_existing_ocr_with_empty_text() -> None: + body = '' + image_index = chunk_by_dom(body)[0].index + normalized = NormalizedPostContent( + text="[image: updated caption]", + image_results=( + ImageContentResult( + image_index, + "image/png", + "described", + SimpleNamespace(caption="updated caption", extracted_text="", tags=()), + ), + ), + ) + conn = _Connection( + ( + { + "content_sha256": hashlib.sha256(b"hello").hexdigest(), + "description_status_code": "described", + "extracted_text": "prior OCR", + "caption": "prior caption", + }, + ) + ) + + with pytest.raises(RuntimeError, match="refusing to replace non-empty image OCR"): + _persist(conn, "post-4", body, normalized_result=normalized) + + assert not any("delete from post_content_unit" in query for query, _args in conn.executed) + + def test_legacy_embed_and_malformed_vectors_never_write_vectors() -> None: conn = _Connection() legacy = _LegacyEmbed([float("nan")])
+ {cell} +
{cell}