fix: preserve semantic image evidence tables

CHANGELOG.md

@@ -13,6 +13,11 @@ All notable changes to this project are documented here. Format follows
   semantic-unit parser and buyer body renderer. See the [product and
   technical gap baseline](docs/product-technical-gap-baseline.md) and
   [ADR 0103](docs/adr/0103-semantic-document-evidence-contract.md).
+- Preserve multiline VISION table rows, render parent and region OCR tables
+  accessibly, and request source-visible entity, relationship, layout, and
+  document-purpose evidence instead of a generic image caption. VISION calls
+  now share the structure channel's 600-second deep-agent runtime boundary;
+  an empty same-image retry can no longer erase previously observed OCR.
 
 - `make smoke` and `make seed` now run through the locked project `uv`
   environment, so local OIDC and synthetic-data workflows resolve the same

docker/contextual-orchestrator/start.py

@@ -14,9 +14,11 @@
 
 
 def _pop_first_env(*names: str) -> str:
-    """Read the first configured alias without leaving credentials in the environment."""
+    """Read the first alias, removing quotes preserved by Docker env files."""
     for name in names:
         value = os.environ.pop(name, "").strip()
+        if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}:
+            value = value[1:-1]
         if value:
             return value
     return ""
@@ -27,7 +29,7 @@ def main() -> None:
     provider_key = _pop_first_env("LLM_GATEWAY_API_KEY", "LLM_API_KEY", "NVIDIA_NIM_API_KEY")
     if not provider_key:
         raise SystemExit("LLM_GATEWAY_API_KEY or LLM_API_KEY is required to start the real LLM service")
-    auth_token = os.environ.get("CONTEXTUAL_ORCHESTRATOR_TOKEN", "").strip()
+    auth_token = _pop_first_env("CONTEXTUAL_ORCHESTRATOR_TOKEN")
     if not auth_token:
         raise SystemExit("CONTEXTUAL_ORCHESTRATOR_TOKEN is required to start the authenticated LLM service")
 
@@ -36,14 +38,14 @@ def main() -> None:
         raise SystemExit("LLM_GATEWAY_API_URL or LLM_GATEWAY_URL is required to start the gateway")
     if not provider_url.rstrip("/").endswith("/v1"):
         provider_url = provider_url.rstrip("/") + "/v1"
-    raw_limit = os.environ.pop("LLM_GATEWAY_MAX_OUTPUT_TOKENS", "4096").strip()
+    raw_limit = _pop_first_env("LLM_GATEWAY_MAX_OUTPUT_TOKENS") or "4096"
     try:
         max_output_tokens = int(raw_limit)
     except ValueError as exc:
         raise SystemExit("LLM_GATEWAY_MAX_OUTPUT_TOKENS must be an integer") from exc
     if not 64 <= max_output_tokens <= 4096:
         raise SystemExit("LLM_GATEWAY_MAX_OUTPUT_TOKENS must be between 64 and 4096")
-    raw_body_limit = os.environ.pop("CONTEXTUAL_ORCHESTRATOR_MAX_BODY_BYTES", str(8 * 1024 * 1024)).strip()
+    raw_body_limit = _pop_first_env("CONTEXTUAL_ORCHESTRATOR_MAX_BODY_BYTES") or str(8 * 1024 * 1024)
     try:
         max_body_bytes = int(raw_body_limit)
     except ValueError as exc:
@@ -56,7 +58,7 @@ def main() -> None:
         agent["base_url"] = provider_url
         agent["credential_key"] = "LLM_GATEWAY_API_KEY"
         agent.setdefault("provider_protocol", "auto")
-    embedding_model = os.environ.get("LLM_GATEWAY_EMBEDDING_MODEL", "").strip()
+    embedding_model = _pop_first_env("LLM_GATEWAY_EMBEDDING_MODEL")
     if embedding_model:
         embedding_agents = [
             agent

docs/adr/0103-semantic-document-evidence-contract.md

@@ -36,6 +36,18 @@ recognizable header/separator/data shape and otherwise preserves plain text.
 5. Keep the frontend's raw-source fallback aligned with the persisted unit
    labels. Persisted row units render as accessible tables; unresolved
    structure remains visibly unresolved and actionable.
+6. Apply the same narrow Markdown-table renderer to persisted image OCR.
+   VISION output may use multiple `TEXT` lines so row boundaries survive; its
+   caption names only visible entities, relationships, layout, and document
+   purpose rather than offering a generic one-sentence description. The
+   client allows 600 seconds for deep orchestrator work; a 180-second local
+   cutoff already terminated a valid live response before delivery.
+7. Serialize replacement per source post and reject a same-image retry when
+   its content hash matches non-empty persisted OCR but the retry returns no
+   OCR. Provider completion is transport evidence, not permission to erase a
+   stronger prior observation. During an operator backfill, this typed
+   preservation failure skips only the affected post, records it in the
+   aggregate result, and allows the remaining selected posts to continue.
 
 ## Rejected alternatives
 
@@ -54,6 +66,10 @@ recognizable header/separator/data shape and otherwise preserves plain text.
   markup or image base64.
 - The database keeps the existing normalized unit tables; this decision adds
   no denormalized JSON field or new service.
+- A weaker same-image VISION retry fails before replacement, leaving the
+  prior committed evidence available for a later orchestrator retry.
+- A protected retry does not abort an entire operator batch; the skipped-post
+  count is visible to the operator without exposing raw post content.
 - Markdown dialects outside the narrow recognized shape remain plain text and
   are reported as a future parser extension rather than guessed.
 
@@ -62,4 +78,5 @@ recognizable header/separator/data shape and otherwise preserves plain text.
 The baseline's synthetic tests cover numeric superscript footnotes, marker
 footnotes, nested `ol`/`ul`/`oi` order and depth, HTML/OOXML rows, Markdown
 rows, React table rendering, and unresolved indentation. Full CI remains the
-release gate.
+release gate. A persistence regression test proves that an empty same-hash
+VISION retry cannot delete previously observed OCR.

docs/adr/0105-mathematical-script-semantic-normalization.md

@@ -0,0 +1,50 @@
+# ADR 0105: Preserve explicit metric scripts in semantic text
+
+**Status:** Accepted on this PR; not protected-main truth  
+**Date:** 2026-08-21  
+**Owners:** LineageWeave ingestion and buyer-surface maintainers
+
+## Context
+
+Source posts commonly encode a unit such as `m<sup>3</sup>`, `m<sub>3</sub>`,
+`m^3`, or `m_3` with HTML or plain-text notation. Dropping the markup changes
+the searchable meaning to `m3`, while treating every numeric `sup` element as
+mathematics would break the existing numeric-footnote contract. Full MathML
+parsing is not yet justified by the current product surface, but the loss of
+explicit unit scripts is a buyer-visible defect.
+
+MathML 4 defines `msup`, `msub`, and `msubsup` as structural script elements;
+HTML `sup`/`sub` are a permitted lighter-weight notation when detailed
+mathematical markup is not required. This decision therefore adds a bounded
+normalization boundary and keeps the source representation unchanged.
+
+## Decision
+
+1. Preserve the immutable source body exactly as imported.
+2. In derived semantic text only, normalize an explicitly bounded metric base
+   (`m`, `cm`, `mm`, `km`, or `kg`, optionally preceded by a number) followed
+   by numeric `sup`/`sub` markup or plain-text `^`/`_` notation into Unicode
+   superscript/subscript digits. For example, `5m<sup>3</sup>` and `5m^3`
+   become `5m³`, while `m<sub>3</sub>` and `m_3` become `m₃`.
+3. Keep ordinary numeric superscripts and caret expressions on prose under the existing footnote
+   role contract. Do not infer a mathematical formula from an arbitrary word.
+4. Apply the same bounded normalization in backend semantic chunks and the
+   React buyer display so search text and visible text agree.
+5. Defer full MathML/LaTeX parsing, expression trees, and ontology term
+   creation until an authorized fixture demonstrates a need beyond metric
+   scripts. Any such change requires a new ADR and parser contract.
+
+## Consequences
+
+- Search and the buyer popup retain the visible distinction between `m³` and
+  `m3` without exposing source HTML to the embedding model.
+- Existing numeric-footnote tests remain unchanged because the bounded metric
+  pattern is the only new conversion.
+- The current implementation does not claim to understand arbitrary equations;
+  unsupported script markup remains ordinary source text and must not be
+  presented as a parsed ontology expression.
+
+## References (APA 7th)
+
+World Wide Web Consortium. (2026). *Mathematical Markup Language (MathML)
+Version 4.0* (W3C Recommendation). https://www.w3.org/TR/mathml4/

frontend/src/PostBody.stories.tsx

@@ -10,6 +10,9 @@ export default meta;
 
 type Story = StoryObj<typeof meta>;
 
+const TINY_PNG =
+  "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=";
+
 export const MarkdownTableEvidence: Story = {
   args: {
     body: "| Workstream | State |\n| --- | --- |\n| Alpha | Ready |",
@@ -44,6 +47,22 @@ export const MarkdownTableFallback: Story = {
   },
 };
 
+export const ImageOcrTableEvidence: Story = {
+  args: {
+    body: `<img src="data:image/png;base64,${TINY_PNG}" />`,
+    imageContent: [
+      {
+        unit_index: 0,
+        mime_type: "image/png",
+        status_code: "completed",
+        extracted_text: "| Workstream | State |\n| --- | --- |\n| Alpha | Ready |",
+        caption: "A synthetic workstream status table.",
+        tags: ["table"],
+      },
+    ],
+  },
+};
+
 export const NumericFootnote: Story = {
   args: {
     body: "<p>Evidence remains attached to the source.</p><p><sup>1</sup> Source note.</p>",

frontend/src/PostBody.test.tsx

@@ -279,6 +279,43 @@ describe("PostBody", () => {
     expect(screen.getAllByRole("row")).toHaveLength(2);
   });
 
+  it("renders table-shaped image OCR as accessible evidence", () => {
+    render(
+      <PostBody
+        body={'<img src="data:image/png;base64,QQ==" />'}
+        imageContent={[
+          {
+            unit_index: 0,
+            mime_type: "image/png",
+            status_code: "completed",
+            extracted_text: "| Project | Status |\n| --- | --- |\n| Alpha | Ready |",
+            caption: "A synthetic project status table.",
+            tags: ["table"],
+            regions: [
+              {
+                region_index: 0,
+                x_ratio: 0,
+                y_ratio: 0,
+                width_ratio: 1,
+                height_ratio: 1,
+                status_code: "completed",
+                extracted_text: "| Owner | Action |\n| --- | --- |\n| Team A | Review |",
+                caption: "The table region assigns an action to a team.",
+                tags: ["assignment"],
+              },
+            ],
+          },
+        ]}
+      />,
+    );
+
+    expect(screen.getAllByRole("table")).toHaveLength(2);
+    expect(screen.getByRole("columnheader", { name: "Project" })).toBeInTheDocument();
+    expect(screen.getByText("Ready")).toBeInTheDocument();
+    expect(screen.getByText("The table region assigns an action to a team.")).toBeInTheDocument();
+    expect(screen.getByRole("columnheader", { name: "Owner" })).toBeInTheDocument();
+  });
+
   it("marks persisted footnotes as footnote evidence", () => {
     render(
       <PostBody
@@ -371,6 +408,28 @@ describe("PostBody", () => {
     expect(screen.getByText("Panel")).toBeInTheDocument();
   });
 
+  it("keeps escaped pipe characters inside image OCR table cells", () => {
+    render(
+      <PostBody
+        body={'<img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=" />'}
+        imageContent={[
+          {
+            unit_index: 0,
+            mime_type: "image/png",
+            status_code: "described",
+            extracted_text: "| Item | State |\n| --- | --- |\n| Review \\| approve | Ready |",
+            caption: "A table image with an escaped separator.",
+            tags: [],
+          },
+        ]}
+      />,
+    );
+
+    expect(screen.getByRole("table")).toBeInTheDocument();
+    expect(screen.getByText("Review | approve")).toBeInTheDocument();
+    expect(screen.getByText("Ready")).toBeInTheDocument();
+  });
+
   it("keeps source-image placement while showing persisted OCR and caption evidence", () => {
     render(
       <PostBody

frontend/src/PostBody.tsx

@@ -12,7 +12,7 @@ function parsePipeDelimitedTable(text: string): string[][] | null {
   const rows = text
     .split(/\r?\n/)
     .map((row) => {
-      const cells = row.split("|").map((cell) => cell.trim());
+      const cells = row.split(/(?<!\\)\|/).map((cell) => cell.trim().replace(/\\\|/g, "|"));
       if (cells[0] === "") cells.shift();
       if (cells[cells.length - 1] === "") cells.pop();
       return cells;
@@ -27,10 +27,20 @@ function parsePipeDelimitedTable(text: string): string[][] | null {
 function renderImageText(text: string) {
   const rows = parsePipeDelimitedTable(text);
   if (!rows) return <p>{text}</p>;
+  const [header, ...bodyRows] = rows;
   return (
     <table className="post-body-table post-image-text-table">
+      <thead>
+        <tr>
+          {header.map((cell, cellIndex) => (
+            <th key={`post-image-text-header-${cellIndex}`} scope="col">
+              {cell}
+            </th>
+          ))}
+        </tr>
+      </thead>
       <tbody>
-        {rows.map((row, rowIndex) => (
+        {bodyRows.map((row, rowIndex) => (
           <tr key={`post-image-text-row-${rowIndex}`}>
             {row.map((cell, cellIndex) => (
               <td key={`post-image-text-cell-${rowIndex}-${cellIndex}`}>{cell}</td>
@@ -77,7 +87,14 @@ function renderImageEvidence(
           <ol>
             {imageContent.regions.map((region) => (
               <li key={region.region_index}>
-                <span>{region.caption || region.extracted_text || t("Unknown")}</span>
+                {region.caption ? <p>{region.caption}</p> : null}
+                {region.extracted_text ? (
+                  <div className="post-image-region-text">
+                    {renderImageText(region.extracted_text)}
+                  </div>
+                ) : region.caption ? null : (
+                  t("Unknown")
+                )}
                 {region.tags.length ? (
                   <small>
                     {t("Image tags")}: {region.tags.join(", ")}

frontend/src/postBodyDisplay.test.ts

@@ -67,6 +67,18 @@ describe("splitPostBody", () => {
     ]);
   });
 
+  it("preserves explicit metric superscripts and subscripts", () => {
+    expect(splitPostBody("<p>Volume: 5m<sup>3</sup>, index m<sub>3</sub>.</p>")).toEqual([
+      { kind: "text", text: "Volume: 5m³, index m₃." },
+    ]);
+  });
+
+  it("normalizes plain-text metric superscripts and subscripts", () => {
+    expect(splitPostBody("<p>Volume: 5m^3, index m_3, braced m^{2}.</p>")).toEqual([
+      { kind: "text", text: "Volume: 5m³, index m₃, braced m²." },
+    ]);
+  });
+
   it("preserves HTML and Word footnote blocks as footnote paragraphs", () => {
     expect(
       splitPostBody(
@@ -128,6 +140,21 @@ describe("splitPostBody", () => {
     ]);
   });
 
+  it("normalizes metric scripts inside Markdown table cells", () => {
+    expect(
+      splitMarkdownTableBody("| Metric | Index |\n| --- | --- |\n| 5m^3 | m<sub>3</sub> |"),
+    ).toEqual([{ kind: "table", rows: [["Metric", "Index"], ["5m³", "m₃"]] }]);
+  });
+
+  it("unescapes pipe characters inside Markdown cells without accepting a short delimiter", () => {
+    expect(
+      splitMarkdownTableBody(
+        "| Project | Notes |\n| :--- | ---: |\n| Alpha | Ready \\| review |",
+      ),
+    ).toEqual([{ kind: "table", rows: [["Project", "Notes"], ["Alpha", "Ready | review"]] }]);
+    expect(splitMarkdownTableBody("| Project | Status |\n| -- | -- |\n| Alpha | Ready |")).toBeNull();
+  });
+
   it("leaves a plain-text post unchanged so existing popups keep their wording", () => {
     expect(splitPostBody("The full body text.")).toEqual([
       { kind: "text", text: "The full body text." },

frontend/src/postBodyDisplay.ts

@@ -32,6 +32,27 @@ const NUMERIC_FOOTNOTE_MARKER = "\u0003lw-numeric-footnote\u0004";
 const FOOTNOTE_BLOCK_MARKER = "\u0005lw-footnote-block\u0006";
 const FOOTNOTE_BLOCK_OPEN = /<\s*(?:footnote|endnote|w:footnote|w:endnote)\b[^>]*>/gi;
 const NUMERIC_SUPERSCRIPT = /<sup\b[^>]*>\s*(\d{1,3})\s*<\/sup>/gi;
+const SUPERSCRIPT_DIGITS = "⁰¹²³⁴⁵⁶⁷⁸⁹";
+const SUBSCRIPT_DIGITS = "₀₁₂₃₄₅₆₇₈₉";
+const METRIC_MARKUP =
+  /((?<![A-Za-z])(?:\d+(?:\.\d+)?\s*)?(?:km|cm|mm|kg|m))\s*<(sup|sub)\b[^>]*>\s*(\d{1,3})\s*<\/\2>/gi;
+const METRIC_PLAIN_SCRIPT =
+  /((?<![A-Za-z])(?:\d+(?:\.\d+)?\s*)?(?:km|cm|mm|kg|m))\s*(\^|_)\s*(?:\{(\d{1,3})\}|(\d{1,3}))/gi;
+
+function normalizeMetricMarkup(raw: string): string {
+  return raw
+    .replace(METRIC_MARKUP, (_match, base: string, kind: string, digits: string) => {
+      const table = kind.toLowerCase() === "sup" ? SUPERSCRIPT_DIGITS : SUBSCRIPT_DIGITS;
+      return `${base}${[...digits].map((digit) => table[Number(digit)]).join("")}`;
+    })
+    .replace(
+      METRIC_PLAIN_SCRIPT,
+      (_match, base: string, kind: string, bracedDigits: string, digits: string) => {
+        const table = kind === "^" ? SUPERSCRIPT_DIGITS : SUBSCRIPT_DIGITS;
+        return `${base}${[...(bracedDigits || digits)].map((digit) => table[Number(digit)]).join("")}`;
+      },
+    );
+}
 
 function stripIndentMarkers(value: string): string {
   return value
@@ -224,7 +245,7 @@ function isDecodableBase64(raw: string): boolean {
 
 function pushText(segments: PostBodySegment[], raw: string, indentUnit: number): void {
   const text = stripHtmlTags(
-    raw
+    normalizeMetricMarkup(raw)
       .replace(FOOTNOTE_BLOCK_OPEN, FOOTNOTE_BLOCK_MARKER)
       .replace(NUMERIC_SUPERSCRIPT, `${NUMERIC_FOOTNOTE_MARKER}$1`),
   );
@@ -257,7 +278,7 @@ function markdownCells(line: string): string[] | null {
   const value = line.trim().replace(/^\|/, "").replace(/(?<!\\)\|$/, "");
   if (!value.includes("|")) return null;
   const cells = value.split(/(?<!\\)\|/).map((cell) => cell.trim().replace(/\\\|/g, "|"));
-  return cells.length >= 2 && cells.every(Boolean) ? cells : null;
+  return cells.length >= 2 && cells.every(Boolean) ? cells.map(normalizeMetricMarkup) : null;
 }
 
 function isMarkdownSeparatorRow(cells: string[] | null): boolean {