fix: preserve semantic document evidence units

CHANGELOG.md

@@ -19,7 +19,7 @@ All notable changes to this project are documented here. Format follows
 
 - Board now names Weekly VOC as an ISO-8601 week list filter. The control
   keeps Voice of Customer posts for the latest week present in the loaded
-  list (UTC Thursday rule) and tells the buyer to open a post to read
+  list (UTC Thursday rule) and tells the reader to open a post to read
   Event Lineage. Reset filters returns every VOC type and every week.
   No TEPP theta is invented (ADR 0092).
 
@@ -28,30 +28,27 @@ All notable changes to this project are documented here. Format follows
 ### Changed
 
 - Renamed "Buyer" terminology to reader/workspace naming across the frontend
-  shell, backend evidence helpers, and living docs (ADR 0119). LineageWeave
-  has no explicit buyer role, so `BuyerNav`/`BuyerDestination` became
-  `WorkspaceNav`/`WorkspaceDestination`, `.buyer-gnb*` CSS became
-  `.workspace-gnb*`, and prose referring to the reading user now says
-  "reader" instead of "buyer". Historical ADRs and changelog entries keep
-  their original wording as a point-in-time record.
+  shell, backend evidence helpers, and living docs (ADR 0119). Historical ADRs
+  and changelog entries retain their point-in-time wording.
 
 ### Fixed
 
+- Preserve source-order nested list units, numeric superscript footnotes,
+  HTML/OOXML table rows, and recognizable Markdown table rows across the
+  semantic-unit parser and buyer body renderer. See the [product and
+  technical gap baseline](docs/product-technical-gap-baseline.md) and
+  [ADR 0103](docs/adr/0103-semantic-document-evidence-contract.md).
+- Preserve multiline VISION table rows, render parent and region OCR tables
+  accessibly, and request source-visible entity, relationship, layout, and
+  document-purpose evidence instead of a generic image caption. VISION calls
+  now share the structure channel's 600-second deep-agent runtime boundary;
+  an empty same-image retry can no longer erase previously observed OCR.
 - Removed the completed one-shot Global Ask package-manager repair workflow;
   normal product CI remains the only branch validation path.
+
 - `make smoke` and `make seed` now run through the locked project `uv`
   environment, so local OIDC and synthetic-data workflows resolve the same
   pinned dependencies as CI.
-- The workspace Event Lineage global Search action now retries focus after the
-  board finishes loading, so navigation from Customer master, Calendar, or
-  Ask Agent lands the cursor in the search box. The handled request is consumed,
-  so later board navigation does not steal focus, and Search closes an open
-  mobile drawer like every other destination change.
-- Mobile Event Lineage evidence cards now read their translated column labels
-  from the rendered cells instead of hardcoded English CSS, and the two drawer
-  close controls have distinct accessible names.
-- Event Lineage SVG edges now retain their instance-specific direction markers,
-  so parent-to-child arrows remain visible when multiple lineage groups render.
 - All OpenAI-compatible chat-completion consumers now validate the shared
   response envelope before parsing it, preventing malformed provider bodies
   from escaping as raw `KeyError` or response-shape details.
@@ -70,26 +67,6 @@ All notable changes to this project are documented here. Format follows
 - Large corpora now use bounded post and Event Lineage landing projections so
   buyers can open complete post-specific detail from a responsive first view.
 
-## [2.11.0] - 2026-08-18
-
-### Added
-
-- Relation verification now preserves a separately authorized internal source
-  post containing normalized organization and relationship context. The
-  counterparty popup can open that evidence without treating it as an
-  external-search URL or changing the external verification status.
-- Large corpora now use bounded post and Event Lineage landing projections so
-  the React screen remains usable before opening complete post-specific detail.
-
-## [2.10.0] - 2026-08-18
-
-### Added
-
-- Production OIDC can now use a real Keyverse issuer through
-  `KEYVERSE_ISSUER` and `KEYVERSE_CLIENT_ID`. The backend discovers the
-  provider's JWKS and verifies the issuer; Compose keeps local Keycloak only
-  as an explicit development fallback and does not emulate Keyverse.
-
 ## [2.12.5] - 2026-08-18
 
 ### Fixed

docker/contextual-orchestrator/start.py

@@ -14,9 +14,11 @@
 
 
 def _pop_first_env(*names: str) -> str:
-    """Read the first configured alias without leaving credentials in the environment."""
+    """Read the first alias, removing quotes preserved by Docker env files."""
     for name in names:
         value = os.environ.pop(name, "").strip()
+        if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}:
+            value = value[1:-1]
         if value:
             return value
     return ""
@@ -27,7 +29,7 @@ def main() -> None:
     provider_key = _pop_first_env("LLM_GATEWAY_API_KEY", "LLM_API_KEY", "NVIDIA_NIM_API_KEY")
     if not provider_key:
         raise SystemExit("LLM_GATEWAY_API_KEY or LLM_API_KEY is required to start the real LLM service")
-    auth_token = os.environ.get("CONTEXTUAL_ORCHESTRATOR_TOKEN", "").strip()
+    auth_token = _pop_first_env("CONTEXTUAL_ORCHESTRATOR_TOKEN")
     if not auth_token:
         raise SystemExit("CONTEXTUAL_ORCHESTRATOR_TOKEN is required to start the authenticated LLM service")
 
@@ -36,14 +38,14 @@ def main() -> None:
         raise SystemExit("LLM_GATEWAY_API_URL or LLM_GATEWAY_URL is required to start the gateway")
     if not provider_url.rstrip("/").endswith("/v1"):
         provider_url = provider_url.rstrip("/") + "/v1"
-    raw_limit = os.environ.pop("LLM_GATEWAY_MAX_OUTPUT_TOKENS", "4096").strip()
+    raw_limit = _pop_first_env("LLM_GATEWAY_MAX_OUTPUT_TOKENS") or "4096"
     try:
         max_output_tokens = int(raw_limit)
     except ValueError as exc:
         raise SystemExit("LLM_GATEWAY_MAX_OUTPUT_TOKENS must be an integer") from exc
     if not 64 <= max_output_tokens <= 4096:
         raise SystemExit("LLM_GATEWAY_MAX_OUTPUT_TOKENS must be between 64 and 4096")
-    raw_body_limit = os.environ.pop("CONTEXTUAL_ORCHESTRATOR_MAX_BODY_BYTES", str(8 * 1024 * 1024)).strip()
+    raw_body_limit = _pop_first_env("CONTEXTUAL_ORCHESTRATOR_MAX_BODY_BYTES") or str(8 * 1024 * 1024)
     try:
         max_body_bytes = int(raw_body_limit)
     except ValueError as exc:
@@ -56,7 +58,7 @@ def main() -> None:
         agent["base_url"] = provider_url
         agent["credential_key"] = "LLM_GATEWAY_API_KEY"
         agent.setdefault("provider_protocol", "auto")
-    embedding_model = os.environ.get("LLM_GATEWAY_EMBEDDING_MODEL", "").strip()
+    embedding_model = _pop_first_env("LLM_GATEWAY_EMBEDDING_MODEL")
     if embedding_model:
         embedding_agents = [
             agent

docs/adr/0103-semantic-document-evidence-contract.md

@@ -0,0 +1,82 @@
+# ADR 0103 — Preserve semantic document evidence across source and buyer views
+
+**Decision status:** Proposed on the stacked product-gap branch  
+**Date:** 2026-08-20  
+**Figma File ID:** `1Su3lDRmiZdcUs47t1QwIX`  
+**Related baseline:** [Product and Technical Gap Baseline](../product-technical-gap-baseline.md)
+
+## Context
+
+Live aggregate inspection identified recurring buyer-visible loss at the
+source boundary: superscript footnotes, nested list order/depth, table rows,
+and Markdown tables were not represented consistently between Python
+ingestion, PostgreSQL units, and the React popup. A flattened string cannot
+reconstruct a table row, a branch in a list, or the position of an image. It
+also makes a later LLM summary less auditable.
+
+The HTML Living Standard defines the semantic elements used by the source
+boundary, including `ol`, `li`, `table`, `tr`, and `sup`. CommonMark provides a
+versioned baseline for Markdown block parsing; table syntax remains an
+extension in many Markdown dialects, so the implementation accepts only a
+recognizable header/separator/data shape and otherwise preserves plain text.
+
+## Decision
+
+1. Parse source content into ordered semantic units before embedding or
+   summarization. A unit retains its source label, source order, indentation
+   metadata, and image position.
+2. Group HTML and OOXML table cells into row units. Markdown tables are
+   recognized only when a header row is immediately followed by a separator
+   row; the separator itself is not evidence content.
+3. Treat explicit CSS/OOXML indentation as authoritative. List-container
+   nesting contributes structural depth but must not double-count an explicit
+   source width.
+4. Mark a numeric footnote only when the source uses a numeric `sup` marker;
+   a numeric table cell or ordinary numbered text is not a footnote by itself.
+5. Keep the frontend's raw-source fallback aligned with the persisted unit
+   labels. Persisted row units render as accessible tables; unresolved
+   structure remains visibly unresolved and actionable.
+6. Apply the same narrow Markdown-table renderer to persisted image OCR.
+   VISION output may use multiple `TEXT` lines so row boundaries survive; its
+   caption names only visible entities, relationships, layout, and document
+   purpose rather than offering a generic one-sentence description. The
+   client allows 600 seconds for deep orchestrator work; a 180-second local
+   cutoff already terminated a valid live response before delivery.
+7. Serialize replacement per source post and reject a same-image retry when
+   its content hash matches non-empty persisted OCR but the retry returns no
+   OCR. Provider completion is transport evidence, not permission to erase a
+   stronger prior observation. During an operator backfill, this typed
+   preservation failure skips only the affected post, records it in the
+   aggregate result, and allows the remaining selected posts to continue.
+
+## Rejected alternatives
+
+- Flattening all bodies into one embedding string: loses row, list, and image
+  boundaries and cannot be repaired at display time.
+- Treating every leading number as a footnote: mislabels table rows and
+  numbered instructions.
+- Calling a provider directly from the parser: violates the orchestrator
+  trust boundary and makes evidence/cost lineage incomplete.
+- Creating a separate parsing service: the existing shared chunker and
+  persistence boundary are sufficient; Ponytail favors the smaller change.
+
+## Consequences
+
+- Search and summaries receive smaller, meaningful units without exposing raw
+  markup or image base64.
+- The database keeps the existing normalized unit tables; this decision adds
+  no denormalized JSON field or new service.
+- A weaker same-image VISION retry fails before replacement, leaving the
+  prior committed evidence available for a later orchestrator retry.
+- A protected retry does not abort an entire operator batch; the skipped-post
+  count is visible to the operator without exposing raw post content.
+- Markdown dialects outside the narrow recognized shape remain plain text and
+  are reported as a future parser extension rather than guessed.
+
+## Verification
+
+The baseline's synthetic tests cover numeric superscript footnotes, marker
+footnotes, nested `ol`/`ul`/`oi` order and depth, HTML/OOXML rows, Markdown
+rows, React table rendering, and unresolved indentation. Full CI remains the
+release gate. A persistence regression test proves that an empty same-hash
+VISION retry cannot delete previously observed OCR.

docs/adr/0105-mathematical-script-semantic-normalization.md

@@ -1,29 +1,50 @@
 # ADR 0105: Preserve explicit metric scripts in semantic text
 
-- Status: Accepted
-- Date: 2026-08-21
+**Status:** Accepted on this PR; not protected-main truth  
+**Date:** 2026-08-21  
+**Owners:** LineageWeave ingestion and buyer-surface maintainers
 
 ## Context
 
-Source posts may encode a metric unit such as `m<sup>3</sup>` or an indexed
-quantity such as `m<sub>3</sub>`. Removing the script element loses searchable
-and buyer-visible mathematical meaning, while treating every numeric
-superscript as mathematics would break the existing footnote contract.
+Source posts commonly encode a unit such as `m<sup>3</sup>`, `m<sub>3</sub>`,
+`m^3`, or `m_3` with HTML or plain-text notation. Dropping the markup changes
+the searchable meaning to `m3`, while treating every numeric `sup` element as
+mathematics would break the existing numeric-footnote contract. Full MathML
+parsing is not yet justified by the current product surface, but the loss of
+explicit unit scripts is a buyer-visible defect.
+
+MathML 4 defines `msup`, `msub`, and `msubsup` as structural script elements;
+HTML `sup`/`sub` are a permitted lighter-weight notation when detailed
+mathematical markup is not required. This decision therefore adds a bounded
+normalization boundary and keeps the source representation unchanged.
 
 ## Decision
 
-1. Preserve the original source body unchanged.
-2. In derived semantic text, normalize only an explicit bounded metric base
+1. Preserve the immutable source body exactly as imported.
+2. In derived semantic text only, normalize an explicitly bounded metric base
    (`m`, `cm`, `mm`, `km`, or `kg`, optionally preceded by a number) followed
-   by one-to-three numeric `sup` or `sub` elements into Unicode
-   superscript/subscript digits. For example, `5m<sup>3</sup>` becomes `5m³`.
-3. Leave ordinary numeric superscripts on prose under the existing footnote
-   role contract.
-4. Apply the same normalization in backend chunks and frontend rendering.
+   by numeric `sup`/`sub` markup or plain-text `^`/`_` notation into Unicode
+   superscript/subscript digits. For example, `5m<sup>3</sup>` and `5m^3`
+   become `5m³`, while `m<sub>3</sub>` and `m_3` become `m₃`.
+3. Keep ordinary numeric superscripts and caret expressions on prose under the existing footnote
+   role contract. Do not infer a mathematical formula from an arbitrary word.
+4. Apply the same bounded normalization in backend semantic chunks and the
+   React buyer display so search text and visible text agree.
+5. Defer full MathML/LaTeX parsing, expression trees, and ontology term
+   creation until an authorized fixture demonstrates a need beyond metric
+   scripts. Any such change requires a new ADR and parser contract.
 
 ## Consequences
 
-Metric exponents remain searchable and readable without inventing formula
-semantics. Arbitrary mathematical markup beyond this bounded case remains an
-explicit open gap and must be covered by a later ADR and fixture before being
-normalized.
+- Search and the buyer popup retain the visible distinction between `m³` and
+  `m3` without exposing source HTML to the embedding model.
+- Existing numeric-footnote tests remain unchanged because the bounded metric
+  pattern is the only new conversion.
+- The current implementation does not claim to understand arbitrary equations;
+  unsupported script markup remains ordinary source text and must not be
+  presented as a parsed ontology expression.
+
+## References (APA 7th)
+
+World Wide Web Consortium. (2026). *Mathematical Markup Language (MathML)
+Version 4.0* (W3C Recommendation). https://www.w3.org/TR/mathml4/

docs/adr/0103-source-whitespace-is-not-authoritative-structure.md → docs/adr/0105-source-whitespace-is-not-authoritative-structure.md

@@ -1,4 +1,4 @@
-# ADR 0103: Source-only whitespace is not authoritative structure
+# ADR 0105: Source-only whitespace is not authoritative structure
 
 - Status: Accepted
 - Date: 2026-08-20

docs/doctoring/PRODUCT_TECHNICAL_GAP_REFERENCES.md

@@ -0,0 +1,24 @@
+# Product technical-gap references
+
+These references are the normative basis for the semantic-unit boundary in
+[ADR 0103](../adr/0103-semantic-document-evidence-contract.md). Dates and
+versions are recorded so a later standards refresh can be reviewed rather
+than silently changing parser behavior.
+
+## APA 7th edition
+
+CommonMark. (2024). *CommonMark spec (Version 0.31.2)*. https://spec.commonmark.org/0.31.2/
+
+WHATWG. (2026). *HTML: Living Standard*. https://html.spec.whatwg.org/multipage/
+
+## Applied mapping
+
+| Source | Boundary used in LineageWeave |
+| --- | --- |
+| CommonMark (2024) | Recognizable Markdown block/header/separator shape; unrecognized dialects remain source text. |
+| WHATWG (2026) | HTML list, table-row, and `sup` semantics; source order and element identity are retained as unit metadata. |
+
+These standards define syntax and semantics, not an LLM extraction license.
+Provider-derived summaries, image descriptions, project boundaries, and
+5W1H values still require contextual-orchestrator provenance and explicit
+source evidence.

frontend/src/PostBody.stories.tsx

@@ -0,0 +1,70 @@
+import type { Meta, StoryObj } from "@storybook/react-vite";
+import { PostBody } from "./PostBody";
+
+const meta = {
+  title: "Evidence/PostBody",
+  component: PostBody,
+} satisfies Meta<typeof PostBody>;
+
+export default meta;
+
+type Story = StoryObj<typeof meta>;
+
+const TINY_PNG =
+  "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=";
+
+export const MarkdownTableEvidence: Story = {
+  args: {
+    body: "| Workstream | State |\n| --- | --- |\n| Alpha | Ready |",
+    structureUnits: [
+      {
+        unit_index: 0,
+        unit_kind_code: "plain_text",
+        unit_label: "markdown_tr",
+        unit_text: "Workstream | State",
+        indent_level: 0,
+        indent_source_code: "unresolved",
+        indent_confidence: 0,
+        indent_evidence: "Markdown table row",
+      },
+      {
+        unit_index: 1,
+        unit_kind_code: "plain_text",
+        unit_label: "markdown_tr",
+        unit_text: "Alpha | Ready",
+        indent_level: 0,
+        indent_source_code: "unresolved",
+        indent_confidence: 0,
+        indent_evidence: "Markdown table row",
+      },
+    ],
+  },
+};
+
+export const MarkdownTableFallback: Story = {
+  args: {
+    body: "Intro.\n\n| Workstream | State |\n| --- | --- |\n| Alpha | Ready |\n\nNext action.",
+  },
+};
+
+export const ImageOcrTableEvidence: Story = {
+  args: {
+    body: `<img src="data:image/png;base64,${TINY_PNG}" />`,
+    imageContent: [
+      {
+        unit_index: 0,
+        mime_type: "image/png",
+        status_code: "completed",
+        extracted_text: "| Workstream | State |\n| --- | --- |\n| Alpha | Ready |",
+        caption: "A synthetic workstream status table.",
+        tags: ["table"],
+      },
+    ],
+  },
+};
+
+export const NumericFootnote: Story = {
+  args: {
+    body: "<p>Evidence remains attached to the source.</p><p><sup>1</sup> Source note.</p>",
+  },
+};