feat: preserve metric superscript and subscript semantics

docs/adr/0105-mathematical-script-semantic-normalization.md

@@ -0,0 +1,50 @@
+# ADR 0105: Preserve explicit metric scripts in semantic text
+
+**Status:** Accepted on this PR; not protected-main truth  
+**Date:** 2026-08-21  
+**Owners:** LineageWeave ingestion and buyer-surface maintainers
+
+## Context
+
+Source posts commonly encode a unit such as `m<sup>3</sup>`, `m<sub>3</sub>`,
+`m^3`, or `m_3` with HTML or plain-text notation. Dropping the markup changes
+the searchable meaning to `m3`, while treating every numeric `sup` element as
+mathematics would break the existing numeric-footnote contract. Full MathML
+parsing is not yet justified by the current product surface, but the loss of
+explicit unit scripts is a buyer-visible defect.
+
+MathML 4 defines `msup`, `msub`, and `msubsup` as structural script elements;
+HTML `sup`/`sub` are a permitted lighter-weight notation when detailed
+mathematical markup is not required. This decision therefore adds a bounded
+normalization boundary and keeps the source representation unchanged.
+
+## Decision
+
+1. Preserve the immutable source body exactly as imported.
+2. In derived semantic text only, normalize an explicitly bounded metric base
+   (`m`, `cm`, `mm`, `km`, or `kg`, optionally preceded by a number) followed
+   by numeric `sup`/`sub` markup or plain-text `^`/`_` notation into Unicode
+   superscript/subscript digits. For example, `5m<sup>3</sup>` and `5m^3`
+   become `5m³`, while `m<sub>3</sub>` and `m_3` become `m₃`.
+3. Keep ordinary numeric superscripts and caret expressions on prose under the existing footnote
+   role contract. Do not infer a mathematical formula from an arbitrary word.
+4. Apply the same bounded normalization in backend semantic chunks and the
+   React buyer display so search text and visible text agree.
+5. Defer full MathML/LaTeX parsing, expression trees, and ontology term
+   creation until an authorized fixture demonstrates a need beyond metric
+   scripts. Any such change requires a new ADR and parser contract.
+
+## Consequences
+
+- Search and the buyer popup retain the visible distinction between `m³` and
+  `m3` without exposing source HTML to the embedding model.
+- Existing numeric-footnote tests remain unchanged because the bounded metric
+  pattern is the only new conversion.
+- The current implementation does not claim to understand arbitrary equations;
+  unsupported script markup remains ordinary source text and must not be
+  presented as a parsed ontology expression.
+
+## References (APA 7th)
+
+World Wide Web Consortium. (2026). *Mathematical Markup Language (MathML)
+Version 4.0* (W3C Recommendation). https://www.w3.org/TR/mathml4/

frontend/src/postBodyDisplay.test.ts

@@ -67,6 +67,18 @@ describe("splitPostBody", () => {
     ]);
   });
 
+  it("preserves explicit metric superscripts and subscripts", () => {
+    expect(splitPostBody("<p>Volume: 5m<sup>3</sup>, index m<sub>3</sub>.</p>")).toEqual([
+      { kind: "text", text: "Volume: 5m³, index m₃." },
+    ]);
+  });
+
+  it("normalizes plain-text metric superscripts and subscripts", () => {
+    expect(splitPostBody("<p>Volume: 5m^3, index m_3, braced m^{2}.</p>")).toEqual([
+      { kind: "text", text: "Volume: 5m³, index m₃, braced m²." },
+    ]);
+  });
+
   it("preserves HTML and Word footnote blocks as footnote paragraphs", () => {
     expect(
       splitPostBody(
@@ -128,6 +140,12 @@ describe("splitPostBody", () => {
     ]);
   });
 
+  it("normalizes metric scripts inside Markdown table cells", () => {
+    expect(
+      splitMarkdownTableBody("| Metric | Index |\n| --- | --- |\n| 5m^3 | m<sub>3</sub> |"),
+    ).toEqual([{ kind: "table", rows: [["Metric", "Index"], ["5m³", "m₃"]] }]);
+  });
+
   it("unescapes pipe characters inside Markdown cells without accepting a short delimiter", () => {
     expect(
       splitMarkdownTableBody(

frontend/src/postBodyDisplay.ts

@@ -32,6 +32,27 @@ const NUMERIC_FOOTNOTE_MARKER = "\u0003lw-numeric-footnote\u0004";
 const FOOTNOTE_BLOCK_MARKER = "\u0005lw-footnote-block\u0006";
 const FOOTNOTE_BLOCK_OPEN = /<\s*(?:footnote|endnote|w:footnote|w:endnote)\b[^>]*>/gi;
 const NUMERIC_SUPERSCRIPT = /<sup\b[^>]*>\s*(\d{1,3})\s*<\/sup>/gi;
+const SUPERSCRIPT_DIGITS = "⁰¹²³⁴⁵⁶⁷⁸⁹";
+const SUBSCRIPT_DIGITS = "₀₁₂₃₄₅₆₇₈₉";
+const METRIC_MARKUP =
+  /((?<![A-Za-z])(?:\d+(?:\.\d+)?\s*)?(?:km|cm|mm|kg|m))\s*<(sup|sub)\b[^>]*>\s*(\d{1,3})\s*<\/\2>/gi;
+const METRIC_PLAIN_SCRIPT =
+  /((?<![A-Za-z])(?:\d+(?:\.\d+)?\s*)?(?:km|cm|mm|kg|m))\s*(\^|_)\s*(?:\{(\d{1,3})\}|(\d{1,3}))/gi;
+
+function normalizeMetricMarkup(raw: string): string {
+  return raw
+    .replace(METRIC_MARKUP, (_match, base: string, kind: string, digits: string) => {
+      const table = kind.toLowerCase() === "sup" ? SUPERSCRIPT_DIGITS : SUBSCRIPT_DIGITS;
+      return `${base}${[...digits].map((digit) => table[Number(digit)]).join("")}`;
+    })
+    .replace(
+      METRIC_PLAIN_SCRIPT,
+      (_match, base: string, kind: string, bracedDigits: string, digits: string) => {
+        const table = kind === "^" ? SUPERSCRIPT_DIGITS : SUBSCRIPT_DIGITS;
+        return `${base}${[...(bracedDigits || digits)].map((digit) => table[Number(digit)]).join("")}`;
+      },
+    );
+}
 
 function stripIndentMarkers(value: string): string {
   return value
@@ -224,7 +245,7 @@ function isDecodableBase64(raw: string): boolean {
 
 function pushText(segments: PostBodySegment[], raw: string, indentUnit: number): void {
   const text = stripHtmlTags(
-    raw
+    normalizeMetricMarkup(raw)
       .replace(FOOTNOTE_BLOCK_OPEN, FOOTNOTE_BLOCK_MARKER)
       .replace(NUMERIC_SUPERSCRIPT, `${NUMERIC_FOOTNOTE_MARKER}$1`),
   );
@@ -257,7 +278,7 @@ function markdownCells(line: string): string[] | null {
   const value = line.trim().replace(/^\|/, "").replace(/(?<!\\)\|$/, "");
   if (!value.includes("|")) return null;
   const cells = value.split(/(?<!\\)\|/).map((cell) => cell.trim().replace(/\\\|/g, "|"));
-  return cells.length >= 2 && cells.every(Boolean) ? cells : null;
+  return cells.length >= 2 && cells.every(Boolean) ? cells.map(normalizeMetricMarkup) : null;
 }
 
 function isMarkdownSeparatorRow(cells: string[] | null): boolean {

lineageweave/chunking.py

@@ -134,6 +134,7 @@ def _is_footnote_reference(attrs: list[tuple[str, str | None]]) -> bool:
 
 def normalize_semantic_text(text: str) -> str:
     """Remove visual hanging-indent breaks without changing source content."""
+    text = _normalize_metric_markup(_normalize_plain_metric_scripts(text))
     lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n")
     normalized: list[str] = []
     for line in lines:
@@ -296,6 +297,37 @@ def chunk_by_paragraph(text: str) -> list[Chunk]:
 
 
 _SENTENCE_BOUNDARY = re.compile(r"(?<=[.!?])\s+(?=[A-Z0-9가-힣])")
+_SUPERSCRIPT_DIGITS = str.maketrans("0123456789", "⁰¹²³⁴⁵⁶⁷⁸⁹")
+_SUBSCRIPT_DIGITS = str.maketrans("0123456789", "₀₁₂₃₄₅₆₇₈₉")
+_METRIC_MARKUP = re.compile(
+    r"(?P<base>(?<![A-Za-z])(?:\d+(?:\.\d+)?\s*)?(?:km|cm|mm|kg|m))\s*"
+    r"<(?P<kind>sup|sub)\b[^>]*>\s*(?P<digits>\d{1,3})\s*</(?P=kind)>",
+    re.IGNORECASE,
+)
+_METRIC_PLAIN_SCRIPT = re.compile(
+    r"(?P<base>(?<![A-Za-z])(?:\d+(?:\.\d+)?\s*)?(?:km|cm|mm|kg|m))\s*"
+    r"(?P<kind>\^|_)\s*(?:\{(?P<braced_digits>\d{1,3})\}|(?P<digits>\d{1,3}))",
+    re.IGNORECASE,
+)
+
+
+def _normalize_plain_metric_scripts(text: str) -> str:
+    """Normalize bounded plain-text metric exponents and indices."""
+    def replace(match: re.Match[str]) -> str:
+        table = _SUPERSCRIPT_DIGITS if match.group("kind") == "^" else _SUBSCRIPT_DIGITS
+        digits = match.group("braced_digits") or match.group("digits") or ""
+        return f"{match.group('base')}{digits.translate(table)}"
+
+    return _METRIC_PLAIN_SCRIPT.sub(replace, text)
+
+
+def _normalize_metric_markup(html: str) -> str:
+    """Keep explicit metric superscript/subscript digits in semantic text."""
+    def replace(match: re.Match[str]) -> str:
+        table = _SUPERSCRIPT_DIGITS if match.group("kind").lower() == "sup" else _SUBSCRIPT_DIGITS
+        return f"{match.group('base')}{match.group('digits').translate(table)}"
+
+    return _METRIC_MARKUP.sub(replace, html)
 
 
 def chunk_by_sentence(text: str) -> list[Chunk]:
@@ -554,8 +586,7 @@ def _markdown_cells(line: str) -> list[str] | None:
     if "|" not in line:
         return None
     value = line.strip()
-    if value.startswith("|"):
-        value = value[1:]
+    value = value.removeprefix("|")
     if value.endswith("|") and not value.endswith("\\|"):
         value = value[:-1]
     cells = [cell.strip().replace(r"\|", "|") for cell in re.split(r"(?<!\\)\|", value)]
@@ -590,13 +621,13 @@ def flush_pending() -> None:
 
         found_table = True
         flush_pending()
-        entries.append(("markdown_tr", " | ".join(header)))
+        entries.append(("markdown_tr", " | ".join(normalize_semantic_text(cell) for cell in header)))
         index += 2
         while index < len(lines) and lines[index].strip():
             cells = _markdown_cells(lines[index])
             if cells is None:
                 break
-            entries.append(("markdown_tr", " | ".join(cells)))
+            entries.append(("markdown_tr", " | ".join(normalize_semantic_text(cell) for cell in cells)))
             index += 1
 
     flush_pending()
@@ -616,7 +647,10 @@ def _is_markdown_table_row(line: str) -> bool:
 
 def _render_markdown_table_row(line: str) -> str:
     """Keep Markdown table columns as searchable row evidence."""
-    return " | ".join(cell.strip() for cell in line.strip().strip("|").split("|"))
+    return " | ".join(
+        normalize_semantic_text(cell.strip())
+        for cell in line.strip().strip("|").split("|")
+    )
 
 
 def _split_plain_text_units(text: str) -> list[tuple[str, int, str]]:
@@ -695,7 +729,7 @@ def chunk_by_dom(html: str) -> list[Chunk]:
             ]
 
     parser = _BlockTextExtractor()
-    parser.feed(html)
+    parser.feed(_normalize_metric_markup(html))
     entries = parser.finished()
     chunks: list[Chunk] = []
     for index, (

tests/test_chunking.py

@@ -191,6 +191,45 @@ def test_chunk_by_dom_does_not_treat_non_numeric_superscript_as_footnote() -> No
     ]
 
 
+def test_chunk_by_dom_preserves_explicit_metric_superscripts() -> None:
+    """A unit exponent remains searchable mathematical evidence."""
+    chunks = chunk_by_dom("<p>Volume: 5m<sup>3</sup>.</p>")
+
+    assert [(chunk.label, chunk.text) for chunk in chunks] == [
+        ("p", "Volume: 5m³."),
+    ]
+
+
+def test_chunk_by_dom_preserves_explicit_metric_subscripts() -> None:
+    """A unit subscript is retained without changing ordinary footnotes."""
+    chunks = chunk_by_dom("<p>Index m<sub>3</sub> is measured.</p>")
+
+    assert [(chunk.label, chunk.text) for chunk in chunks] == [
+        ("p", "Index m₃ is measured."),
+    ]
+
+
+def test_chunk_by_source_body_normalizes_plain_metric_scripts() -> None:
+    """Plain-text metric scripts retain searchable exponent/index semantics."""
+    chunks = chunk_by_source_body("Volume: 5m^3; index m_3; braced m^{2}.")
+
+    assert [(chunk.label, chunk.text) for chunk in chunks] == [
+        ("", "Volume: 5m³; index m₃; braced m²."),
+    ]
+
+
+def test_chunk_by_source_body_normalizes_metric_scripts_in_markdown_table_cells() -> None:
+    """Markdown table cells retain the same searchable metric semantics as prose."""
+    chunks = chunk_by_source_body(
+        "| Metric | Index |\n| --- | --- |\n| 5m^3 | m_3 |"
+    )
+
+    assert [(chunk.label, chunk.text) for chunk in chunks] == [
+        ("tr", "Metric | Index"),
+        ("tr", "5m³ | m₃"),
+    ]
+
+
 def test_chunk_by_dom_preserves_nested_list_order_and_depth() -> None:
     """Nested list items retain source order and increasing depth."""
     chunks = chunk_by_dom(