diff --git a/docs/adr/0105-mathematical-script-semantic-normalization.md b/docs/adr/0105-mathematical-script-semantic-normalization.md new file mode 100644 index 000000000..304fc52f8 --- /dev/null +++ b/docs/adr/0105-mathematical-script-semantic-normalization.md @@ -0,0 +1,50 @@ +# ADR 0105: Preserve explicit metric scripts in semantic text + +**Status:** Accepted on this PR; not protected-main truth +**Date:** 2026-08-21 +**Owners:** LineageWeave ingestion and buyer-surface maintainers + +## Context + +Source posts commonly encode a unit such as `m3`, `m3`, +`m^3`, or `m_3` with HTML or plain-text notation. Dropping the markup changes +the searchable meaning to `m3`, while treating every numeric `sup` element as +mathematics would break the existing numeric-footnote contract. Full MathML +parsing is not yet justified by the current product surface, but the loss of +explicit unit scripts is a buyer-visible defect. + +MathML 4 defines `msup`, `msub`, and `msubsup` as structural script elements; +HTML `sup`/`sub` are a permitted lighter-weight notation when detailed +mathematical markup is not required. This decision therefore adds a bounded +normalization boundary and keeps the source representation unchanged. + +## Decision + +1. Preserve the immutable source body exactly as imported. +2. In derived semantic text only, normalize an explicitly bounded metric base + (`m`, `cm`, `mm`, `km`, or `kg`, optionally preceded by a number) followed + by numeric `sup`/`sub` markup or plain-text `^`/`_` notation into Unicode + superscript/subscript digits. For example, `5m3` and `5m^3` + become `5m³`, while `m3` and `m_3` become `m₃`. +3. Keep ordinary numeric superscripts and caret expressions on prose under the existing footnote + role contract. Do not infer a mathematical formula from an arbitrary word. +4. Apply the same bounded normalization in backend semantic chunks and the + React buyer display so search text and visible text agree. +5. Defer full MathML/LaTeX parsing, expression trees, and ontology term + creation until an authorized fixture demonstrates a need beyond metric + scripts. Any such change requires a new ADR and parser contract. + +## Consequences + +- Search and the buyer popup retain the visible distinction between `m³` and + `m3` without exposing source HTML to the embedding model. +- Existing numeric-footnote tests remain unchanged because the bounded metric + pattern is the only new conversion. +- The current implementation does not claim to understand arbitrary equations; + unsupported script markup remains ordinary source text and must not be + presented as a parsed ontology expression. + +## References (APA 7th) + +World Wide Web Consortium. (2026). *Mathematical Markup Language (MathML) +Version 4.0* (W3C Recommendation). https://www.w3.org/TR/mathml4/ diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts index 0e85f6025..e7e47cf90 100644 --- a/frontend/src/postBodyDisplay.test.ts +++ b/frontend/src/postBodyDisplay.test.ts @@ -67,6 +67,18 @@ describe("splitPostBody", () => { ]); }); + it("preserves explicit metric superscripts and subscripts", () => { + expect(splitPostBody("

Volume: 5m3, index m3.

")).toEqual([ + { kind: "text", text: "Volume: 5m³, index m₃." }, + ]); + }); + + it("normalizes plain-text metric superscripts and subscripts", () => { + expect(splitPostBody("

Volume: 5m^3, index m_3, braced m^{2}.

")).toEqual([ + { kind: "text", text: "Volume: 5m³, index m₃, braced m²." }, + ]); + }); + it("preserves HTML and Word footnote blocks as footnote paragraphs", () => { expect( splitPostBody( @@ -128,6 +140,12 @@ describe("splitPostBody", () => { ]); }); + it("normalizes metric scripts inside Markdown table cells", () => { + expect( + splitMarkdownTableBody("| Metric | Index |\n| --- | --- |\n| 5m^3 | m3 |"), + ).toEqual([{ kind: "table", rows: [["Metric", "Index"], ["5m³", "m₃"]] }]); + }); + it("unescapes pipe characters inside Markdown cells without accepting a short delimiter", () => { expect( splitMarkdownTableBody( diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts index b08b4803e..588901483 100644 --- a/frontend/src/postBodyDisplay.ts +++ b/frontend/src/postBodyDisplay.ts @@ -32,6 +32,27 @@ const NUMERIC_FOOTNOTE_MARKER = "\u0003lw-numeric-footnote\u0004"; const FOOTNOTE_BLOCK_MARKER = "\u0005lw-footnote-block\u0006"; const FOOTNOTE_BLOCK_OPEN = /<\s*(?:footnote|endnote|w:footnote|w:endnote)\b[^>]*>/gi; const NUMERIC_SUPERSCRIPT = /]*>\s*(\d{1,3})\s*<\/sup>/gi; +const SUPERSCRIPT_DIGITS = "⁰¹²³⁴⁵⁶⁷⁸⁹"; +const SUBSCRIPT_DIGITS = "₀₁₂₃₄₅₆₇₈₉"; +const METRIC_MARKUP = + /((?]*>\s*(\d{1,3})\s*<\/\2>/gi; +const METRIC_PLAIN_SCRIPT = + /((? { + const table = kind.toLowerCase() === "sup" ? SUPERSCRIPT_DIGITS : SUBSCRIPT_DIGITS; + return `${base}${[...digits].map((digit) => table[Number(digit)]).join("")}`; + }) + .replace( + METRIC_PLAIN_SCRIPT, + (_match, base: string, kind: string, bracedDigits: string, digits: string) => { + const table = kind === "^" ? SUPERSCRIPT_DIGITS : SUBSCRIPT_DIGITS; + return `${base}${[...(bracedDigits || digits)].map((digit) => table[Number(digit)]).join("")}`; + }, + ); +} function stripIndentMarkers(value: string): string { return value @@ -224,7 +245,7 @@ function isDecodableBase64(raw: string): boolean { function pushText(segments: PostBodySegment[], raw: string, indentUnit: number): void { const text = stripHtmlTags( - raw + normalizeMetricMarkup(raw) .replace(FOOTNOTE_BLOCK_OPEN, FOOTNOTE_BLOCK_MARKER) .replace(NUMERIC_SUPERSCRIPT, `${NUMERIC_FOOTNOTE_MARKER}$1`), ); @@ -257,7 +278,7 @@ function markdownCells(line: string): string[] | null { const value = line.trim().replace(/^\|/, "").replace(/(? cell.trim().replace(/\\\|/g, "|")); - return cells.length >= 2 && cells.every(Boolean) ? cells : null; + return cells.length >= 2 && cells.every(Boolean) ? cells.map(normalizeMetricMarkup) : null; } function isMarkdownSeparatorRow(cells: string[] | null): boolean { diff --git a/lineageweave/chunking.py b/lineageweave/chunking.py index 541d18deb..7a8bbc28b 100644 --- a/lineageweave/chunking.py +++ b/lineageweave/chunking.py @@ -134,6 +134,7 @@ def _is_footnote_reference(attrs: list[tuple[str, str | None]]) -> bool: def normalize_semantic_text(text: str) -> str: """Remove visual hanging-indent breaks without changing source content.""" + text = _normalize_metric_markup(_normalize_plain_metric_scripts(text)) lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n") normalized: list[str] = [] for line in lines: @@ -296,6 +297,37 @@ def chunk_by_paragraph(text: str) -> list[Chunk]: _SENTENCE_BOUNDARY = re.compile(r"(?<=[.!?])\s+(?=[A-Z0-9가-힣])") +_SUPERSCRIPT_DIGITS = str.maketrans("0123456789", "⁰¹²³⁴⁵⁶⁷⁸⁹") +_SUBSCRIPT_DIGITS = str.maketrans("0123456789", "₀₁₂₃₄₅₆₇₈₉") +_METRIC_MARKUP = re.compile( + r"(?P(?sup|sub)\b[^>]*>\s*(?P\d{1,3})\s*", + re.IGNORECASE, +) +_METRIC_PLAIN_SCRIPT = re.compile( + r"(?P(?\^|_)\s*(?:\{(?P\d{1,3})\}|(?P\d{1,3}))", + re.IGNORECASE, +) + + +def _normalize_plain_metric_scripts(text: str) -> str: + """Normalize bounded plain-text metric exponents and indices.""" + def replace(match: re.Match[str]) -> str: + table = _SUPERSCRIPT_DIGITS if match.group("kind") == "^" else _SUBSCRIPT_DIGITS + digits = match.group("braced_digits") or match.group("digits") or "" + return f"{match.group('base')}{digits.translate(table)}" + + return _METRIC_PLAIN_SCRIPT.sub(replace, text) + + +def _normalize_metric_markup(html: str) -> str: + """Keep explicit metric superscript/subscript digits in semantic text.""" + def replace(match: re.Match[str]) -> str: + table = _SUPERSCRIPT_DIGITS if match.group("kind").lower() == "sup" else _SUBSCRIPT_DIGITS + return f"{match.group('base')}{match.group('digits').translate(table)}" + + return _METRIC_MARKUP.sub(replace, html) def chunk_by_sentence(text: str) -> list[Chunk]: @@ -554,8 +586,7 @@ def _markdown_cells(line: str) -> list[str] | None: if "|" not in line: return None value = line.strip() - if value.startswith("|"): - value = value[1:] + value = value.removeprefix("|") if value.endswith("|") and not value.endswith("\\|"): value = value[:-1] cells = [cell.strip().replace(r"\|", "|") for cell in re.split(r"(? None: found_table = True flush_pending() - entries.append(("markdown_tr", " | ".join(header))) + entries.append(("markdown_tr", " | ".join(normalize_semantic_text(cell) for cell in header))) index += 2 while index < len(lines) and lines[index].strip(): cells = _markdown_cells(lines[index]) if cells is None: break - entries.append(("markdown_tr", " | ".join(cells))) + entries.append(("markdown_tr", " | ".join(normalize_semantic_text(cell) for cell in cells))) index += 1 flush_pending() @@ -616,7 +647,10 @@ def _is_markdown_table_row(line: str) -> bool: def _render_markdown_table_row(line: str) -> str: """Keep Markdown table columns as searchable row evidence.""" - return " | ".join(cell.strip() for cell in line.strip().strip("|").split("|")) + return " | ".join( + normalize_semantic_text(cell.strip()) + for cell in line.strip().strip("|").split("|") + ) def _split_plain_text_units(text: str) -> list[tuple[str, int, str]]: @@ -695,7 +729,7 @@ def chunk_by_dom(html: str) -> list[Chunk]: ] parser = _BlockTextExtractor() - parser.feed(html) + parser.feed(_normalize_metric_markup(html)) entries = parser.finished() chunks: list[Chunk] = [] for index, ( diff --git a/tests/test_chunking.py b/tests/test_chunking.py index 55c3839eb..258bc9ad4 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -191,6 +191,45 @@ def test_chunk_by_dom_does_not_treat_non_numeric_superscript_as_footnote() -> No ] +def test_chunk_by_dom_preserves_explicit_metric_superscripts() -> None: + """A unit exponent remains searchable mathematical evidence.""" + chunks = chunk_by_dom("

Volume: 5m3.

") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("p", "Volume: 5m³."), + ] + + +def test_chunk_by_dom_preserves_explicit_metric_subscripts() -> None: + """A unit subscript is retained without changing ordinary footnotes.""" + chunks = chunk_by_dom("

Index m3 is measured.

") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("p", "Index m₃ is measured."), + ] + + +def test_chunk_by_source_body_normalizes_plain_metric_scripts() -> None: + """Plain-text metric scripts retain searchable exponent/index semantics.""" + chunks = chunk_by_source_body("Volume: 5m^3; index m_3; braced m^{2}.") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("", "Volume: 5m³; index m₃; braced m²."), + ] + + +def test_chunk_by_source_body_normalizes_metric_scripts_in_markdown_table_cells() -> None: + """Markdown table cells retain the same searchable metric semantics as prose.""" + chunks = chunk_by_source_body( + "| Metric | Index |\n| --- | --- |\n| 5m^3 | m_3 |" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("tr", "Metric | Index"), + ("tr", "5m³ | m₃"), + ] + + def test_chunk_by_dom_preserves_nested_list_order_and_depth() -> None: """Nested list items retain source order and increasing depth.""" chunks = chunk_by_dom(