From 17167d65c649e426a556dfe4bd6a33b5cf1d64c9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 09:22:15 +0000 Subject: [PATCH 01/10] feat: render quantity superscripts in post bodies (v2.12.9) Buyer-visible cubic metres arrived as HTML or m^3 and flattened to a caret or concatenated m3. Map quantity scripts to Unicode in semantic units and render React / runs without innerHTML (ADR 0119). Comparison operators and leading footnote carets stay literal. Synthetic fixtures only. --- AGENTS.md | 4 + CHANGELOG.d/2.12.9-quantity-superscripts.md | 8 ++ CHANGELOG.md | 8 ++ docs/adr/0119-quantity-script-display.md | 55 ++++++++ docs/adr/README.md | 2 +- docs/lineage-bi-research-notes.md | 18 +++ docs/product-technical-gap-baseline.md | 5 +- frontend/package.json | 2 +- frontend/src/App.css | 19 +++ frontend/src/PostBody.test.tsx | 23 +++ frontend/src/PostBody.tsx | 26 +++- frontend/src/postBodyDisplay.test.ts | 26 +++- frontend/src/postBodyDisplay.ts | 147 +++++++++++++++++++- lineageweave/__init__.py | 2 +- lineageweave/chunking.py | 120 +++++++++++++++- lineageweave/post_content_normalization.py | 1 + pyproject.toml | 2 +- tests/test_chunking.py | 23 ++- tests/test_post_content_normalization.py | 7 + uv.lock | 2 +- 20 files changed, 480 insertions(+), 20 deletions(-) create mode 100644 CHANGELOG.d/2.12.9-quantity-superscripts.md create mode 100644 docs/adr/0119-quantity-script-display.md diff --git a/AGENTS.md b/AGENTS.md index 1728f9e61..cb2da918a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -148,6 +148,10 @@ contextual-orchestrator owns model discovery and selection. from derived semantic text, while retaining the source body and meaningful list/heading nesting. A buyer-facing post view must render semantic paragraphs, not the authoring application's spacing workaround. +- Quantity HTML ``/`` and caret exponents such as `m^3` become + Unicode in derived units and React ``/`` in the post view + (ADR 0119). Never assign the body to `innerHTML`. Do not treat + `qty < 50` or a leading footnote `^1` as an exponent. - Image descriptions, OCR text, and region evidence are analysis artifacts, not buyer-facing prompt instructions. Buyer UI shows the source content and useful captions/evidence only, with provenance where appropriate. diff --git a/CHANGELOG.d/2.12.9-quantity-superscripts.md b/CHANGELOG.d/2.12.9-quantity-superscripts.md new file mode 100644 index 000000000..b2d3e3541 --- /dev/null +++ b/CHANGELOG.d/2.12.9-quantity-superscripts.md @@ -0,0 +1,8 @@ +# 2.12.9 — Quantity superscript display + +## Fixed + +- Post popups now show cubic metres and similar quantities as superscripts + and subscripts (`12 m³`, `H₂O`) instead of flattened `m^3` or `m3`. + Semantic units store Unicode so embeddings keep the exponent. Comparison + operators and leading footnote carets stay literal (ADR 0119). diff --git a/CHANGELOG.md b/CHANGELOG.md index c8ed1a099..7086e6ce8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,14 @@ All notable changes to this project are documented here. Format follows environment, so local OIDC and synthetic-data workflows resolve the same pinned dependencies as CI. +## [2.12.9] - 2026-08-22 + +### Fixed + +- Quantity superscripts and subscripts (`m³`, `H₂O`) now render as + text-level runs in the post popup and persist as Unicode in semantic + units. Comparison operators and leading footnote carets stay literal. + ## [2.12.6] - 2026-08-20 ### Added diff --git a/docs/adr/0119-quantity-script-display.md b/docs/adr/0119-quantity-script-display.md new file mode 100644 index 000000000..fdb902daf --- /dev/null +++ b/docs/adr/0119-quantity-script-display.md @@ -0,0 +1,55 @@ +# ADR 0119: Render quantity superscripts as text runs, Unicode in units + +- Status: Accepted +- Date: 2026-08-22 +- Depends on: [0061](0061-post-body-character-reference-decoding.md), [0062](0062-semantic-unit-embedding.md), [0102](0102-semantic-source-unit-boundaries.md) + +## Context + +Imported posts write cubic metres and similar quantities as HTML +``/`` or as caret exponents (`m^3`). The display path converted +`` into a caret and then rendered the paragraph as a React text node, +so buyers saw `m^3` instead of a superscript. The DOM chunker dropped the +tags entirely, so embeddings received concatenated `m3`, which is a +different quantity. Comparison operators such as `qty < 50` must remain +plain text, and a leading footnote caret (`^1 …`) is not a unit exponent. + +## Decision + +- Preserve the source body. Derived semantic text maps HTML ``/`` + and quantity caret exponents onto Unicode Super/Subscript characters + (The Unicode Consortium, 2024, §22.4) so search and embeddings can tell + `m³` from `m3` without keeping markup in the unit (Cai, Yu, Wen, & Ma, + 2003). +- The buyer post view splits those Unicode (or leftover caret) runs and + renders them as React ``/`` elements. The body is never + assigned to `innerHTML` (ADR 0061). +- Only a letter, digit, or closing `)` immediately followed by `^` and a + short numeric/`n` exponent is treated as a quantity. A leading `^1` + footnote marker and comparison operators stay literal. +- Unmapped script runs keep a caret or underscore so they remain visible + rather than silently concatenating. Full formula ontology remains out of + scope; this decision covers quantity display and unit-level text. + +## Consequences + +- The post popup shows `12 m³` and `H₂O` as superscripts and subscripts. +- Newly persisted `post_content_unit` text stores Unicode quantities, so + later retrieval does not depend on HTML surviving the chunker. +- Existing concatenated `m3` units stay until re-ingestion; caret-form + units still render through the display splitter. + +## References + +Cai, D., Yu, S., Wen, J.-R., & Ma, W.-Y. (2003). *VIPS: A vision-based +page segmentation algorithm* (Microsoft Research Technical Report +MSR-TR-2003-79). Microsoft Research. + +International Organization for Standardization. (2022). *Quantities and +units — Part 1: General* (ISO 80000-1:2022). + +The Unicode Consortium. (2024). *The Unicode Standard* (Version 16.0.0). +https://www.unicode.org/versions/Unicode16.0.0/ + +WHATWG. (n.d.). *HTML living standard: The `sub` and `sup` elements*. +https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-sub-and-sup-elements diff --git a/docs/adr/README.md b/docs/adr/README.md index 762f1c051..6cb58b93c 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -10,7 +10,7 @@ decision from them. | Supporting document | Normative ADR | |---|---| | [`product-technical-gap-baseline.md`](../product-technical-gap-baseline.md) | Product/technical traceability projection across the ADR set; ADRs remain normative | -| [`lineage-bi-research-notes.md`](../lineage-bi-research-notes.md) | [0084](0084-lineage-research-grounding.md), [0062](0062-semantic-unit-embedding.md), [0064](0064-lineage-evidence-and-tree-assembly.md) | +| [`lineage-bi-research-notes.md`](../lineage-bi-research-notes.md) | [0084](0084-lineage-research-grounding.md), [0062](0062-semantic-unit-embedding.md), [0064](0064-lineage-evidence-and-tree-assembly.md), [0119](0119-quantity-script-display.md) | | [`PROV_O_IMPLEMENTATION.md`](../PROV_O_IMPLEMENTATION.md) | [0065](0065-prov-o-provenance-boundary.md) | | [`PROV_O_IMPLEMENTATION_MATRIX.md`](../PROV_O_IMPLEMENTATION_MATRIX.md) | [0065](0065-prov-o-provenance-boundary.md) | | [`image-content-schema.md`](../image-content-schema.md) | [0066](0066-position-preserving-image-content.md) | diff --git a/docs/lineage-bi-research-notes.md b/docs/lineage-bi-research-notes.md index 94f99d73d..a3fe9e3ba 100644 --- a/docs/lineage-bi-research-notes.md +++ b/docs/lineage-bi-research-notes.md @@ -207,6 +207,8 @@ Anagnostopoulos, E., Batsakis, S., & Petrakis, E. G. M. (2013). CHRONOS: A reaso Browne, W. J., Goldstein, H., & Rasbash, J. (2001). Multiple membership multiple classification (MMMC) models. *Statistical Modelling*, *1*(2), 103-124. https://doi.org/10.1177/1471082X0100100202 +Cai, D., Yu, S., Wen, J.-R., & Ma, W.-Y. (2003). *VIPS: A vision-based page segmentation algorithm* (Microsoft Research Technical Report MSR-TR-2003-79). Microsoft Research. + Chang, J., & Blei, D. M. (2009). Relational topic models for document networks. In D. van Dyk & M. Welling (Eds.), *Proceedings of the 12th International Conference on Artificial Intelligence and Statistics* (pp. 81-88). PMLR. Christen, P. (2012). *Data matching: Concepts and techniques for record linkage, entity resolution, and duplicate detection*. Springer. https://doi.org/10.1007/978-3-642-31164-2 @@ -225,6 +227,8 @@ Gildea, D., & Jurafsky, D. (2002). Automatic labeling of semantic roles. *Comput Hearst, M. A. (1997). TextTiling: Segmenting text into multi-paragraph subtopic passages. *Computational Linguistics*, *23*(1), 33-64. +International Organization for Standardization. (2022). *Quantities and units — Part 1: General* (ISO 80000-1:2022). + Lewis, P., Perez, E., Piktus, A., Petroni, F., Karpukhin, V., Goyal, N., Küttler, H., Lewis, M., Yih, W., Rocktäschel, T., Riedel, S., & Kiela, D. (2020). Retrieval-augmented generation for knowledge-intensive NLP tasks. In H. Larochelle, M. Ranzato, R. Hadsell, M. F. Balcan, & H. Lin (Eds.), *Advances in Neural Information Processing Systems* (Vol. 33, pp. 9459-9474). Curran Associates. Li, M., Lv, T., Chen, J., Cui, L., Lu, Y., Florencio, D., Zhang, C., Li, Z., & Wei, F. (2023). TrOCR: Transformer-based optical character recognition with pre-trained models. *Proceedings of the AAAI Conference on Artificial Intelligence*, *37*(11), 13094-13102. https://doi.org/10.1609/aaai.v37i11.26538 @@ -241,6 +245,8 @@ Sun, Q., Yuan, J., He, S., Guan, X., Yuan, H., Fu, X., Li, J., & Yu, P. S. (2025 Tong, H., Faloutsos, C., & Pan, J.-Y. (2006). Fast random walk with restart and its applications. *Proceedings of the Sixth International Conference on Data Mining (ICDM'06)*, 613-622. https://doi.org/10.1109/ICDM.2006.70 +The Unicode Consortium. (2024). *The Unicode Standard* (Version 16.0.0). https://www.unicode.org/versions/Unicode16.0.0/ + Wang, Q., Fu, Y., Cao, Y., Wang, S., Tian, Z., & Ding, L. (2023). *Recursively summarizing enables long-term dialogue memory in large language models*. arXiv. https://arxiv.org/abs/2308.15022 WHATWG. (2026). *HTML Living Standard — sections 4.3 (sectioning content) and 4.4 (grouping content)*. https://html.spec.whatwg.org/ @@ -379,3 +385,15 @@ summarize-and-replace older turns instead of an unbounded transcript or a hard truncation that silently drops earlier decisions. This is recorded here as the citation this feature would build on, not as a claim that conversation-level compression is implemented today. + +## Quantity scripts in source units (ADR 0119) + +Board exports write cubic metres as HTML `` or as `m^3`. Flattening +those tags concatenates `m3`, which is a different quantity, and leaving +the caret in the buyer view hides the exponent. Derived units map a short +HTML/caret exponent onto Unicode Super/Subscript characters (The Unicode +Consortium, 2024, §22.4) so embeddings keep the unit (Cai, Yu, Wen, & Ma, +2003) while the post view renders React ``/`` instead of +`innerHTML`. ISO 80000-1 treats the exponent on a unit symbol as part of +the quantity, not decoration. Comparison operators and a leading footnote +caret stay literal. Full formula ontology is still open. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index e65883463..3904af44e 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -5,7 +5,10 @@ - **Table Parsing**: `post=00505695-3e61-1fd1-80c6-86bb61c8ddc5` completely fails at parsing tables. - **Indentation**: Incorrect indentation rendering in `post=00505695-7571-1fd1-83c3-d521b187ad5b` and `post=00505695-3e61-1fd1-83c0-497b3c1c455e`. - **Image/Table OCR**: `post=00505695-7571-1fd1-83dd-3d22a61a5734` fails text recognition for tables inside images, markdown parsing fails, and image OCR description is too shallow for Ontology & Semantics. -- **Math/Superscripts**: `post=00505695-9612-1fe1-83a7-e30153323f25` fails to parse superscripts like m^3 properly. Needs strict Ontology grammar for math formulas. +- **Math/Superscripts**: (Display/unit text, ADR 0119) Quantity HTML + ``/`` and caret exponents such as `m^3` now render as + superscripts and persist as Unicode in semantic units. Full formula + ontology grammar remains open. - **Missing UI Elements**: DAG (Directed Acyclic Graph) view is currently missing from the frontend for `post=00505695-7571-1fd1-83c5-895ed333cdbc`. ## 2. LLM Extraction & Knowledge Graph Gaps diff --git a/frontend/package.json b/frontend/package.json index e2e996bbe..dbf43b45f 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "frontend", "private": true, - "version": "2.12.6", + "version": "2.12.9", "type": "module", "scripts": { "dev": "vite", diff --git a/frontend/src/App.css b/frontend/src/App.css index c72aab078..58e2c371e 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -350,6 +350,25 @@ white-space: pre-wrap; } +.post-body-text sup, +.post-body-text sub, +.post-body-table sup, +.post-body-table sub { + font-size: 0.75em; + line-height: 0; + font-weight: 600; +} + +.post-body-text sup, +.post-body-table sup { + vertical-align: super; +} + +.post-body-text sub, +.post-body-table sub { + vertical-align: sub; +} + .post-embedded-image { margin: 0; padding: var(--post-image-padding); diff --git a/frontend/src/PostBody.test.tsx b/frontend/src/PostBody.test.tsx index 1a9b00c4a..76837b5b2 100644 --- a/frontend/src/PostBody.test.tsx +++ b/frontend/src/PostBody.test.tsx @@ -318,4 +318,27 @@ describe("PostBody", () => { expect(screen.getByText("Before").compareDocumentPosition(screen.getByAltText("Source diagram")) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); expect(screen.getByAltText("Source diagram").compareDocumentPosition(screen.getByText("After")) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); }); + + it("renders quantity superscripts as text-level sup without using innerHTML", () => { + const { container } = render(); + + const paragraph = container.querySelector("p.post-body-text"); + const superscript = paragraph?.querySelector("sup"); + expect(superscript?.textContent).toBe("3"); + expect(paragraph?.textContent).toBe("Tank volume is 12 m3."); + }); + + it("renders caret exponents and subscripts from mixed source text", () => { + const { container } = render(); + + expect(container.querySelector("sub")?.textContent).toBe("2"); + expect(container.querySelector("sup")?.textContent).toBe("3"); + }); + + it("keeps comparison operators visible as ordinary text", () => { + render(); + + expect(screen.getByText("Need delivery if qty < 50 and price > 10.")).toBeInTheDocument(); + expect(document.querySelector("sup")).not.toBeInTheDocument(); + }); }); diff --git a/frontend/src/PostBody.tsx b/frontend/src/PostBody.tsx index 6769d0192..080d375c5 100644 --- a/frontend/src/PostBody.tsx +++ b/frontend/src/PostBody.tsx @@ -1,7 +1,19 @@ -import { splitPostBody, type PostBodySegment } from "./postBodyDisplay"; +import { splitPostBody, splitScriptRuns, normalizeScriptText, type PostBodySegment } from "./postBodyDisplay"; import { t } from "./i18n"; import type { PostContentUnit, PostImageContent } from "./api"; -import type { ReactNode } from "react"; +import { Fragment, type ReactNode } from "react"; + +function renderStyledText(text: string): ReactNode { + return splitScriptRuns(text).map((run, index) => { + if (run.script === "super") { + return {run.text}; + } + if (run.script === "sub") { + return {run.text}; + } + return {run.text}; + }); +} function parsePipeDelimitedTable(text: string): string[][] | null { const rows = text @@ -21,14 +33,14 @@ function parsePipeDelimitedTable(text: string): string[][] | null { function renderImageText(text: string) { const rows = parsePipeDelimitedTable(text); - if (!rows) return

{text}

; + if (!rows) return

{renderStyledText(text)}

; return ( {rows.map((row, rowIndex) => ( {row.map((cell, cellIndex) => ( - + ))} ))} @@ -102,7 +114,7 @@ function renderSegment(segment: PostBodySegment, index: number, imageContent?: P : undefined } > - {segment.text} + {renderStyledText(segment.text)}

); case "image": @@ -129,7 +141,7 @@ function isStructuredTableRow(unit: PostContentUnit): boolean { * indentation for every later unresolved unit. */ function normalizedUnitText(value: string): string { - return value.replace(/\s+/g, " ").trim(); + return normalizeScriptText(value.replace(/\s+/g, " ").trim()); } /** @@ -220,7 +232,7 @@ function renderStructuredUnits( {rows.map((row, rowIndex) => ( {row.unit_text.split(/\s*\|\s*/).map((cell, cellIndex) => ( - + ))} ))} diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts index 07ca9514d..b0f89a249 100644 --- a/frontend/src/postBodyDisplay.test.ts +++ b/frontend/src/postBodyDisplay.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { splitPostBody } from "./postBodyDisplay"; +import { splitPostBody, splitScriptRuns, normalizeScriptText } from "./postBodyDisplay"; /** 1x1 transparent PNG — the same synthetic fixture the Python vision tests use. */ const TINY_PNG_B64 = @@ -132,4 +132,28 @@ describe("splitPostBody", () => { ); expect(JSON.stringify(segments)).not.toContain("https://example.test"); }); + + it("turns HTML and caret quantity exponents into unicode without flattening them", () => { + expect(splitPostBody("

Tank volume is 12 m3.

")).toEqual([ + { kind: "text", text: "Tank volume is 12 m³." }, + ]); + expect(splitPostBody("Tank volume is 12 m^3.")).toEqual([ + { kind: "text", text: "Tank volume is 12 m³." }, + ]); + expect(splitPostBody("Coolant is H2O at 10^{-3} M.")).toEqual([ + { kind: "text", text: "Coolant is H₂O at 10⁻³ M." }, + ]); + }); + + it("does not treat a leading footnote caret or a comparison as an exponent", () => { + expect(splitPostBody("^1 See the tank note.")).toEqual([ + { kind: "text", text: "^1 See the tank note." }, + ]); + expect(normalizeScriptText("qty < 50 and price > 10")).toBe("qty < 50 and price > 10"); + expect(splitScriptRuns("Tank volume is 12 m³.")).toEqual([ + { text: "Tank volume is 12 m" }, + { text: "3", script: "super" }, + { text: "." }, + ]); + }); }); diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts index 919e8c0ca..4db78cf41 100644 --- a/frontend/src/postBodyDisplay.ts +++ b/frontend/src/postBodyDisplay.ts @@ -89,9 +89,150 @@ function indentMarker(width: number): string { return width > 0 ? `${INDENT_MARKER}${width}${INDENT_MARKER_END}` : ""; } +const SUPER_ASCII_TO_UNI: Record = { + "0": "⁰", + "1": "¹", + "2": "²", + "3": "³", + "4": "⁴", + "5": "⁵", + "6": "⁶", + "7": "⁷", + "8": "⁸", + "9": "⁹", + "+": "⁺", + "-": "⁻", + "=": "⁼", + "(": "⁽", + ")": "⁾", + n: "ⁿ", + N: "ⁿ", + i: "ⁱ", + I: "ⁱ", +}; +const SUB_ASCII_TO_UNI: Record = { + "0": "₀", + "1": "₁", + "2": "₂", + "3": "₃", + "4": "₄", + "5": "₅", + "6": "₆", + "7": "₇", + "8": "₈", + "9": "₉", + "+": "₊", + "-": "₋", + "=": "₌", + "(": "₍", + ")": "₎", + a: "ₐ", + e: "ₑ", + h: "ₕ", + i: "ᵢ", + k: "ₖ", + l: "ₗ", + m: "ₘ", + n: "ₙ", + o: "ₒ", + p: "ₚ", + s: "ₛ", + t: "ₜ", + x: "ₓ", +}; +const SUPER_UNI_TO_ASCII = Object.fromEntries( + Object.entries(SUPER_ASCII_TO_UNI).map(([ascii, uni]) => [uni, ascii]), +); +const SUB_UNI_TO_ASCII = Object.fromEntries( + Object.entries(SUB_ASCII_TO_UNI).map(([ascii, uni]) => [uni, ascii]), +); +const CARET_EXPONENT = + /(?<=[A-Za-z0-9µμ°ΩÅåÅ)])\^(?:\{([+-]?\d{1,3}|[nNiI])\}|([+-]?\d{1,3}|[nNiI]))/g; + +function applyUnicodeScript(text: string, kind: "super" | "sub"): string { + const table = kind === "super" ? SUPER_ASCII_TO_UNI : SUB_ASCII_TO_UNI; + const values = new Set(Object.values(table)); + const compact = text.trim(); + if (!compact) return text; + if ([...compact].every((ch) => ch in table || values.has(ch) || /\s/.test(ch))) { + return [...text].map((ch) => table[ch] ?? ch).join(""); + } + const prefix = kind === "super" ? "^" : "_"; + const leading = text.match(/^\s*/)?.[0] ?? ""; + const trailing = compact.length ? text.slice(leading.length + compact.length) : ""; + return `${leading}${prefix}${compact}${trailing}`; +} + +function replaceHtmlScripts(text: string): string { + return text + .replace(/]*>(.*?)<\/sup>/gi, (_match, inner: string) => + applyUnicodeScript(String(inner).replace(/<[^>]+>/g, ""), "super"), + ) + .replace(/]*>(.*?)<\/sub>/gi, (_match, inner: string) => + applyUnicodeScript(String(inner).replace(/<[^>]+>/g, ""), "sub"), + ); +} + +export function normalizeScriptText(text: string): string { + return replaceHtmlScripts(text).replace(CARET_EXPONENT, (_match, braced: string, bare: string) => + applyUnicodeScript(braced || bare, "super"), + ); +} + +export type ScriptRun = { text: string; script?: "super" | "sub" }; + +export function splitScriptRuns(text: string): ScriptRun[] { + const runs: ScriptRun[] = []; + const push = (chunk: string, script?: "super" | "sub") => { + if (!chunk) return; + const last = runs[runs.length - 1]; + if (last && last.script === script) { + last.text += chunk; + return; + } + runs.push(script ? { text: chunk, script } : { text: chunk }); + }; + let index = 0; + while (index < text.length) { + const ch = text[index]; + if (ch in SUPER_UNI_TO_ASCII) { + let ascii = SUPER_UNI_TO_ASCII[ch]; + index += 1; + while (index < text.length && text[index] in SUPER_UNI_TO_ASCII) { + ascii += SUPER_UNI_TO_ASCII[text[index]]; + index += 1; + } + push(ascii, "super"); + continue; + } + if (ch in SUB_UNI_TO_ASCII) { + let ascii = SUB_UNI_TO_ASCII[ch]; + index += 1; + while (index < text.length && text[index] in SUB_UNI_TO_ASCII) { + ascii += SUB_UNI_TO_ASCII[text[index]]; + index += 1; + } + push(ascii, "sub"); + continue; + } + if (ch === "^" && index > 0 && /[A-Za-z0-9µμ°ΩÅåÅ)]/.test(text[index - 1])) { + const rest = text.slice(index); + const match = rest.match(/^\^(?:\{([+-]?\d{1,3}|[nNiI])\}|([+-]?\d{1,3}|[nNiI]))/); + if (match) { + push(match[1] || match[2] || "", "super"); + index += match[0].length; + continue; + } + } + push(ch); + index += 1; + } + return runs; +} + function stripHtmlTags(text: string): string { - text = text.replace(/]*>(.*?)<\/sup>/gi, "^$1"); - const withBoundaries = text + const withScripts = replaceHtmlScripts(text); + const withBoundaries = withScripts .replace(BREAK_TAG, "\n") .replace(BLOCK_TAG, (tag) => { if (/^<\//.test(tag)) return "\n\n"; @@ -102,7 +243,7 @@ function stripHtmlTags(text: string): string { /^<\/?w:/i.test(tag) ? "" : " ", ); const decoded = decodeHtmlEntities(withoutTags); - return decoded + return normalizeScriptText(decoded) .split("\n") .map((line) => { if (!line.trim()) return ""; diff --git a/lineageweave/__init__.py b/lineageweave/__init__.py index 95330cb50..edc0fd4a9 100644 --- a/lineageweave/__init__.py +++ b/lineageweave/__init__.py @@ -55,4 +55,4 @@ "sentence_excerpts", ] -__version__ = "2.12.6" +__version__ = "2.12.9" diff --git a/lineageweave/chunking.py b/lineageweave/chunking.py index 6d1b7ccbc..b912de89e 100644 --- a/lineageweave/chunking.py +++ b/lineageweave/chunking.py @@ -130,6 +130,109 @@ def _is_footnote_reference(attrs: list[tuple[str, str | None]]) -> bool: ) +# Unicode Super/Subscript blocks (The Unicode Consortium, 2024, §22.4) plus the +# Latin-1 superscript digits. Quantity display uses these so embeddings keep +# "m³" distinct from "m3" without retaining HTML in the semantic text (ADR 0119). +_SUPERSCRIPT = { + "0": "\u2070", + "1": "\u00b9", + "2": "\u00b2", + "3": "\u00b3", + "4": "\u2074", + "5": "\u2075", + "6": "\u2076", + "7": "\u2077", + "8": "\u2078", + "9": "\u2079", + "+": "\u207a", + "-": "\u207b", + "=": "\u207c", + "(": "\u207d", + ")": "\u207e", + "n": "\u207f", + "N": "\u207f", + "i": "\u2071", + "I": "\u2071", +} +_SUBSCRIPT = { + "0": "\u2080", + "1": "\u2081", + "2": "\u2082", + "3": "\u2083", + "4": "\u2084", + "5": "\u2085", + "6": "\u2086", + "7": "\u2087", + "8": "\u2088", + "9": "\u2089", + "+": "\u208a", + "-": "\u208b", + "=": "\u208c", + "(": "\u208d", + ")": "\u208e", + "a": "\u2090", + "e": "\u2091", + "h": "\u2095", + "i": "\u1d62", + "k": "\u2096", + "l": "\u2097", + "m": "\u2098", + "n": "\u2099", + "o": "\u2092", + "p": "\u209a", + "s": "\u209b", + "t": "\u209c", + "x": "\u2093", +} +_SUPERSCRIPT_VALUES = frozenset(_SUPERSCRIPT.values()) +_SUBSCRIPT_VALUES = frozenset(_SUBSCRIPT.values()) +_INLINE_SCRIPT_TAGS = frozenset({"sup", "sub"}) +_HTML_SUP = re.compile(r"]*>(.*?)", re.IGNORECASE | re.DOTALL) +_HTML_SUB = re.compile(r"]*>(.*?)", re.IGNORECASE | re.DOTALL) +_INNER_TAG = re.compile(r"<[^>]+>") +# Quantity caret after a unit/digit, not a leading footnote marker such as `^1`. +_CARET_EXPONENT = re.compile( + r"(?<=[A-Za-z0-9µμ°ΩÅåÅ)])\^(?:\{([+\-]?\d{1,3}|[nNiI])\}|([+\-]?\d{1,3}|[nNiI]))" +) + + +def apply_unicode_script(text: str, kind: str) -> str: + """Map a short exponent/index run to Unicode, or keep a caret/underscore.""" + table = _SUPERSCRIPT if kind == "sup" else _SUBSCRIPT + values = _SUPERSCRIPT_VALUES if kind == "sup" else _SUBSCRIPT_VALUES + compact = text.strip() + if not compact: + return text + if all(ch in table or ch in values or ch.isspace() for ch in compact): + return "".join(table.get(ch, ch) for ch in text) + prefix = "^" if kind == "sup" else "_" + leading_len = len(text) - len(text.lstrip()) + trailing_len = len(text) - len(text.rstrip()) + leading = text[:leading_len] + trailing = text[len(text) - trailing_len :] if trailing_len else "" + return f"{leading}{prefix}{compact}{trailing}" + + +def _replace_html_script(match: re.Match[str], kind: str) -> str: + inner = _INNER_TAG.sub("", match.group(1)) + for _ in range(3): + decoded = unescape(inner) + if decoded == inner: + break + inner = decoded + return apply_unicode_script(inner, kind) + + +def normalize_script_text(text: str) -> str: + """Turn HTML/caret quantity scripts into Unicode without treating comparisons as tags.""" + replaced = _HTML_SUP.sub(lambda match: _replace_html_script(match, "sup"), text) + replaced = _HTML_SUB.sub(lambda match: _replace_html_script(match, "sub"), replaced) + return _CARET_EXPONENT.sub( + lambda match: apply_unicode_script(match.group(1) or match.group(2), "sup"), + replaced, + ) + + def normalize_semantic_text(text: str) -> str: """Remove visual hanging-indent breaks without changing source content.""" lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n") @@ -144,7 +247,7 @@ def normalize_semantic_text(text: str) -> str: normalized[-1] = f"{normalized[-1]} {stripped}" else: normalized.append(stripped) - return "\n".join(normalized).strip() + return normalize_script_text("\n".join(normalized).strip()) def _source_indent_width(text: str) -> int: @@ -343,6 +446,7 @@ def __init__(self) -> None: super().__init__() self._stack: list[tuple[str, list[str], str | None, int, bool]] = [] self._unscoped_buffer: list[str] = [] + self._script_stack: list[str] = [] # Each entry is ("text", str, tag_name, style) or # ("image", (mime_type, bytes), "", None) -- a single sequence in # true document order, so an image's index among its siblings @@ -351,6 +455,9 @@ def __init__(self) -> None: def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: """Collect relevant text state when an HTML start tag is encountered.""" + if tag in _INLINE_SCRIPT_TAGS: + self._script_stack.append(tag) + return if tag == "img": src = next((value for name, value in attrs if name == "src" and value), None) if src: @@ -401,11 +508,18 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: """Handle self-closing block tags without losing XML indentation state.""" self.handle_starttag(tag, attrs) - if tag in _DOM_BLOCK_TAGS: + if tag in _DOM_BLOCK_TAGS or tag in _INLINE_SCRIPT_TAGS: self.handle_endtag(tag) def handle_endtag(self, tag: str) -> None: """Close the relevant text state when an HTML end tag is encountered.""" + if tag in _INLINE_SCRIPT_TAGS: + if tag in self._script_stack: + while self._script_stack: + closed = self._script_stack.pop() + if closed == tag: + break + return if tag in _DOM_BLOCK_TAGS and self._stack and self._stack[-1][0] == tag: declared_width = sum(entry[3] for entry in self._stack) tag_name, buffer, style, _, is_footnote = self._stack.pop() @@ -447,6 +561,8 @@ def handle_data(self, data: str) -> None: text = decoded had_nbsp = "\xa0" in text text = text.replace("\xa0", " ") + if self._script_stack: + text = apply_unicode_script(text, self._script_stack[-1]) if self._stack and (text.strip() or had_nbsp): self._stack[-1][1].append(text) elif text.strip() or had_nbsp: diff --git a/lineageweave/post_content_normalization.py b/lineageweave/post_content_normalization.py index 196b7e7b4..000d06c40 100644 --- a/lineageweave/post_content_normalization.py +++ b/lineageweave/post_content_normalization.py @@ -42,6 +42,7 @@ _HTML_OPEN_TAG = re.compile( r"<\s*/?\s*(?:article|section|nav|aside|header|footer|div|p|li|td|th|tr|" r"table|blockquote|h[1-6]|img|br|hr|ul|ol|span|strong|em|b|i|u|a|" + r"sup|sub|" r"html|body|head|style|script|font|center|pre)\b", re.IGNORECASE, ) diff --git a/pyproject.toml b/pyproject.toml index cb4be2916..b16184f77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "lineageweave" -version = "2.12.6" +version = "2.12.9" description = "Reconstructs git-branch-style lineage DAGs from scattered short records using multi-channel score fusion and LLM adjudication." readme = "README.md" license = { text = "MIT" } diff --git a/tests/test_chunking.py b/tests/test_chunking.py index d37a300cc..c2121e71c 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -7,6 +7,7 @@ chunk_by_paragraph, chunk_by_sentence, chunk_by_source_body, + normalize_script_text, normalize_semantic_text, ) @@ -124,7 +125,7 @@ def test_chunk_by_dom_labels_html_and_word_footnote_markup() -> None: assert [(chunk.label, chunk.text) for chunk in chunks] == [ ("p", "Body text"), ("footnote", "HTML footnote body"), - ("footnote", "1 Word footnote body"), + ("footnote", "¹ Word footnote body"), ] @@ -410,3 +411,23 @@ def test_chunk_by_dom_preserves_style_per_block_independently() -> None: assert chunks[0].style == "color:blue" assert chunks[1].style is None + + +def test_normalize_script_text_maps_quantity_exponents_and_leaves_comparisons() -> None: + assert normalize_script_text("Tank volume is 12 m3.") == "Tank volume is 12 m³." + assert normalize_script_text("Tank volume is 12 m^3.") == "Tank volume is 12 m³." + assert normalize_script_text("Coolant is H2O.") == "Coolant is H₂O." + assert normalize_script_text("qty < 50 and price > 10") == "qty < 50 and price > 10" + assert normalize_script_text("^1 See the tank note.") == "^1 See the tank note." + + +def test_chunk_by_dom_keeps_html_quantity_scripts_as_unicode() -> None: + chunks = chunk_by_dom("

Tank volume is 12 m3 of H2O.

") + + assert [chunk.text for chunk in chunks] == ["Tank volume is 12 m³ of H₂O."] + + +def test_chunk_by_source_body_maps_plain_caret_quantities() -> None: + chunks = chunk_by_source_body("Reserve 12 m^3 and 10^{-3} M stock.") + + assert chunks[0].text == "Reserve 12 m³ and 10⁻³ M stock." diff --git a/tests/test_post_content_normalization.py b/tests/test_post_content_normalization.py index 0beead6f4..3c7c7c0d7 100644 --- a/tests/test_post_content_normalization.py +++ b/tests/test_post_content_normalization.py @@ -375,6 +375,13 @@ def test_comparison_operators_in_plain_text_are_not_treated_as_html() -> None: assert result.formatting_hints == () +def test_quantity_superscripts_normalize_to_unicode_for_embeddings() -> None: + html = normalize_post_body("

Tank volume is 12 m3.

") + assert html.text == "Tank volume is 12 m³." + caret = normalize_post_body("Tank volume is 12 m^3.") + assert caret.text == "Tank volume is 12 m³." + + def test_image_gets_an_explicit_placeholder_when_no_vision_client_is_available() -> None: b64 = base64.b64encode(_PNG_1X1).decode("ascii") html = f'' diff --git a/uv.lock b/uv.lock index 10bcf9ff1..62f71a5c9 100644 --- a/uv.lock +++ b/uv.lock @@ -454,7 +454,7 @@ wheels = [ [[package]] name = "lineageweave" -version = "2.12.6" +version = "2.12.9" source = { editable = "." } dependencies = [ { name = "certifi" }, From 8b8fa765270e0376e83faece6081be763322d067 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 19:37:45 +0900 Subject: [PATCH 02/10] fix(frontend): use OIDC return-url helpers and guard AdminPanel render Same root cause as ContextualWisdomLab/LineageWeave#426: the login button built an unsanitized returnUrl inline instead of the safe returnUrlFromLocation()/rememberOidcReturnUrl() pair (open-redirect guard plus the sessionStorage/localStorage fallback main.tsx already reads back on the OIDC callback), and AdminPanel's accessToken prop (string, required) was rendered from a variable that is string | undefined at that scope -- a real tsc error, not the intended authenticated-only render path. --- frontend/src/App.test.tsx | 3 +++ frontend/src/App.tsx | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/frontend/src/App.test.tsx b/frontend/src/App.test.tsx index 7462abd2c..70eb27590 100644 --- a/frontend/src/App.test.tsx +++ b/frontend/src/App.test.tsx @@ -41,6 +41,9 @@ describe("App, unauthenticated", () => { state: expect.objectContaining({ returnUrl: expect.stringMatching(/^\//) }), }), ); + // Persisted as a fallback in case the OIDC state round-trip is dropped + // (see oidcReturnUrl.ts's restoreOidcReturnUrl, consumed in main.tsx). + expect(window.sessionStorage.getItem("lineageweave.oidc.returnUrl")).toMatch(/^\//); }); }); diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 6fba0dd41..6e52be55d 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -4610,7 +4610,8 @@ export default function App({ showLabPanels = false }: { showLabPanels?: boolean
- {destination === "admin" ? : null}
@@ -4690,7 +4690,7 @@ export default function App({ showLabPanels = false }: { showLabPanels?: boolean }} /> ) : null} - {destination === "admin" ? : null} + {destination === "admin" && accessToken ? : null}
From d716da59eb94e52d0de1b73c8762eefcfcc7c557 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 20:15:50 +0900 Subject: [PATCH 03/10] fix: preserve superscript exponent case and stop unclosed-tag bleed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two real bugs flagged by review (PR #427): - SUPER_ASCII_TO_UNI/SUB_ASCII_TO_UNI map both lower- and uppercase ASCII letters to the same Unicode script character (n/N -> ⁿ, i/I -> ⁱ). Building the reverse table with the last-inserted key winning meant decoding a stored ⁿ always produced uppercase N regardless of what was actually stored -- "m^n" round-tripped as "m^N". Keep the first (lowercase) mapping instead. - An unclosed / tag never reaches handle_endtag, so nothing ever popped it off _script_stack; HTMLParser does not implicitly close it at a block boundary the way a browser would. Every later text run in the document then got treated as still inside the sup/ sub context, corrupting unrelated later paragraphs. Clear the stack in _finish_block, the single chokepoint every block boundary (a sibling block opening, this block's own endtag, or EOF) already routes through. --- frontend/src/postBodyDisplay.test.ts | 21 +++++++++++++++++++++ frontend/src/postBodyDisplay.ts | 20 ++++++++++++++------ lineageweave/chunking.py | 7 +++++++ tests/test_chunking.py | 17 +++++++++++++++++ 4 files changed, 59 insertions(+), 6 deletions(-) diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts index b0f89a249..f8d91f6e2 100644 --- a/frontend/src/postBodyDisplay.test.ts +++ b/frontend/src/postBodyDisplay.test.ts @@ -156,4 +156,25 @@ describe("splitPostBody", () => { { text: "." }, ]); }); + + it("decodes a stored superscript letter deterministically to lowercase", () => { + // "n" and "N" both encode to the same Unicode "ⁿ" (there is no distinct + // uppercase superscript N), so decoding must pick one case consistently + // rather than depending on object key iteration order (regression: used + // to always decode to uppercase because "N"/"I" were inserted after + // "n"/"i" in the forward table). + expect(splitScriptRuns("mⁿ")).toEqual([ + { text: "m" }, + { text: "n", script: "super" }, + ]); + expect(splitScriptRuns("xⁱ")).toEqual([ + { text: "x" }, + { text: "i", script: "super" }, + ]); + expect(splitPostBody("mN")).toEqual([{ kind: "text", text: "mⁿ" }]); + expect(splitScriptRuns(normalizeScriptText("mN"))).toEqual([ + { text: "m" }, + { text: "n", script: "super" }, + ]); + }); }); diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts index 4db78cf41..cd2a854e1 100644 --- a/frontend/src/postBodyDisplay.ts +++ b/frontend/src/postBodyDisplay.ts @@ -140,12 +140,20 @@ const SUB_ASCII_TO_UNI: Record = { t: "ₜ", x: "ₓ", }; -const SUPER_UNI_TO_ASCII = Object.fromEntries( - Object.entries(SUPER_ASCII_TO_UNI).map(([ascii, uni]) => [uni, ascii]), -); -const SUB_UNI_TO_ASCII = Object.fromEntries( - Object.entries(SUB_ASCII_TO_UNI).map(([ascii, uni]) => [uni, ascii]), -); +// Two ASCII keys can map to the same Unicode character (e.g. "n" and "N" +// both produce "ⁿ"). Building the reverse table naively lets the +// last-inserted ASCII key win, so decoding always yields one fixed case +// regardless of what was actually stored. Keep the first (lowercase, since +// it is listed first above) mapping instead, so round-tripping preserves case. +function buildUnicodeToAsciiTable(table: Record): Record { + const reverse: Record = {}; + for (const [ascii, uni] of Object.entries(table)) { + if (!(uni in reverse)) reverse[uni] = ascii; + } + return reverse; +} +const SUPER_UNI_TO_ASCII = buildUnicodeToAsciiTable(SUPER_ASCII_TO_UNI); +const SUB_UNI_TO_ASCII = buildUnicodeToAsciiTable(SUB_ASCII_TO_UNI); const CARET_EXPONENT = /(?<=[A-Za-z0-9µμ°ΩÅåÅ)])\^(?:\{([+-]?\d{1,3}|[nNiI])\}|([+-]?\d{1,3}|[nNiI]))/g; diff --git a/lineageweave/chunking.py b/lineageweave/chunking.py index b912de89e..65fd16827 100644 --- a/lineageweave/chunking.py +++ b/lineageweave/chunking.py @@ -534,6 +534,13 @@ def _finish_block( is_footnote: bool = False, ) -> None: """Emit one block buffer, including a block closed only at EOF.""" + # An unclosed / never reaches handle_endtag, so nothing else + # pops it off _script_stack. Every block boundary (a sibling block + # opening, this block's own endtag, or EOF) routes through here, so + # clearing here stops a dangling script tag from bleeding into later, + # unrelated blocks -- mirroring how a browser would not let inline + # formatting survive a block-level boundary. + self._script_stack.clear() raw_text = "".join(buffer) for raw_unit, source_indent in _split_dom_units(raw_text): text = normalize_semantic_text(raw_unit) diff --git a/tests/test_chunking.py b/tests/test_chunking.py index c2121e71c..f3b000cda 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -427,6 +427,23 @@ def test_chunk_by_dom_keeps_html_quantity_scripts_as_unicode() -> None: assert [chunk.text for chunk in chunks] == ["Tank volume is 12 m³ of H₂O."] +def test_chunk_by_dom_unclosed_sup_does_not_corrupt_later_paragraphs() -> None: + """A malformed, never-closed must not leak its script context into + every later block. HTMLParser (unlike a browser) does not implicitly + close an unclosed inline tag at a block boundary, so a naive + _script_stack would otherwise stay "open" for the rest of the document.""" + html = ( + "

Tank volume is 12 m3

" + "

Unrelated paragraph mentions n2 and o2 plainly.

" + ) + chunks = chunk_by_dom(html) + + assert [chunk.text for chunk in chunks] == [ + "Tank volume is 12 m³", + "Unrelated paragraph mentions n2 and o2 plainly.", + ] + + def test_chunk_by_source_body_maps_plain_caret_quantities() -> None: chunks = chunk_by_source_body("Reserve 12 m^3 and 10^{-3} M stock.") From 534de7ceb59d4c98b8540ef2f6db51beef351a1a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 15:51:58 +0900 Subject: [PATCH 04/10] fix(frontend): decode entities and match multiline sup/sub content replaceHtmlScripts checked convertibility of raw, un-decoded / inner HTML, so an entity like   inside the tag (common in Word/Outlook/Google Docs HTML exports) made the digit run fail the "all convertible" check and fall back to a literal caret instead of a superscript, unlike the Python backend (lineageweave/chunking.py), which unescapes entities before mapping. Decode entities on the captured inner text first, mirroring the Python path. Also add the dotAll flag to the / regexes so tag content split across a newline (common in pretty-printed HTML exports) still matches, matching the Python regexes' re.DOTALL flag. --- frontend/src/postBodyDisplay.test.ts | 21 +++++++++++++++++++++ frontend/src/postBodyDisplay.ts | 8 ++++---- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts index f8d91f6e2..c974f3dfa 100644 --- a/frontend/src/postBodyDisplay.test.ts +++ b/frontend/src/postBodyDisplay.test.ts @@ -145,6 +145,27 @@ describe("splitPostBody", () => { ]); }); + it("decodes HTML entities inside a sup/sub tag before mapping to unicode", () => { + // Office-tool HTML export pads content with  . The raw, + // un-decoded " 3" must not fail the all-convertible check and fall + // back to a literal caret (regression: entities weren't decoded before + // the convertibility check, matching the Python backend which unescapes + // first). + expect(splitPostBody("

Volume is 12 m 3.

")).toEqual([ + { kind: "text", text: "Volume is 12 m ³." }, + ]); + }); + + it("matches sup/sub content split across a newline", () => { + // Pretty-printed source HTML puts tag content on its own line + // (regression: the regex lacked the dotAll flag, so `.` could not cross + // the newline and the whole tag passed through unmatched, leaving a + // plain un-superscripted "3" instead of "³"). + expect(splitPostBody("

Tank volume is 12 m\n3\n.

")).toEqual([ + { kind: "text", text: "Tank volume is 12 m ³ ." }, + ]); + }); + it("does not treat a leading footnote caret or a comparison as an exponent", () => { expect(splitPostBody("^1 See the tank note.")).toEqual([ { kind: "text", text: "^1 See the tank note." }, diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts index cd2a854e1..2773eadf2 100644 --- a/frontend/src/postBodyDisplay.ts +++ b/frontend/src/postBodyDisplay.ts @@ -173,11 +173,11 @@ function applyUnicodeScript(text: string, kind: "super" | "sub"): string { function replaceHtmlScripts(text: string): string { return text - .replace(/]*>(.*?)<\/sup>/gi, (_match, inner: string) => - applyUnicodeScript(String(inner).replace(/<[^>]+>/g, ""), "super"), + .replace(/]*>(.*?)<\/sup>/gis, (_match, inner: string) => + applyUnicodeScript(decodeHtmlEntities(String(inner).replace(/<[^>]+>/g, "")), "super"), ) - .replace(/]*>(.*?)<\/sub>/gi, (_match, inner: string) => - applyUnicodeScript(String(inner).replace(/<[^>]+>/g, ""), "sub"), + .replace(/]*>(.*?)<\/sub>/gis, (_match, inner: string) => + applyUnicodeScript(decodeHtmlEntities(String(inner).replace(/<[^>]+>/g, "")), "sub"), ); } From 0e0b0b7f49f3be76ca1afe8cb3f54bf082ca8413 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 17:29:02 +0900 Subject: [PATCH 05/10] fix(chunking): normalize markdown table scripts --- lineageweave/chunking.py | 2 +- tests/test_chunking.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lineageweave/chunking.py b/lineageweave/chunking.py index 65fd16827..6082ab250 100644 --- a/lineageweave/chunking.py +++ b/lineageweave/chunking.py @@ -660,7 +660,7 @@ def flush() -> None: flush() units.extend( ( - _render_markdown_table_row(row), + normalize_semantic_text(_render_markdown_table_row(row)), _source_indent_width(row), "tr", ) diff --git a/tests/test_chunking.py b/tests/test_chunking.py index f3b000cda..b11fd9500 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -220,7 +220,7 @@ def test_chunk_by_source_body_splits_plain_lists_and_markdown_tables() -> None: | Field | Value | | --- | --- | -| Owner | Buyer | +| Volume | 12 m^3 | """ chunks = chunk_by_source_body(body) @@ -229,7 +229,7 @@ def test_chunk_by_source_body_splits_plain_lists_and_markdown_tables() -> None: ("", "1. Background continuation stays with the first item."), ("", "2. Decision"), ("tr", "Field | Value"), - ("tr", "Owner | Buyer"), + ("tr", "Volume | 12 m³"), ] From 446ceddd2a447a970cfaf2b6b858e79a0efe4b0d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 17:44:40 +0900 Subject: [PATCH 06/10] fix: bound script markup to table cells --- lineageweave/chunking.py | 1 + tests/test_chunking.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/lineageweave/chunking.py b/lineageweave/chunking.py index 6082ab250..2caf21e0e 100644 --- a/lineageweave/chunking.py +++ b/lineageweave/chunking.py @@ -483,6 +483,7 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None self._stack[-1] = (tag_name, buffer, style, indent_width, True) return if tag in _TABLE_CELL_TAGS: + self._script_stack.clear() if self._stack and self._stack[-1][0] in _TABLE_ROW_TAGS and self._stack[-1][1]: self._stack[-1][1].append(" | ") return diff --git a/tests/test_chunking.py b/tests/test_chunking.py index b11fd9500..8d1c457c6 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -427,6 +427,12 @@ def test_chunk_by_dom_keeps_html_quantity_scripts_as_unicode() -> None: assert [chunk.text for chunk in chunks] == ["Tank volume is 12 m³ of H₂O."] +def test_chunk_by_dom_unclosed_sup_does_not_cross_table_cells() -> None: + chunks = chunk_by_dom("
{cell}{renderStyledText(cell)}
{cell}{renderStyledText(cell)}
m3Acme Corp
") + + assert [chunk.text for chunk in chunks] == ["m³ | Acme Corp"] + + def test_chunk_by_dom_unclosed_sup_does_not_corrupt_later_paragraphs() -> None: """A malformed, never-closed must not leak its script context into every later block. HTMLParser (unlike a browser) does not implicitly From f445be7ae34674b0e61d2870e4c67d9d1e61e602 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 11:40:07 -0700 Subject: [PATCH 07/10] fix: preserve deterministic source unit structure (#515) * fix: preserve deterministic source unit structure * fix: scope table cell positions per row * fix: close remaining semantic structure gaps * fix: preserve escaped non-script markup * fix: normalize encoded nested script markup * docs(adr): reserve unique quantity-script decision number --- AGENTS.md | 2 +- CHANGELOG.d/2.12.9-quantity-superscripts.md | 2 +- ...lay.md => 0165-quantity-script-display.md} | 2 +- docs/adr/README.md | 2 +- docs/lineage-bi-research-notes.md | 2 +- docs/product-technical-gap-baseline.md | 2 +- frontend/src/postBodyDisplay.test.ts | 65 ++++++++++- frontend/src/postBodyDisplay.ts | 26 +++-- lineageweave/chunking.py | 104 +++++++++++++---- tests/test_chunking.py | 105 +++++++++++++++++- 10 files changed, 271 insertions(+), 41 deletions(-) rename docs/adr/{0119-quantity-script-display.md => 0165-quantity-script-display.md} (97%) diff --git a/AGENTS.md b/AGENTS.md index cb2da918a..151568bcb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -150,7 +150,7 @@ contextual-orchestrator owns model discovery and selection. paragraphs, not the authoring application's spacing workaround. - Quantity HTML ``/`` and caret exponents such as `m^3` become Unicode in derived units and React ``/`` in the post view - (ADR 0119). Never assign the body to `innerHTML`. Do not treat + (ADR 0165). Never assign the body to `innerHTML`. Do not treat `qty < 50` or a leading footnote `^1` as an exponent. - Image descriptions, OCR text, and region evidence are analysis artifacts, not buyer-facing prompt instructions. Buyer UI shows the source content and diff --git a/CHANGELOG.d/2.12.9-quantity-superscripts.md b/CHANGELOG.d/2.12.9-quantity-superscripts.md index b2d3e3541..13498dbe4 100644 --- a/CHANGELOG.d/2.12.9-quantity-superscripts.md +++ b/CHANGELOG.d/2.12.9-quantity-superscripts.md @@ -5,4 +5,4 @@ - Post popups now show cubic metres and similar quantities as superscripts and subscripts (`12 m³`, `H₂O`) instead of flattened `m^3` or `m3`. Semantic units store Unicode so embeddings keep the exponent. Comparison - operators and leading footnote carets stay literal (ADR 0119). + operators and leading footnote carets stay literal (ADR 0165). diff --git a/docs/adr/0119-quantity-script-display.md b/docs/adr/0165-quantity-script-display.md similarity index 97% rename from docs/adr/0119-quantity-script-display.md rename to docs/adr/0165-quantity-script-display.md index fdb902daf..9b38bb6f8 100644 --- a/docs/adr/0119-quantity-script-display.md +++ b/docs/adr/0165-quantity-script-display.md @@ -1,4 +1,4 @@ -# ADR 0119: Render quantity superscripts as text runs, Unicode in units +# ADR 0165: Render quantity superscripts as text runs, Unicode in units - Status: Accepted - Date: 2026-08-22 diff --git a/docs/adr/README.md b/docs/adr/README.md index 6cb58b93c..b6819594f 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -10,7 +10,7 @@ decision from them. | Supporting document | Normative ADR | |---|---| | [`product-technical-gap-baseline.md`](../product-technical-gap-baseline.md) | Product/technical traceability projection across the ADR set; ADRs remain normative | -| [`lineage-bi-research-notes.md`](../lineage-bi-research-notes.md) | [0084](0084-lineage-research-grounding.md), [0062](0062-semantic-unit-embedding.md), [0064](0064-lineage-evidence-and-tree-assembly.md), [0119](0119-quantity-script-display.md) | +| [`lineage-bi-research-notes.md`](../lineage-bi-research-notes.md) | [0084](0084-lineage-research-grounding.md), [0062](0062-semantic-unit-embedding.md), [0064](0064-lineage-evidence-and-tree-assembly.md), [0165](0165-quantity-script-display.md) | | [`PROV_O_IMPLEMENTATION.md`](../PROV_O_IMPLEMENTATION.md) | [0065](0065-prov-o-provenance-boundary.md) | | [`PROV_O_IMPLEMENTATION_MATRIX.md`](../PROV_O_IMPLEMENTATION_MATRIX.md) | [0065](0065-prov-o-provenance-boundary.md) | | [`image-content-schema.md`](../image-content-schema.md) | [0066](0066-position-preserving-image-content.md) | diff --git a/docs/lineage-bi-research-notes.md b/docs/lineage-bi-research-notes.md index a3fe9e3ba..a0f202468 100644 --- a/docs/lineage-bi-research-notes.md +++ b/docs/lineage-bi-research-notes.md @@ -386,7 +386,7 @@ hard truncation that silently drops earlier decisions. This is recorded here as the citation this feature would build on, not as a claim that conversation-level compression is implemented today. -## Quantity scripts in source units (ADR 0119) +## Quantity scripts in source units (ADR 0165) Board exports write cubic metres as HTML `` or as `m^3`. Flattening those tags concatenates `m3`, which is a different quantity, and leaving diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 3904af44e..90c84ceeb 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -5,7 +5,7 @@ - **Table Parsing**: `post=00505695-3e61-1fd1-80c6-86bb61c8ddc5` completely fails at parsing tables. - **Indentation**: Incorrect indentation rendering in `post=00505695-7571-1fd1-83c3-d521b187ad5b` and `post=00505695-3e61-1fd1-83c0-497b3c1c455e`. - **Image/Table OCR**: `post=00505695-7571-1fd1-83dd-3d22a61a5734` fails text recognition for tables inside images, markdown parsing fails, and image OCR description is too shallow for Ontology & Semantics. -- **Math/Superscripts**: (Display/unit text, ADR 0119) Quantity HTML +- **Math/Superscripts**: (Display/unit text, ADR 0165) Quantity HTML ``/`` and caret exponents such as `m^3` now render as superscripts and persist as Unicode in semantic units. Full formula ontology grammar remains open. diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts index c974f3dfa..20e266122 100644 --- a/frontend/src/postBodyDisplay.test.ts +++ b/frontend/src/postBodyDisplay.test.ts @@ -47,17 +47,21 @@ describe("splitPostBody", () => { ]); }); - it("reads CSS box shorthand indentation and markerless footnotes", () => { + it("reads CSS box shorthand indentation", () => { expect( splitPostBody( '
  • Outer
' + - '
  • Nested
' + - "

*Tier 2: note

", + '
  • Nested
', ), ).toEqual([ { kind: "text", text: "Outer", indentLevel: 7 }, { kind: "text", text: "Nested", indentLevel: 10 }, - { kind: "text", text: "*Tier 2: note", role: "footnote" }, + ]); + }); + + it("does not infer a footnote from a bare marker", () => { + expect(splitPostBody("

*Synthetic list item

")).toEqual([ + { kind: "text", text: "*Synthetic list item" }, ]); }); @@ -156,6 +160,55 @@ describe("splitPostBody", () => { ]); }); + it("normalizes entity-encoded quantity syntax without leaking raw markup", () => { + expect( + splitPostBody("

Reserve 12 m^3 and x<sup>2</sup> units.

"), + ).toEqual([{ kind: "text", text: "Reserve 12 m³ and x² units." }]); + }); + + it("keeps encoded non-script inline markup literal", () => { + expect(splitPostBody("

Keep <b>bold</b> literal.

")).toEqual([ + { kind: "text", text: "Keep bold literal." }, + ]); + }); + + it("keeps encoded non-script block markup literal", () => { + expect( + splitPostBody( + "

Keep <table><tr><td>grid</td></tr></table> literal.

", + ), + ).toEqual([ + { kind: "text", text: "Keep
grid
literal." }, + ]); + }); + + it("normalizes nested-encoded script tags and their inner entity", () => { + expect( + splitPostBody( + "

Volume is m&lt;sup&gt;&nbsp;3&lt;/sup&gt;.

", + ), + ).toEqual([{ kind: "text", text: "Volume is m ³." }]); + }); + + it("normalizes encoded script content wrapped in encoded inline markup", () => { + expect( + splitPostBody("

x<sup><span>2</span></sup>

"), + ).toEqual([{ kind: "text", text: "x²" }]); + }); + + it("keeps encoded script-prefixed custom and namespaced tags literal", () => { + expect( + splitPostBody( + "

Keep <sup-note>2</sup-note> and <sub:item>3</sub:item> literal.

", + ), + ).toEqual([ + { + kind: "text", + text: "Keep 2 and 3 literal.", + }, + ]); + }); + it("matches sup/sub content split across a newline", () => { // Pretty-printed source HTML puts tag content on its own line // (regression: the regex lacked the dotAll flag, so `.` could not cross @@ -178,6 +231,10 @@ describe("splitPostBody", () => { ]); }); + it("keeps mixed script content as a visible fallback", () => { + expect(splitPostBody("x3a")).toEqual([{ kind: "text", text: "x^3a" }]); + }); + it("decodes a stored superscript letter deterministically to lowercase", () => { // "n" and "N" both encode to the same Unicode "ⁿ" (there is no distinct // uppercase superscript N), so decoding must pick one case consistently diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts index 2773eadf2..f037a8f1a 100644 --- a/frontend/src/postBodyDisplay.ts +++ b/frontend/src/postBodyDisplay.ts @@ -20,7 +20,6 @@ const BLOCK_TAG = /<\/?(?:article|blockquote|div|h[1-6]|li|ol|p|section|table|tbody|td|tfoot|th|thead|tr|ul|w:p|w:tbl|w:tr|w:tc)\b[^>]*>/gi; const WORD_INDENT_TAG = /]*\/?\s*>/gi; const LIST_ITEM_START = /^\s*(?:[-*•·]\s+|[*†‡](?=\S)|(?:\d{1,3}|[A-Za-z가-힣])[.)]\s+|[①-⑳]\s+)/; -const FOOTNOTE_START = /^\s*[*†‡](?=\S)/; const INDENT_MARKER = "\u0001lw-indent:"; const INDENT_MARKER_END = "\u0002"; const INDENT_MARKER_PATTERN = /lw-indent:(\d+)/g; @@ -156,6 +155,9 @@ const SUPER_UNI_TO_ASCII = buildUnicodeToAsciiTable(SUPER_ASCII_TO_UNI); const SUB_UNI_TO_ASCII = buildUnicodeToAsciiTable(SUB_ASCII_TO_UNI); const CARET_EXPONENT = /(?<=[A-Za-z0-9µμ°ΩÅåÅ)])\^(?:\{([+-]?\d{1,3}|[nNiI])\}|([+-]?\d{1,3}|[nNiI]))/g; +const ENCODED_CARET = /&(?:amp;)*(?:#0*94|#x0*5e);/gi; +const ENCODED_SCRIPT_TAG = + /&(?:amp;)*(?:lt|#0*60|#x0*3c);\s*\/?\s*(?:sup|sub)(?=\s|\/|&(?:amp;)*(?:gt|#0*62|#x0*3e);).*?&(?:amp;)*(?:gt|#0*62|#x0*3e);/gis; function applyUnicodeScript(text: string, kind: "super" | "sub"): string { const table = kind === "super" ? SUPER_ASCII_TO_UNI : SUB_ASCII_TO_UNI; @@ -174,17 +176,25 @@ function applyUnicodeScript(text: string, kind: "super" | "sub"): string { function replaceHtmlScripts(text: string): string { return text .replace(/]*>(.*?)<\/sup>/gis, (_match, inner: string) => - applyUnicodeScript(decodeHtmlEntities(String(inner).replace(/<[^>]+>/g, "")), "super"), + applyUnicodeScript(decodeHtmlEntities(String(inner)).replace(/<[^>]+>/g, ""), "super"), ) .replace(/]*>(.*?)<\/sub>/gis, (_match, inner: string) => - applyUnicodeScript(decodeHtmlEntities(String(inner).replace(/<[^>]+>/g, "")), "sub"), + applyUnicodeScript(decodeHtmlEntities(String(inner)).replace(/<[^>]+>/g, ""), "sub"), ); } +function decodeScriptEntities(text: string): string { + return text + .replace(ENCODED_SCRIPT_TAG, (tag) => decodeHtmlEntities(tag)) + .replace(ENCODED_CARET, (caret) => decodeHtmlEntities(caret)); +} + export function normalizeScriptText(text: string): string { - return replaceHtmlScripts(text).replace(CARET_EXPONENT, (_match, braced: string, bare: string) => - applyUnicodeScript(braced || bare, "super"), + const withCarets = decodeScriptEntities(text).replace( + CARET_EXPONENT, + (_match, braced: string, bare: string) => applyUnicodeScript(braced || bare, "super"), ); + return replaceHtmlScripts(withCarets); } export type ScriptRun = { text: string; script?: "super" | "sub" }; @@ -239,7 +249,7 @@ export function splitScriptRuns(text: string): ScriptRun[] { } function stripHtmlTags(text: string): string { - const withScripts = replaceHtmlScripts(text); + const withScripts = normalizeScriptText(text); const withBoundaries = withScripts .replace(BREAK_TAG, "\n") .replace(BLOCK_TAG, (tag) => { @@ -250,8 +260,7 @@ function stripHtmlTags(text: string): string { const withoutTags = withBoundaries.replace(HTML_TAG, (tag) => /^<\/?w:/i.test(tag) ? "" : " ", ); - const decoded = decodeHtmlEntities(withoutTags); - return normalizeScriptText(decoded) + return decodeHtmlEntities(withoutTags) .split("\n") .map((line) => { if (!line.trim()) return ""; @@ -358,7 +367,6 @@ function pushText(segments: PostBodySegment[], raw: string, indentUnit: number): kind: "text", text: normalized, ...(indentLevel > 0 ? { indentLevel } : {}), - ...(FOOTNOTE_START.test(normalized) ? { role: "footnote" as const } : {}), }); } } diff --git a/lineageweave/chunking.py b/lineageweave/chunking.py index 2caf21e0e..511634ff7 100644 --- a/lineageweave/chunking.py +++ b/lineageweave/chunking.py @@ -97,11 +97,11 @@ # readable and attributable as one unit. _TABLE_ROW_TAGS = frozenset({"tr", "w:tr"}) _TABLE_CELL_TAGS = frozenset({"td", "th", "w:tc"}) +_TABLE_TAGS = frozenset({"table", "w:tbl"}) _LIST_ITEM_START = re.compile( r"^(?:[-*•·]\s+|[*†‡](?=\S)|(?:\d{1,3}|[A-Za-z가-힣])[.)]\s+|[①-⑳]\s+)" ) -_FOOTNOTE_START = re.compile(r"^[*†‡](?=\S)") def _is_footnote_block(tag: str, attrs: list[tuple[str, str | None]]) -> bool: @@ -132,7 +132,7 @@ def _is_footnote_reference(attrs: list[tuple[str, str | None]]) -> bool: # Unicode Super/Subscript blocks (The Unicode Consortium, 2024, §22.4) plus the # Latin-1 superscript digits. Quantity display uses these so embeddings keep -# "m³" distinct from "m3" without retaining HTML in the semantic text (ADR 0119). +# "m³" distinct from "m3" without retaining HTML in the semantic text (ADR 0165). _SUPERSCRIPT = { "0": "\u2070", "1": "\u00b9", @@ -214,23 +214,24 @@ def apply_unicode_script(text: str, kind: str) -> str: def _replace_html_script(match: re.Match[str], kind: str) -> str: - inner = _INNER_TAG.sub("", match.group(1)) + inner = match.group(1) for _ in range(3): decoded = unescape(inner) if decoded == inner: break inner = decoded - return apply_unicode_script(inner, kind) + return apply_unicode_script(_INNER_TAG.sub("", inner), kind) def normalize_script_text(text: str) -> str: """Turn HTML/caret quantity scripts into Unicode without treating comparisons as tags.""" - replaced = _HTML_SUP.sub(lambda match: _replace_html_script(match, "sup"), text) - replaced = _HTML_SUB.sub(lambda match: _replace_html_script(match, "sub"), replaced) - return _CARET_EXPONENT.sub( + replaced = _CARET_EXPONENT.sub( lambda match: apply_unicode_script(match.group(1) or match.group(2), "sup"), - replaced, + text, ) + replaced = _HTML_SUP.sub(lambda match: _replace_html_script(match, "sup"), replaced) + replaced = _HTML_SUB.sub(lambda match: _replace_html_script(match, "sub"), replaced) + return replaced def normalize_semantic_text(text: str) -> str: @@ -447,6 +448,9 @@ def __init__(self) -> None: self._stack: list[tuple[str, list[str], str | None, int, bool]] = [] self._unscoped_buffer: list[str] = [] self._script_stack: list[str] = [] + self._table_cell_counts: list[int] = [] + self._table_depth = 0 + self._table_row_depths: list[int] = [] # Each entry is ("text", str, tag_name, style) or # ("image", (mime_type, bytes), "", None) -- a single sequence in # true document order, so an image's index among its siblings @@ -455,6 +459,8 @@ def __init__(self) -> None: def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: """Collect relevant text state when an HTML start tag is encountered.""" + if tag in _TABLE_TAGS: + self._table_depth += 1 if tag in _INLINE_SCRIPT_TAGS: self._script_stack.append(tag) return @@ -484,16 +490,29 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None return if tag in _TABLE_CELL_TAGS: self._script_stack.clear() - if self._stack and self._stack[-1][0] in _TABLE_ROW_TAGS and self._stack[-1][1]: - self._stack[-1][1].append(" | ") + if self._stack and self._stack[-1][0] in _TABLE_ROW_TAGS: + if self._table_cell_counts[-1]: + self._stack[-1][1].append(" | ") + self._table_cell_counts[-1] += 1 return + if ( + tag in _TABLE_ROW_TAGS + and self._table_row_depths + and self._table_row_depths[-1] == self._table_depth + ): + declared_width = sum(entry[3] for entry in self._stack) + tag_name, buffer, style, _, is_footnote = self._stack.pop() + self._finish_block(tag_name, buffer, style, declared_width, is_footnote) # A rich-text editor commonly wraps a table cell in a nested

or #

. Keep that content in the open row; otherwise the nested block # closes first and destroys the row/column boundary. - if any(entry[0] in _TABLE_ROW_TAGS for entry in self._stack): + if ( + tag not in _TABLE_ROW_TAGS + and any(entry[0] in _TABLE_ROW_TAGS for entry in self._stack) + ): return if tag in _DOM_BLOCK_TAGS: - if self._stack and self._stack[-1][1]: + if tag not in _TABLE_ROW_TAGS and self._stack and self._stack[-1][1]: tag_name, buffer, style, _, is_footnote = self._stack[-1] declared_width = sum(entry[3] for entry in self._stack) self._finish_block(tag_name, buffer, style, declared_width, is_footnote) @@ -505,15 +524,26 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None self._stack.append( (tag, [], style, _declared_indent_width(tag, attrs), is_footnote) ) + if tag in _TABLE_ROW_TAGS: + self._table_cell_counts.append(0) + self._table_row_depths.append(self._table_depth) def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: """Handle self-closing block tags without losing XML indentation state.""" self.handle_starttag(tag, attrs) - if tag in _DOM_BLOCK_TAGS or tag in _INLINE_SCRIPT_TAGS: + if tag in _DOM_BLOCK_TAGS or tag in _INLINE_SCRIPT_TAGS or tag in _TABLE_TAGS: self.handle_endtag(tag) def handle_endtag(self, tag: str) -> None: """Close the relevant text state when an HTML end tag is encountered.""" + if ( + tag in _TABLE_TAGS + and self._table_row_depths + and self._table_row_depths[-1] == self._table_depth + ): + declared_width = sum(entry[3] for entry in self._stack) + tag_name, buffer, style, _, is_footnote = self._stack.pop() + self._finish_block(tag_name, buffer, style, declared_width, is_footnote) if tag in _INLINE_SCRIPT_TAGS: if tag in self._script_stack: while self._script_stack: @@ -521,10 +551,15 @@ def handle_endtag(self, tag: str) -> None: if closed == tag: break return + if tag in _TABLE_CELL_TAGS: + self._script_stack.clear() + return if tag in _DOM_BLOCK_TAGS and self._stack and self._stack[-1][0] == tag: declared_width = sum(entry[3] for entry in self._stack) tag_name, buffer, style, _, is_footnote = self._stack.pop() self._finish_block(tag_name, buffer, style, declared_width, is_footnote) + if tag in _TABLE_TAGS: + self._table_depth = max(0, self._table_depth - 1) def _finish_block( self, @@ -543,11 +578,14 @@ def _finish_block( # formatting survive a block-level boundary. self._script_stack.clear() raw_text = "".join(buffer) + if tag_name in _TABLE_ROW_TAGS: + self._table_cell_counts.pop() + self._table_row_depths.pop() for raw_unit, source_indent in _split_dom_units(raw_text): text = normalize_semantic_text(raw_unit) if text: indent_width = declared_width + source_indent - label = "footnote" if is_footnote or _FOOTNOTE_START.match(text) else tag_name + label = "footnote" if is_footnote else tag_name self._finished.append( ( "text", @@ -619,15 +657,31 @@ def flush() -> None: ) +def _markdown_table_cells(line: str) -> list[str]: + """Return cells while removing only optional outer pipe delimiters.""" + cells = line.strip().split("|") + if cells and not cells[0]: + cells.pop(0) + if cells and not cells[-1]: + cells.pop() + return cells + + def _is_markdown_table_row(line: str) -> bool: """Recognize a pipe row only when it has at least two cells.""" - cells = line.strip().strip("|").split("|") - return len(cells) >= 2 and all(cell.strip() for cell in cells) + cells = _markdown_table_cells(line) + return len(cells) >= 2 and any(cell.strip() for cell in cells) + + +def _is_empty_markdown_table_row(line: str, column_count: int) -> bool: + """Recognize an all-empty row only inside an established table.""" + cells = _markdown_table_cells(line) + return len(cells) == column_count and not any(cell.strip() for cell in cells) def _render_markdown_table_row(line: str) -> str: """Keep Markdown table columns as searchable row evidence.""" - return " | ".join(cell.strip() for cell in line.strip().strip("|").split("|")) + return " | ".join(cell.strip() for cell in _markdown_table_cells(line)) def _split_plain_text_units(text: str) -> list[tuple[str, int, str]]: @@ -653,8 +707,20 @@ def flush() -> None: continue if _is_markdown_table_row(line): rows: list[str] = [] - while index < len(lines) and _is_markdown_table_row(lines[index]): - rows.append(lines[index]) + column_count = len(_markdown_table_cells(line)) + while index < len(lines): + candidate = lines[index] + established = ( + len(rows) >= 2 + and bool(_MARKDOWN_TABLE_SEPARATOR.match(rows[1])) + and len(_markdown_table_cells(rows[1])) == column_count + ) + if not _is_markdown_table_row(candidate) and not ( + established + and _is_empty_markdown_table_row(candidate, column_count) + ): + break + rows.append(candidate) index += 1 data_rows = [row for row in rows if not _MARKDOWN_TABLE_SEPARATOR.match(row)] if len(data_rows) >= 2: diff --git a/tests/test_chunking.py b/tests/test_chunking.py index 8d1c457c6..ab70048b3 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -105,11 +105,62 @@ def test_chunk_by_dom_keeps_nested_table_cell_blocks_in_their_row() -> None: assert [(chunk.label, chunk.text) for chunk in chunks] == [("tr", "No. | Company")] -def test_chunk_by_dom_labels_markerless_footnotes() -> None: - chunks = chunk_by_dom("

Body text

*Tier 2: follow-up note

") +def test_chunk_by_dom_preserves_empty_table_cells() -> None: + chunks = chunk_by_dom( + "
Synthetic item
" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("tr", "| Synthetic item |") + ] + + +def test_chunk_by_dom_preserves_self_closing_empty_table_cells() -> None: + html_chunks = chunk_by_dom( + "
LeftRight
" + ) + word_chunks = chunk_by_dom( + "LeftRight" + ) + + assert [(chunk.label, chunk.text) for chunk in html_chunks] == [ + ("tr", "Left | | Right") + ] + assert [(chunk.label, chunk.text) for chunk in word_chunks] == [ + ("w:tr", "Left | | Right") + ] + + +def test_chunk_by_dom_scopes_cell_positions_to_nested_table_rows() -> None: + chunks = chunk_by_dom( + "
Outer left" + "
Inner leftInner right
" + "		Outer right
" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("tr", "Inner left | Inner right"), + ("tr", "Outer left | Outer right"), + ] + + +def test_chunk_by_dom_implicitly_closes_sibling_rows_at_the_same_table_depth() -> None: + chunks = chunk_by_dom( + "" + "
First leftFirst right
Second leftSecond right
" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("tr", "First left | First right"), + ("tr", "Second left | Second right"), + ] + + +def test_chunk_by_dom_does_not_infer_a_footnote_from_a_bare_marker() -> None: + chunks = chunk_by_dom("

Body text

*Synthetic list item

") assert [(chunk.label, chunk.text) for chunk in chunks] == [ ("p", "Body text"), - ("footnote", "*Tier 2: follow-up note"), + ("p", "*Synthetic list item"), ] @@ -233,6 +284,38 @@ def test_chunk_by_source_body_splits_plain_lists_and_markdown_tables() -> None: ] +def test_chunk_by_source_body_preserves_empty_markdown_table_cells() -> None: + chunks = chunk_by_source_body( + "| Key | Value | State |\n" + "| --- | --- | --- |\n" + "| A | | Open |" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("tr", "Key | Value | State"), + ("tr", "A | | Open"), + ] + + +def test_chunk_by_source_body_keeps_contextual_all_empty_markdown_rows() -> None: + chunks = chunk_by_source_body( + "| Key | Value | State |\n" + "| --- | --- | --- |\n" + "| | | |" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("tr", "Key | Value | State"), + ("tr", "| |"), + ] + + +def test_chunk_by_source_body_does_not_promote_a_standalone_empty_pipe_line() -> None: + chunks = chunk_by_source_body("| | |") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [("", "| | |")] + + def test_chunk_by_dom_joins_visual_continuation_lines_but_keeps_list_items() -> None: html = ( '

1. 배경
' @@ -421,12 +504,28 @@ def test_normalize_script_text_maps_quantity_exponents_and_leaves_comparisons() assert normalize_script_text("^1 See the tank note.") == "^1 See the tank note." +def test_normalize_script_text_keeps_mixed_script_content_as_a_visible_fallback() -> None: + assert normalize_script_text("x3a") == "x^3a" + + +def test_normalize_script_text_decodes_nested_inline_markup_before_stripping() -> None: + assert normalize_script_text("x<span>2</span>") == "x²" + + def test_chunk_by_dom_keeps_html_quantity_scripts_as_unicode() -> None: chunks = chunk_by_dom("

Tank volume is 12 m3 of H2O.

") assert [chunk.text for chunk in chunks] == ["Tank volume is 12 m³ of H₂O."] +def test_chunk_by_dom_normalizes_entity_encoded_quantity_scripts() -> None: + chunks = chunk_by_dom( + "

Reserve 12 m^3 and x<sup>2</sup> units.

" + ) + + assert [chunk.text for chunk in chunks] == ["Reserve 12 m³ and x² units."] + + def test_chunk_by_dom_unclosed_sup_does_not_cross_table_cells() -> None: chunks = chunk_by_dom("
m3Acme Corp
") From fccc3a5ff27933c96b538365d4d1fd1d2f658d21 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 24 Aug 2026 04:13:57 +0900 Subject: [PATCH 08/10] test: keep malformed cell blocks inside their row --- tests/test_chunking.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_chunking.py b/tests/test_chunking.py index ab70048b3..5892c3697 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -156,6 +156,20 @@ def test_chunk_by_dom_implicitly_closes_sibling_rows_at_the_same_table_depth() - ] +def test_chunk_by_dom_closes_sibling_rows_past_an_unclosed_cell_block() -> None: + """Malformed inline cell markup cannot displace its owning row.""" + + chunks = chunk_by_dom( + "" + "
First left
First right
Second leftSecond right
" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("tr", "First left | First right"), + ("tr", "Second left | Second right"), + ] + + def test_chunk_by_dom_does_not_infer_a_footnote_from_a_bare_marker() -> None: chunks = chunk_by_dom("

Body text

*Synthetic list item

") assert [(chunk.label, chunk.text) for chunk in chunks] == [ From 0290387775bb8a1bed2c4fa629b888aca1071627 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 24 Aug 2026 04:16:30 +0900 Subject: [PATCH 09/10] test: cover malformed quantity table boundaries --- tests/test_chunking.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_chunking.py b/tests/test_chunking.py index 5892c3697..20a5e0dae 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -156,6 +156,18 @@ def test_chunk_by_dom_implicitly_closes_sibling_rows_at_the_same_table_depth() - ] +def test_chunk_by_dom_closes_an_unclosed_row_at_the_table_boundary() -> None: + """A malformed final row still emits before its table closes.""" + + chunks = chunk_by_dom( + "
Only leftOnly right
" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("tr", "Only left | Only right") + ] + + def test_chunk_by_dom_closes_sibling_rows_past_an_unclosed_cell_block() -> None: """Malformed inline cell markup cannot displace its owning row.""" @@ -514,6 +526,7 @@ def test_normalize_script_text_maps_quantity_exponents_and_leaves_comparisons() assert normalize_script_text("Tank volume is 12 m3.") == "Tank volume is 12 m³." assert normalize_script_text("Tank volume is 12 m^3.") == "Tank volume is 12 m³." assert normalize_script_text("Coolant is H2O.") == "Coolant is H₂O." + assert normalize_script_text("x ") == "x " assert normalize_script_text("qty < 50 and price > 10") == "qty < 50 and price > 10" assert normalize_script_text("^1 See the tank note.") == "^1 See the tank note." From 882bbe6a7545d8c5368773b2dae6ca4adc7b5d52 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 12:19:41 -0700 Subject: [PATCH 10/10] fix: preserve encoded-script source parity (#520) * fix: normalize encoded scripts in plain source * fix: keep invalid encoded script markup literal * fix: normalize persisted script units at display boundary --- frontend/src/PostBody.test.tsx | 95 ++++++++++++++++++++++++++++ frontend/src/PostBody.tsx | 20 ++++-- frontend/src/postBodyDisplay.test.ts | 16 +++++ frontend/src/postBodyDisplay.ts | 23 +++++-- lineageweave/chunking.py | 41 ++++++++++-- tests/test_chunking.py | 32 ++++++++++ 6 files changed, 212 insertions(+), 15 deletions(-) diff --git a/frontend/src/PostBody.test.tsx b/frontend/src/PostBody.test.tsx index 76837b5b2..3eaba4970 100644 --- a/frontend/src/PostBody.test.tsx +++ b/frontend/src/PostBody.test.tsx @@ -58,6 +58,101 @@ describe("PostBody", () => { expect(screen.getByText("Embedded image")).toBeInTheDocument(); }); + it("renders raw and persisted encoded non-script markup as the same inert text", () => { + const encoded = + "Keep <b>bold</b>, <sup-note>2</sup-note>, " + + "<sub:item>3</sub:item>, and <script>alert(1)</script> literal."; + const visible = + "Keep bold, 2, 3, and literal."; + const { container, rerender } = render(${encoded}

`} />); + + expect(screen.getByText(visible)).toBeInTheDocument(); + expect(container.querySelector("b")).toBeNull(); + expect(container.querySelector("script")).toBeNull(); + expect(container.querySelector("sup-note")).toBeNull(); + + rerender( + ${encoded}

`} + structureUnits={[ + { + unit_index: 0, + unit_kind_code: "plain_text", + unit_text: encoded, + indent_level: 0, + indent_source_code: "explicit", + indent_confidence: 1, + indent_evidence: "Synthetic encoded source", + }, + ]} + />, + ); + + expect(screen.getByText(visible)).toBeInTheDocument(); + expect(container.querySelector("b")).toBeNull(); + expect(container.querySelector("script")).toBeNull(); + expect(container.querySelector("sup-note")).toBeNull(); + }); + + it("renders raw and legacy persisted encoded scripts with the same semantics", () => { + const encoded = + "Volume x<sup>2</sup>, coolant H<sub>2</sub>O, and area m&#94;3."; + const { container, rerender } = render(${encoded}

`} />); + + expect([...container.querySelectorAll("sup")].map((node) => node.textContent)).toEqual([ + "2", + "3", + ]); + expect(container.querySelector("sub")?.textContent).toBe("2"); + + rerender( + ${encoded}

`} + structureUnits={[ + { + unit_index: 0, + unit_kind_code: "plain_text", + unit_text: encoded, + indent_level: 0, + indent_source_code: "explicit", + indent_confidence: 1, + indent_evidence: "Synthetic legacy persisted unit", + }, + ]} + />, + ); + + expect([...container.querySelectorAll("sup")].map((node) => node.textContent)).toEqual([ + "2", + "3", + ]); + expect(container.querySelector("sub")?.textContent).toBe("2"); + }); + + it("normalizes legacy encoded scripts in persisted table cells", () => { + const { container } = render( + , + ); + + const superscript = container.querySelector("td sup"); + expect(superscript?.textContent).toBe("3"); + expect(superscript?.closest("td")?.textContent).toBe("12 m3"); + }); + it("renders authoritative LLM structure levels for semantic list units", () => { render( ( {row.unit_text.split(/\s*\|\s*/).map((cell, cellIndex) => ( - {renderStyledText(cell)} + + {renderStyledText(displayUnitText(cell))} + ))} ))} @@ -251,7 +263,7 @@ function renderStructuredUnits( renderSegment( { kind: "text", - text: unit.unit_text, + text: displayUnitText(unit.unit_text), ...(unit.unit_label === "footnote" || sourceText?.role === "footnote" ? { role: "footnote" as const } : {}), diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts index 20e266122..3a5a5e3af 100644 --- a/frontend/src/postBodyDisplay.test.ts +++ b/frontend/src/postBodyDisplay.test.ts @@ -166,6 +166,22 @@ describe("splitPostBody", () => { ).toEqual([{ kind: "text", text: "Reserve 12 m³ and x² units." }]); }); + it("keeps invalid encoded script pairs literal", () => { + expect( + splitPostBody( + "

Keep x<sup>2 unmatched; x<sup/>2 self-closing; " + + "x<sup class="unit">2</sup> attributed; and " + + "x<sup>2</sub> mismatched.

", + ), + ).toEqual([ + { + kind: "text", + text: + 'Keep x2 unmatched; x2 self-closing; x2 attributed; and x2 mismatched.', + }, + ]); + }); + it("keeps encoded non-script inline markup literal", () => { expect(splitPostBody("

Keep <b>bold</b> literal.

")).toEqual([ { kind: "text", text: "Keep bold literal." }, diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts index f037a8f1a..6a4f17540 100644 --- a/frontend/src/postBodyDisplay.ts +++ b/frontend/src/postBodyDisplay.ts @@ -39,8 +39,10 @@ export function decodeHtmlEntities(text: string): string { const decoder = document.createElement("textarea"); let decoded = text; for (let pass = 0; pass < 3; pass += 1) { - decoder.innerHTML = decoded; - const next = decoder.value; + const next = decoded.replace(/&(?:#[0-9]+|#x[0-9a-f]+|[a-z][a-z0-9]+);/gi, (entity) => { + decoder.innerHTML = entity; + return decoder.value; + }); if (next === decoded) break; decoded = next; } @@ -156,8 +158,15 @@ const SUB_UNI_TO_ASCII = buildUnicodeToAsciiTable(SUB_ASCII_TO_UNI); const CARET_EXPONENT = /(?<=[A-Za-z0-9µμ°ΩÅåÅ)])\^(?:\{([+-]?\d{1,3}|[nNiI])\}|([+-]?\d{1,3}|[nNiI]))/g; const ENCODED_CARET = /&(?:amp;)*(?:#0*94|#x0*5e);/gi; -const ENCODED_SCRIPT_TAG = - /&(?:amp;)*(?:lt|#0*60|#x0*3c);\s*\/?\s*(?:sup|sub)(?=\s|\/|&(?:amp;)*(?:gt|#0*62|#x0*3e);).*?&(?:amp;)*(?:gt|#0*62|#x0*3e);/gis; +const ENCODED_LT = String.raw`&(?:amp;)*(?:lt|#0*60|#x0*3c);`; +const ENCODED_GT = String.raw`&(?:amp;)*(?:gt|#0*62|#x0*3e);`; +const ENCODED_SCRIPT_TOKEN = + `${ENCODED_LT}\\s*/?\\s*(?:sup|sub)(?=\\s|/|${ENCODED_GT})`; +const ENCODED_SCRIPT_PAIR = new RegExp( + `${ENCODED_LT}(sup|sub)${ENCODED_GT}` + + `((?:(?!${ENCODED_SCRIPT_TOKEN}).)*?)${ENCODED_LT}/\\1${ENCODED_GT}`, + "gis", +); function applyUnicodeScript(text: string, kind: "super" | "sub"): string { const table = kind === "super" ? SUPER_ASCII_TO_UNI : SUB_ASCII_TO_UNI; @@ -185,7 +194,11 @@ function replaceHtmlScripts(text: string): string { function decodeScriptEntities(text: string): string { return text - .replace(ENCODED_SCRIPT_TAG, (tag) => decodeHtmlEntities(tag)) + .replace( + ENCODED_SCRIPT_PAIR, + (_pair, kind: string, inner: string) => + `<${kind.toLowerCase()}>${inner}`, + ) .replace(ENCODED_CARET, (caret) => decodeHtmlEntities(caret)); } diff --git a/lineageweave/chunking.py b/lineageweave/chunking.py index 511634ff7..fc66e2ec3 100644 --- a/lineageweave/chunking.py +++ b/lineageweave/chunking.py @@ -194,6 +194,18 @@ def _is_footnote_reference(attrs: list[tuple[str, str | None]]) -> bool: _CARET_EXPONENT = re.compile( r"(?<=[A-Za-z0-9µμ°ΩÅåÅ)])\^(?:\{([+\-]?\d{1,3}|[nNiI])\}|([+\-]?\d{1,3}|[nNiI]))" ) +_ENCODED_CARET = re.compile(r"&(?:amp;)*(?:#0*94|#x0*5e);", re.IGNORECASE) +_ENCODED_LT = r"&(?:amp;)*(?:lt|#0*60|#x0*3c);" +_ENCODED_GT = r"&(?:amp;)*(?:gt|#0*62|#x0*3e);" +_ENCODED_SCRIPT_TOKEN = ( + rf"{_ENCODED_LT}\s*/?\s*(?:sup|sub)(?=\s|/|{_ENCODED_GT})" +) +_ENCODED_SCRIPT_PAIR = re.compile( + rf"{_ENCODED_LT}(?Psup|sub){_ENCODED_GT}" + rf"(?P(?:(?!{_ENCODED_SCRIPT_TOKEN}).)*?)" + rf"{_ENCODED_LT}/(?P=kind){_ENCODED_GT}", + re.IGNORECASE | re.DOTALL, +) def apply_unicode_script(text: str, kind: str) -> str: @@ -213,13 +225,30 @@ def apply_unicode_script(text: str, kind: str) -> str: return f"{leading}{prefix}{compact}{trailing}" -def _replace_html_script(match: re.Match[str], kind: str) -> str: - inner = match.group(1) +def _decode_html_entities(text: str) -> str: for _ in range(3): - decoded = unescape(inner) - if decoded == inner: + decoded = unescape(text) + if decoded == text: break - inner = decoded + text = decoded + return text + + +def _decode_script_entities(text: str) -> str: + decoded_pairs = _ENCODED_SCRIPT_PAIR.sub( + lambda match: ( + f"<{match.group('kind').lower()}>{match.group('inner')}" + f"" + ), + text, + ) + return _ENCODED_CARET.sub( + lambda match: _decode_html_entities(match.group(0)), decoded_pairs + ) + + +def _replace_html_script(match: re.Match[str], kind: str) -> str: + inner = _decode_html_entities(match.group(1)) return apply_unicode_script(_INNER_TAG.sub("", inner), kind) @@ -227,7 +256,7 @@ def normalize_script_text(text: str) -> str: """Turn HTML/caret quantity scripts into Unicode without treating comparisons as tags.""" replaced = _CARET_EXPONENT.sub( lambda match: apply_unicode_script(match.group(1) or match.group(2), "sup"), - text, + _decode_script_entities(text), ) replaced = _HTML_SUP.sub(lambda match: _replace_html_script(match, "sup"), replaced) replaced = _HTML_SUB.sub(lambda match: _replace_html_script(match, "sub"), replaced) diff --git a/tests/test_chunking.py b/tests/test_chunking.py index ab70048b3..21cf75b05 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -284,6 +284,38 @@ def test_chunk_by_source_body_splits_plain_lists_and_markdown_tables() -> None: ] +def test_chunk_by_source_body_normalizes_entity_encoded_quantity_scripts() -> None: + chunks = chunk_by_source_body( + "Reserve 12 m^3, x<sup>2</sup>, and H<sub>2</sub>O." + ) + + assert [chunk.text for chunk in chunks] == ["Reserve 12 m³, x², and H₂O."] + + +def test_chunk_by_source_body_keeps_invalid_encoded_script_pairs_literal() -> None: + bodies = ( + "Keep x<sup>2 unmatched.", + "Keep x<sup/>2 self-closing.", + "Keep x<sup class="unit">2</sup> attributed.", + "Keep x<sup>2</sub> mismatched.", + ) + + assert [chunk_by_source_body(body)[0].text for body in bodies] == list(bodies) + combined = " ".join(bodies) + assert chunk_by_source_body(combined)[0].text == combined + + +def test_chunk_by_source_body_keeps_encoded_non_script_markup_inert() -> None: + body = ( + "Keep <b>bold</b>, <sup-note>2</sup-note>, " + "and <script>alert(1)</script> literal." + ) + + chunks = chunk_by_source_body(body) + + assert [(chunk.unit_type, chunk.text) for chunk in chunks] == [("plain_text", body)] + + def test_chunk_by_source_body_preserves_empty_markdown_table_cells() -> None: chunks = chunk_by_source_body( "| Key | Value | State |\n"