fix: preserve numbered footnotes and empty table cells

lineageweave/chunking.py

@@ -102,18 +102,33 @@
     r"^(?:[-*•·]\s+|[*†‡](?=\S)|(?:\d{1,3}|[A-Za-z가-힣])[.)]\s+|[①-⑳]\s+)"
 )
 _FOOTNOTE_START = re.compile(r"^[*†‡](?=\S)")
+_FOOTNOTE_IDENTIFIER = re.compile(
+    r"^(?:fn|ftn|endnote)[:_-]?\d+(?:[:_-][a-z0-9]+)*$",
+    re.IGNORECASE,
+)
+
+
+def _is_footnote_identifier(value: str) -> bool:
+    """Recognize a numbered footnote definition identifier, not a backlink."""
+    normalized = value.strip().lstrip("#_")
+    return bool(_FOOTNOTE_IDENTIFIER.fullmatch(normalized))
 
 
 def _is_footnote_block(tag: str, attrs: list[tuple[str, str | None]]) -> bool:
     """Recognize semantic footnote markup emitted by HTML and Word exports."""
     if tag.casefold().rsplit(":", 1)[-1] in {"footnote", "endnote"}:
         return True
-    values = " ".join(
+    values = [
         value or ""
         for name, value in attrs
         if name.casefold() in {"class", "id", "role", "data-role"}
-    ).casefold()
-    return "footnote" in values or "endnote" in values
+    ]
+    joined = " ".join(values).casefold()
+    return (
+        "footnote" in joined
+        or "endnote" in joined
+        or any(_is_footnote_identifier(value) for value in values)
+    )
 
 
 def _is_footnote_reference(attrs: list[tuple[str, str | None]]) -> bool:
@@ -125,8 +140,9 @@ def _is_footnote_reference(attrs: list[tuple[str, str | None]]) -> bool:
     }
     href = values.get("href", "")
     anchor_values = (values.get("id", ""), values.get("name", ""))
-    return "ftnref" in href and any(
-        "ftn" in value and "ftnref" not in value for value in anchor_values
+    href_is_reference = bool(re.search(r"(?:fn|ftn)ref", href))
+    return href_is_reference and any(
+        _is_footnote_identifier(value) for value in anchor_values
     )
 
 
@@ -364,6 +380,7 @@ def __init__(self) -> None:
         super().__init__()
         self._stack: list[tuple[str, list[str], str | None, int, bool]] = []
         self._unscoped_buffer: list[str] = []
+        self._table_cell_counts: dict[int, int] = {}
         # Each entry is ("text", str, tag_name, style) or
         # ("image", (mime_type, bytes), "", None) -- a single sequence in
         # true document order, so an image's index among its siblings
@@ -397,8 +414,18 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None
             self._stack[-1] = (tag_name, buffer, style, indent_width, True)
             return
         if tag in _TABLE_CELL_TAGS:
-            if self._stack and self._stack[-1][0] in _TABLE_ROW_TAGS and self._stack[-1][1]:
-                self._stack[-1][1].append(" | ")
+            if self._stack and self._stack[-1][0] in _TABLE_ROW_TAGS:
+                row_buffer = self._stack[-1][1]
+                row_key = id(row_buffer)
+                cell_count = self._table_cell_counts.get(row_key, 0)
+                first_cell_is_empty = (
+                    cell_count == 1 and not "".join(row_buffer).strip()
+                )
+                if cell_count or "".join(row_buffer).strip():
+                    row_buffer.append(" | ")
+                    if first_cell_is_empty:
+                        row_buffer.append(" | ")
+                self._table_cell_counts[row_key] = cell_count + 1
             return
         # A rich-text editor commonly wraps a table cell in a nested <p> or
         # <div>. Keep that content in the open row; otherwise the nested block
@@ -442,6 +469,7 @@ def _finish_block(
     ) -> None:
         """Emit one block buffer, including a block closed only at EOF."""
         raw_text = "".join(buffer)
+        self._table_cell_counts.pop(id(buffer), None)
         for raw_unit, source_indent in _split_dom_units(raw_text):
             text = normalize_semantic_text(raw_unit)
             if text:

tests/test_chunking.py

@@ -104,6 +104,15 @@ def test_chunk_by_dom_keeps_nested_table_cell_blocks_in_their_row() -> None:
     assert [(chunk.label, chunk.text) for chunk in chunks] == [("tr", "No. | Company")]
 
 
+def test_chunk_by_dom_preserves_boundary_before_first_cell_after_row_text() -> None:
+    """Malformed editor markup must not merge row text into the first cell."""
+    chunks = chunk_by_dom(
+        "<table><tr><p>caption</p><td>A</td><td>B</td></tr></table>"
+    )
+
+    assert [(chunk.label, chunk.text) for chunk in chunks] == [("tr", "caption | A | B")]
+
+
 def test_chunk_by_dom_labels_markerless_footnotes() -> None:
     chunks = chunk_by_dom("<p>Body text</p><p>*Tier 2: follow-up note</p>")
     assert [(chunk.label, chunk.text) for chunk in chunks] == [
@@ -142,6 +151,72 @@ def test_chunk_by_dom_does_not_label_body_footnote_citation_as_footnote() -> Non
     ]
 
 
+def test_chunk_by_dom_labels_numbered_html_footnote_ids() -> None:
+    """Common Markdown HTML footnote ids identify the definition block."""
+    html = '<ol><li id="fn1"><a href="#fnref1">1</a> Definition.</li></ol>'
+
+    chunks = chunk_by_dom(html)
+
+    assert [(chunk.label, chunk.text) for chunk in chunks] == [
+        ("footnote", "1 Definition."),
+    ]
+
+
+def test_chunk_by_dom_does_not_label_short_non_footnote_ids() -> None:
+    """Short application ids must not be mistaken for numbered footnotes."""
+    chunks = chunk_by_dom('<p id="en1">Engagement summary.</p>')
+
+    assert [(chunk.label, chunk.text) for chunk in chunks] == [
+        ("p", "Engagement summary."),
+    ]
+
+
+def test_chunk_by_dom_keeps_numbered_footnote_backlinks_in_body_paragraphs() -> None:
+    html = (
+        '<p>Body cites <a href="#fn1" id="fnref1">[1]</a>.</p>'
+        '<p id="fn1">Definition.</p>'
+    )
+
+    chunks = chunk_by_dom(html)
+
+    assert [(chunk.label, chunk.text) for chunk in chunks] == [
+        ("p", "Body cites [1]."),
+        ("footnote", "Definition."),
+    ]
+
+
+def test_chunk_by_dom_labels_colon_separated_footnote_ids() -> None:
+    html = (
+        '<p>Body cites <a href="#fn:1" id="fnref:1">[1]</a>.</p>'
+        '<p><a href="#_ftnref1_body" name="_ftn1_body">[1]</a> Definition.</p>'
+    )
+
+    chunks = chunk_by_dom(html)
+
+    assert [(chunk.label, chunk.text) for chunk in chunks] == [
+        ("p", "Body cites [1]."),
+        ("footnote", "[1] Definition."),
+    ]
+
+
+def test_chunk_by_dom_labels_bare_word_footnote_backlinks() -> None:
+    html = '<p><a href="#_ftnref" name="_ftn1_body">[1]</a> Definition.</p>'
+
+    chunks = chunk_by_dom(html)
+
+    assert [(chunk.label, chunk.text) for chunk in chunks] == [
+        ("footnote", "[1] Definition."),
+    ]
+
+
+def test_chunk_by_dom_preserves_empty_table_cells_as_columns() -> None:
+    html = "<table><tr><td></td><td>Company</td><td></td><td>Result</td></tr></table>"
+
+    chunks = chunk_by_dom(html)
+
+    assert [chunk.text for chunk in chunks] == ["|  | Company |  | Result"]
+
+
 def test_chunk_by_dom_labels_ooxml_footnote_containers() -> None:
     chunks = chunk_by_dom(
         "<w:footnote w:id='1'><w:p>OOXML footnote body</w:p></w:footnote>"