First block of text.
Second block of text.
`,
+ imageContent: [
+ {
+ unit_index: 0,
+ mime_type: "image/png",
+ status_code: "completed",
+ extracted_text: "| Workstream | State |\n| --- | --- |\n| Alpha | Ready |",
+ caption: "A synthetic workstream status table.",
+ tags: ["table"],
+ },
+ ],
+ },
+};
+
+export const NumericFootnote: Story = {
+ args: {
+ body: "Evidence remains attached to the source.
1 Source note.
", + }, +}; diff --git a/frontend/src/PostBody.test.tsx b/frontend/src/PostBody.test.tsx index e1322bb96..6698d75fe 100644 --- a/frontend/src/PostBody.test.tsx +++ b/frontend/src/PostBody.test.tsx @@ -133,6 +133,39 @@ describe("PostBody", () => { expect(screen.queryAllByText("No.")).toHaveLength(1); }); + it("renders persisted Markdown table rows as a table", () => { + render( +{block.text}
; + } + const [header, ...rows] = block.rows; + return ( +| + {cell} + | + ))} +
|---|
| + {cell} + | + ))} +
{renderStructuredUnits(body, structureUnits, imageContent)}
;
}
+ const markdownBlocks = !hasPersistedStructuralUnits ? splitMarkdownTableBody(body) : null;
+ if (markdownBlocks) {
+ return {renderMarkdownBlocks(markdownBlocks)}
;
+ }
return (
{splitPostBody(body).map((segment, index) => {
diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts
index ff51c8562..cfa010798 100644
--- a/frontend/src/postBodyDisplay.test.ts
+++ b/frontend/src/postBodyDisplay.test.ts
@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
-import { splitPostBody } from "./postBodyDisplay";
+import { splitMarkdownTableBody, splitPostBody } from "./postBodyDisplay";
/** 1x1 transparent PNG — the same synthetic fixture the Python vision tests use. */
const TINY_PNG_B64 =
@@ -61,6 +61,100 @@ describe("splitPostBody", () => {
]);
});
+ it("recognizes numeric superscript-style footnotes", () => {
+ expect(splitPostBody("HTML note. " +
+ "Word note. Parent ]*\bsrc\s*=\s*["']data:(image\/[a-zA-Z0-9.+-]+);base64,([A-Za-z0-9+/=\s]+)["'][^>]*>/gi;
const HTML_TAG = /<\/?[a-zA-Z][^>]*>/g;
const BREAK_TAG = /]*>/gi;
const BLOCK_TAG =
- /<\/?(?:article|blockquote|div|h[1-6]|li|ol|p|section|table|tbody|td|tfoot|th|thead|tr|ul|w:p|w:tbl|w:tr|w:tc)\b[^>]*>/gi;
+ /<\/?(?:article|blockquote|div|endnote|footnote|h[1-6]|li|oi|ol|p|section|table|tbody|td|tfoot|th|thead|tr|ul|w:endnote|w:footnote|w:p|w:tbl|w:tr|w:tc)\b[^>]*>/gi;
const WORD_INDENT_TAG = /]*\/?\s*>/gi;
const LIST_ITEM_START = /^\s*(?:[-*•·]\s+|[*†‡](?=\S)|(?:\d{1,3}|[A-Za-z가-힣])[.)]\s+|[①-⑳]\s+)/;
-const FOOTNOTE_START = /^\s*[*†‡](?=\S)/;
+const FOOTNOTE_START = /^\s*[*†‡]+(?=\S)/;
const INDENT_MARKER = "\u0001lw-indent:";
const INDENT_MARKER_END = "\u0002";
const INDENT_MARKER_PATTERN = /lw-indent:(\d+)/g;
+const NUMERIC_FOOTNOTE_MARKER = "\u0003lw-numeric-footnote\u0004";
+const FOOTNOTE_BLOCK_MARKER = "\u0005lw-footnote-block\u0006";
+const FOOTNOTE_BLOCK_OPEN = /<\s*(?:footnote|endnote|w:footnote|w:endnote)\b[^>]*>/gi;
+const NUMERIC_SUPERSCRIPT = /]*>\s*(\d{1,3})\s*<\/sup>/gi;
const SUPERSCRIPT_DIGITS = "⁰¹²³⁴⁵⁶⁷⁸⁹";
const SUBSCRIPT_DIGITS = "₀₁₂₃₄₅₆₇₈₉";
const METRIC_MARKUP =
/((?]*>\s*(\d{1,3})\s*<\/\2>/gi;
+const METRIC_PLAIN_SCRIPT =
+ /((? {
+ return raw
+ .replace(METRIC_MARKUP, (_match, base: string, kind: string, digits: string) => {
const table = kind.toLowerCase() === "sup" ? SUPERSCRIPT_DIGITS : SUBSCRIPT_DIGITS;
return `${base}${[...digits].map((digit) => table[Number(digit)]).join("")}`;
- },
- );
+ })
+ .replace(
+ METRIC_PLAIN_SCRIPT,
+ (_match, base: string, kind: string, bracedDigits: string, digits: string) => {
+ const table = kind === "^" ? SUPERSCRIPT_DIGITS : SUBSCRIPT_DIGITS;
+ return `${base}${[...(bracedDigits || digits)].map((digit) => table[Number(digit)]).join("")}`;
+ },
+ );
}
function stripIndentMarkers(value: string): string {
@@ -79,7 +94,7 @@ function lengthToIndentUnits(value: string): number {
function declaredIndentWidth(tag: string): number {
const name = tag.match(/^<\/?\s*([a-z0-9:]+)/i)?.[1]?.toLowerCase() ?? "";
- let width = name === "blockquote" || name === "ul" || name === "ol" ? 4 : 0;
+ let width = name === "blockquote" || name === "ul" || name === "ol" || name === "oi" ? 4 : 0;
const style = tag.match(/\bstyle\s*=\s*(["'])(.*?)\1/i)?.[2] ?? "";
for (const match of style.matchAll(
/(?:^|;)\s*(?:margin-left|padding-left|padding-inline-start|text-indent)\s*:\s*([^;]+)/gi,
@@ -104,17 +119,34 @@ function indentMarker(width: number): string {
}
function stripHtmlTags(text: string): string {
- text = normalizeMetricMarkup(text).replace(/]*>(.*?)<\/sup>/gi, "^$1");
+ text = text.replace(/]*>(.*?)<\/sup>/gi, "^$1");
+ let listDepth = 0;
const withBoundaries = text
.replace(BREAK_TAG, "\n")
.replace(BLOCK_TAG, (tag) => {
- if (/^<\//.test(tag)) return "\n\n";
+ const closing = /^<\//.test(tag);
+ const listContainer = /^<\s*\/?\s*(?:ul|ol|oi)\b/i.test(tag);
+ if (closing) {
+ if (listContainer) listDepth = Math.max(0, listDepth - 1);
+ return "\n\n";
+ }
+ if (listContainer) {
+ listDepth += 1;
+ return "\n\n";
+ }
+ if (/^<\s*\/?\s*(?:footnote|endnote|w:footnote|w:endnote)\b/i.test(tag)) {
+ return closing ? "\n\n" : `\n\n${FOOTNOTE_BLOCK_MARKER}`;
+ }
+ if (/^<\s*li\b/i.test(tag)) {
+ return `\n\n${indentMarker(Math.max(listDepth * 4, declaredIndentWidth(tag)))}`;
+ }
return `\n\n${indentMarker(declaredIndentWidth(tag))}`;
})
.replace(WORD_INDENT_TAG, (tag) => indentMarker(declaredIndentWidth(tag)));
- const withoutTags = withBoundaries.replace(HTML_TAG, (tag) =>
- /^<\/?w:/i.test(tag) ? "" : " ",
- );
+ const withoutTags = withBoundaries.replace(HTML_TAG, (tag) => {
+ if (/^<\/?w:/i.test(tag) || /^<\/?sup\b/i.test(tag)) return "";
+ return " ";
+ });
const decoded = decodeHtmlEntities(withoutTags);
return decoded
.split("\n")
@@ -212,10 +244,20 @@ function isDecodableBase64(raw: string): boolean {
}
function pushText(segments: PostBodySegment[], raw: string, indentUnit: number): void {
- const text = stripHtmlTags(raw);
+ const text = stripHtmlTags(
+ normalizeMetricMarkup(raw)
+ .replace(FOOTNOTE_BLOCK_OPEN, FOOTNOTE_BLOCK_MARKER)
+ .replace(NUMERIC_SUPERSCRIPT, `${NUMERIC_FOOTNOTE_MARKER}$1`),
+ );
+ let pendingFootnoteBlock = false;
for (const paragraph of splitSemanticParagraphs(text)) {
+ const hasNumericSuperscriptMarker = paragraph.includes(NUMERIC_FOOTNOTE_MARKER);
+ const hasFootnoteBlockMarker = paragraph.includes(FOOTNOTE_BLOCK_MARKER);
+ pendingFootnoteBlock ||= hasFootnoteBlockMarker;
const indentLevel = indentationLevel(paragraph, indentUnit);
- const normalized = stripIndentMarkers(paragraph)
+ const normalized = stripIndentMarkers(
+ paragraph.replaceAll(NUMERIC_FOOTNOTE_MARKER, "").replaceAll(FOOTNOTE_BLOCK_MARKER, ""),
+ )
.replace(/^[ \t]+/, "")
.replace(/[ \t]+$/gm, "");
if (normalized.trim()) {
@@ -223,10 +265,67 @@ function pushText(segments: PostBodySegment[], raw: string, indentUnit: number):
kind: "text",
text: normalized,
...(indentLevel > 0 ? { indentLevel } : {}),
- ...(FOOTNOTE_START.test(normalized) ? { role: "footnote" as const } : {}),
+ ...(hasNumericSuperscriptMarker || pendingFootnoteBlock || FOOTNOTE_START.test(normalized)
+ ? { role: "footnote" as const }
+ : {}),
});
+ pendingFootnoteBlock = false;
+ }
+ }
+}
+
+function markdownCells(line: string): string[] | null {
+ const value = line.trim().replace(/^\|/, "").replace(/(? cell.trim().replace(/\\\|/g, "|"));
+ return cells.length >= 2 && cells.every(Boolean) ? cells.map(normalizeMetricMarkup) : null;
+}
+
+function isMarkdownSeparatorRow(cells: string[] | null): boolean {
+ return Boolean(cells?.every((cell) => /^:?-{3,}:?$/.test(cell)));
+}
+
+/**
+ * Split the narrow Markdown-table shape supported by the ingestion boundary.
+ * The return value preserves prose and skips only the delimiter row.
+ */
+export function splitMarkdownTableBody(body: string): MarkdownBodyBlock[] | null {
+ const lines = body.replace(/\r\n?/g, "\n").split("\n");
+ const blocks: MarkdownBodyBlock[] = [];
+ let prose: string[] = [];
+ let foundTable = false;
+
+ const flushProse = () => {
+ const text = prose.join("\n").trim();
+ if (text) blocks.push({ kind: "prose", text });
+ prose = [];
+ };
+
+ let index = 0;
+ while (index < lines.length) {
+ const header = markdownCells(lines[index]);
+ const separator = markdownCells(lines[index + 1] ?? "");
+ if (!header || !isMarkdownSeparatorRow(separator)) {
+ prose.push(lines[index]);
+ index += 1;
+ continue;
}
+
+ foundTable = true;
+ flushProse();
+ const rows = [header];
+ index += 2;
+ while (index < lines.length && lines[index].trim()) {
+ const row = markdownCells(lines[index]);
+ if (!row) break;
+ rows.push(row);
+ index += 1;
+ }
+ blocks.push({ kind: "table", rows });
}
+
+ flushProse();
+ return foundTable ? blocks : null;
}
export function splitPostBody(body: string): PostBodySegment[] {
diff --git a/lineageweave/chunking.py b/lineageweave/chunking.py
index 61b5f28cd..64b8aed53 100644
--- a/lineageweave/chunking.py
+++ b/lineageweave/chunking.py
@@ -65,9 +65,10 @@
"footer",
"div",
"p",
- "ol",
- "ul",
"li",
+ "ul",
+ "ol",
+ "oi",
"footnote",
"endnote",
"w:footnote",
@@ -97,11 +98,12 @@
# readable and attributable as one unit.
_TABLE_ROW_TAGS = frozenset({"tr", "w:tr"})
_TABLE_CELL_TAGS = frozenset({"td", "th", "w:tc"})
+_LIST_CONTAINER_TAGS = frozenset({"ul", "ol", "oi"})
_LIST_ITEM_START = re.compile(
r"^(?:[-*•·]\s+|[*†‡](?=\S)|(?:\d{1,3}|[A-Za-z가-힣])[.)]\s+|[①-⑳]\s+)"
)
-_FOOTNOTE_START = re.compile(r"^[*†‡](?=\S)")
+_FOOTNOTE_START = re.compile(r"^[*†‡]+(?=\S)")
def _is_footnote_block(tag: str, attrs: list[tuple[str, str | None]]) -> bool:
@@ -132,6 +134,7 @@ def _is_footnote_reference(attrs: list[tuple[str, str | None]]) -> bool:
def normalize_semantic_text(text: str) -> str:
"""Remove visual hanging-indent breaks without changing source content."""
+ text = _normalize_metric_markup(_normalize_plain_metric_scripts(text))
lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n")
normalized: list[str] = []
for line in lines:
@@ -199,7 +202,7 @@ def _shorthand_left_value(raw: str) -> str:
def _declared_indent_width(tag: str, attrs: list[tuple[str, str | None]]) -> int:
"""Read HTML CSS and WordprocessingML paragraph indentation declarations."""
- width = 4 if tag in {"blockquote", "ul", "ol"} else 0
+ width = 4 if tag in {"blockquote", "ul", "ol", "oi"} else 0
style = next((value or "" for name, value in attrs if name == "style"), "")
for match in re.finditer(
r"(?:^|;)\s*(?:margin-left|padding-left|padding-inline-start|text-indent)\s*:\s*([^;]+)",
@@ -301,6 +304,21 @@ def chunk_by_paragraph(text: str) -> list[Chunk]:
r"<(?Psup|sub)\b[^>]*>\s*(?P\d{1,3})\s*",
re.IGNORECASE,
)
+_METRIC_PLAIN_SCRIPT = re.compile(
+ r"(?P\d{1,3})\}|(?P\d{1,3}))",
+ re.IGNORECASE,
+)
+
+
+def _normalize_plain_metric_scripts(text: str) -> str:
+ """Normalize bounded plain-text metric exponents and indices."""
+ def replace(match: re.Match[str]) -> str:
+ table = _SUPERSCRIPT_DIGITS if match.group("kind") == "^" else _SUBSCRIPT_DIGITS
+ digits = match.group("braced_digits") or match.group("digits") or ""
+ return f"{match.group('base')}{digits.translate(table)}"
+
+ return _METRIC_PLAIN_SCRIPT.sub(replace, text)
def _normalize_metric_markup(html: str) -> str:
@@ -361,15 +379,27 @@ class _BlockTextExtractor(HTMLParser):
"""
def __init__(self) -> None:
+ """Initialize parser buffers for ordered text, image, and footnote units."""
super().__init__()
self._stack: list[tuple[str, list[str], str | None, int, bool]] = []
self._unscoped_buffer: list[str] = []
+ self._active_superscripts: list[tuple[int, list[str]]] = []
+ self._numeric_superscript_buffers: set[int] = set()
# Each entry is ("text", str, tag_name, style) or
# ("image", (mime_type, bytes), "", None) -- a single sequence in
# true document order, so an image's index among its siblings
# reflects where it actually sat.
self._finished: list[tuple[str, object, str, str | None, int, int]] = []
+ def _declared_stack_width(self) -> int:
+ """Combine list depth with explicit width without double counting."""
+ list_depth = sum(entry[0] in _LIST_CONTAINER_TAGS for entry in self._stack)
+ explicit_width = sum(
+ max(0, entry[3] - 4) if entry[0] in _LIST_CONTAINER_TAGS else entry[3]
+ for entry in self._stack
+ )
+ return max(explicit_width, list_depth * 4)
+
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
"""Collect relevant text state when an HTML start tag is encountered."""
if tag == "img":
@@ -379,6 +409,10 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None
if decoded is not None:
self._finished.append(("image", decoded, "", None, 0, 0))
return
+ if tag == "sup":
+ if self._stack:
+ self._active_superscripts.append((id(self._stack[-1][1]), []))
+ return
if tag in {"br", "w:br"} and self._stack:
self._stack[-1][1].append("\n")
return
@@ -400,15 +434,39 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None
if self._stack and self._stack[-1][0] in _TABLE_ROW_TAGS and self._stack[-1][1]:
self._stack[-1][1].append(" | ")
return
- # A rich-text editor commonly wraps a table cell in a nested
1 Source note
")).toEqual([ + { kind: "text", text: "1 Source note", role: "footnote" }, + ]); + }); + + it("preserves explicit metric superscripts and subscripts", () => { + expect(splitPostBody("Volume: 5m3, index m3.
")).toEqual([ + { kind: "text", text: "Volume: 5m³, index m₃." }, + ]); + }); + + it("normalizes plain-text metric superscripts and subscripts", () => { + expect(splitPostBody("Volume: 5m^3, index m_3, braced m^{2}.
")).toEqual([ + { kind: "text", text: "Volume: 5m³, index m₃, braced m²." }, + ]); + }); + + it("preserves HTML and Word footnote blocks as footnote paragraphs", () => { + expect( + splitPostBody( + "Body evidence.
" + + "Next action.
", + ), + ).toEqual([ + { kind: "text", text: "Body evidence." }, + { kind: "text", text: "HTML note.", role: "footnote" }, + { kind: "text", text: "Word note.", role: "footnote" }, + { kind: "text", text: "Next action." }, + ]); + }); + + it("limits numeric superscript footnote roles to their source paragraph", () => { + expect( + splitPostBody( + "Evidence remains attached to the source.
" + + "1 Source note.
" + + "Continue with the next source action.
", + ), + ).toEqual([ + { kind: "text", text: "Evidence remains attached to the source." }, + { kind: "text", text: "1 Source note.", role: "footnote" }, + { kind: "text", text: "Continue with the next source action." }, + ]); + }); + + it("keeps exporter list containers in the visible nesting hierarchy", () => { + expect(splitPostBody("- Child
- First
- Second
- First
- Second
or - #
. Keep that content in the open row; otherwise the nested block
- # closes first and destroys the row/column boundary.
+ # Nested blocks belong to the open table row. Keep list-item text
+ # readable without letting a nested list become a separate chunk.
if any(entry[0] in _TABLE_ROW_TAGS for entry in self._stack):
+ if tag == "li" and self._stack[-1][1]:
+ self._stack[-1][1].append(" ")
+ return
+ if tag in _LIST_CONTAINER_TAGS:
+ # Emit a parent list item before entering its nested list. Closing
+ # tags otherwise make the child appear before the parent in the
+ # finished list, which destroys the source order buyers use to
+ # read a hierarchy.
+ if self._stack and self._stack[-1][0] == "li" and self._stack[-1][1]:
+ tag_name, buffer, style, indent_width, is_footnote = self._stack[-1]
+ self._stack[-1] = (tag_name, [], style, indent_width, is_footnote)
+ self._finish_block(
+ tag_name,
+ buffer,
+ style,
+ self._declared_stack_width(),
+ is_footnote,
+ )
+ style = next((value for name, value in attrs if name == "style" and value), None)
+ is_footnote = _is_footnote_block(tag, attrs) or any(
+ entry[4] for entry in self._stack
+ )
+ self._stack.append(
+ (tag, [], style, _declared_indent_width(tag, attrs), is_footnote)
+ )
return
if tag in _DOM_BLOCK_TAGS:
if self._stack and self._stack[-1][1]:
tag_name, buffer, style, _, is_footnote = self._stack[-1]
- declared_width = sum(entry[3] for entry in self._stack)
+ declared_width = self._declared_stack_width()
self._finish_block(tag_name, buffer, style, declared_width, is_footnote)
buffer.clear()
style = next((value for name, value in attrs if name == "style" and value), None)
@@ -427,8 +485,14 @@ def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> N
def handle_endtag(self, tag: str) -> None:
"""Close the relevant text state when an HTML end tag is encountered."""
+ if tag == "sup":
+ if self._active_superscripts:
+ buffer_id, content = self._active_superscripts.pop()
+ if re.fullmatch(r"\s*\d{1,3}\s*", "".join(content)):
+ self._numeric_superscript_buffers.add(buffer_id)
+ return
if tag in _DOM_BLOCK_TAGS and self._stack and self._stack[-1][0] == tag:
- declared_width = sum(entry[3] for entry in self._stack)
+ declared_width = self._declared_stack_width()
tag_name, buffer, style, _, is_footnote = self._stack.pop()
self._finish_block(tag_name, buffer, style, declared_width, is_footnote)
@@ -442,11 +506,16 @@ def _finish_block(
) -> None:
"""Emit one block buffer, including a block closed only at EOF."""
raw_text = "".join(buffer)
+ superscript_marker = id(buffer) in self._numeric_superscript_buffers
for raw_unit, source_indent in _split_dom_units(raw_text):
text = normalize_semantic_text(raw_unit)
if text:
indent_width = declared_width + source_indent
- label = "footnote" if is_footnote or _FOOTNOTE_START.match(text) else tag_name
+ label = (
+ "footnote"
+ if is_footnote or superscript_marker or _FOOTNOTE_START.match(text)
+ else tag_name
+ )
self._finished.append(
(
"text",
@@ -457,6 +526,7 @@ def _finish_block(
declared_width,
)
)
+ self._numeric_superscript_buffers.discard(id(buffer))
def handle_data(self, data: str) -> None:
"""Collect character data from the current HTML text region."""
@@ -466,6 +536,8 @@ def handle_data(self, data: str) -> None:
if decoded == text:
break
text = decoded
+ if self._active_superscripts:
+ self._active_superscripts[-1][1].append(text)
had_nbsp = "\xa0" in text
text = text.replace("\xa0", " ")
if self._stack and (text.strip() or had_nbsp):
@@ -476,7 +548,7 @@ def handle_data(self, data: str) -> None:
def finished(self) -> list[tuple[str, object, str, str | None, int, int]]:
"""Return the normalized records collected from the HTML fragment."""
while self._stack:
- declared_width = sum(entry[3] for entry in self._stack)
+ declared_width = self._declared_stack_width()
tag_name, buffer, style, _, is_footnote = self._stack.pop()
self._finish_block(tag_name, buffer, style, declared_width, is_footnote)
if not self._finished:
@@ -494,10 +566,10 @@ def _split_dom_units(raw_text: str) -> list[tuple[str, int]]:
current: list[str] = []
def flush() -> None:
+ """Move the current non-empty DOM unit into the result list."""
if current:
raw_unit = "\n".join(current)
- if raw_unit.strip():
- units.append((raw_unit, _source_indent_width(raw_unit)))
+ units.append((raw_unit, _source_indent_width(raw_unit)))
current.clear()
for line in raw_text.replace("\r\n", "\n").replace("\r", "\n").split("\n"):
@@ -511,6 +583,62 @@ def flush() -> None:
return units
+_MARKDOWN_SEPARATOR_CELL = re.compile(r"^:?-{3,}:?$")
+
+
+def _markdown_cells(line: str) -> list[str] | None:
+ """Return Markdown table cells, or ``None`` for a non-table line."""
+ if "|" not in line:
+ return None
+ value = line.strip()
+ value = value.removeprefix("|")
+ if value.endswith("|") and not value.endswith("\\|"):
+ value = value[:-1]
+ cells = [cell.strip().replace(r"\|", "|") for cell in re.split(r"(?= 2 and all(cells) else None
+
+
+def _markdown_table_entries(text: str) -> list[tuple[str, str]]:
+ """Extract table rows while retaining non-table prose around the table."""
+ lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n")
+ entries: list[tuple[str, str]] = []
+ pending: list[str] = []
+ found_table = False
+
+ def flush_pending() -> None:
+ """Emit prose accumulated outside a recognized Markdown table."""
+ if pending:
+ value = normalize_semantic_text("\n".join(pending))
+ if value:
+ entries.append(("", value))
+ pending.clear()
+
+ index = 0
+ while index < len(lines):
+ header = _markdown_cells(lines[index])
+ separator = _markdown_cells(lines[index + 1]) if index + 1 < len(lines) else None
+ if header is None or separator is None or not all(
+ _MARKDOWN_SEPARATOR_CELL.fullmatch(cell) for cell in separator
+ ):
+ pending.append(lines[index])
+ index += 1
+ continue
+
+ found_table = True
+ flush_pending()
+ entries.append(("markdown_tr", " | ".join(normalize_semantic_text(cell) for cell in header)))
+ index += 2
+ while index < len(lines) and lines[index].strip():
+ cells = _markdown_cells(lines[index])
+ if cells is None:
+ break
+ entries.append(("markdown_tr", " | ".join(normalize_semantic_text(cell) for cell in cells)))
+ index += 1
+
+ flush_pending()
+ return entries if found_table else []
+
+
_MARKDOWN_TABLE_SEPARATOR = re.compile(
r"^\s*\|?\s*:?-{3,}:?\s*(?:\|\s*:?-{3,}:?\s*)+\|?\s*$"
)
@@ -524,7 +652,10 @@ def _is_markdown_table_row(line: str) -> bool:
def _render_markdown_table_row(line: str) -> str:
"""Keep Markdown table columns as searchable row evidence."""
- return " | ".join(cell.strip() for cell in line.strip().strip("|").split("|"))
+ return " | ".join(
+ normalize_semantic_text(cell.strip())
+ for cell in line.strip().strip("|").split("|")
+ )
def _split_plain_text_units(text: str) -> list[tuple[str, int, str]]:
@@ -534,11 +665,11 @@ def _split_plain_text_units(text: str) -> list[tuple[str, int, str]]:
current: list[str] = []
def flush() -> None:
+ """Emit the current authored plain-text semantic unit."""
if current:
raw_unit = "\n".join(current)
normalized = normalize_semantic_text(raw_unit)
- if normalized:
- units.append((normalized, _source_indent_width(raw_unit), ""))
+ units.append((normalized, _source_indent_width(raw_unit), ""))
current.clear()
index = 0
@@ -589,6 +720,19 @@ def chunk_by_dom(html: str) -> list[Chunk]:
is what lets the image be placed back where it actually was relative
to the surrounding text chunks.
"""
+ if "<" not in html:
+ markdown_entries = _markdown_table_entries(html)
+ if markdown_entries:
+ return [
+ Chunk(
+ text=text,
+ unit_type="plain_text",
+ index=index,
+ label=label,
+ )
+ for index, (label, text) in enumerate(markdown_entries)
+ ]
+
parser = _BlockTextExtractor()
parser.feed(_normalize_metric_markup(html))
entries = parser.finished()
diff --git a/lineageweave/image_content.py b/lineageweave/image_content.py
index f6500ffd3..a0d298a91 100644
--- a/lineageweave/image_content.py
+++ b/lineageweave/image_content.py
@@ -141,7 +141,8 @@ class ImageDescription:
Attributes:
extracted_text: OCR result -- every piece of legible text found in
the image, empty string if none.
- caption: one-sentence description of what the image shows.
+ caption: factual description of the visible entities, relationships,
+ and layout that make the image useful as semantic evidence.
tags: short tags for the main objects/subjects, for independent
keyword search separate from the free-text caption.
"""
@@ -177,15 +178,18 @@ def describe(self, image_bytes: bytes, mime_type: str) -> ImageDescription: # p
_RESPONSE_FORMAT = (
- "Examine this image. Reply with EXACTLY three lines, no extra commentary:\n"
+ "Examine this image. Reply with exactly the three labeled sections below and no "
+ "extra commentary. TEXT may span multiple lines; CAPTION and TAGS stay on their "
+ "labeled lines.\n"
"TEXT: \n"
+ "If the image contains a table, preserve its row/column structure as a Markdown "
+ "pipe table: one row per line, with a separator row immediately after the visible "
+ "header. Never flatten a table into an unstructured word list or invent a header "
+ "that is not visible.>\n"
"CAPTION: <2-4 concise, evidence-grounded sentences describing the visible layout, "
"objects, relationships, directions, measurements, and labels; do not guess "
- "anything that is not visible>\n"
- "TAGS: "
+ "anything that is not visible. Omit anything the pixels do not support.>\n"
+ "TAGS: "
)
_REGION_RESPONSE_FORMAT = (
"Find distinct meaningful visual regions in this image for separate OCR and description. "
@@ -251,14 +255,15 @@ def _parse_description(content: str) -> ImageDescription:
remainder = _strip_outer_markdown_emphasis(match.group(2))
if remainder:
fields[label].append(remainder)
- multiline_field = "TEXT" if label == "TEXT" else None
+ multiline_field = label if label in {"TEXT", "CAPTION"} else None
continue
- if re.match(r"^\s*[*_`>#\-\s]*[A-Za-z][A-Za-z0-9 _-]*\s*:", line):
- multiline_field = None
- continue
- if multiline_field == "TEXT" and line.strip():
- fields["TEXT"].append(_strip_outer_markdown_emphasis(line))
+ if multiline_field in {"TEXT", "CAPTION"} and line.strip():
+ # A colon is common inside OCR (for example ``Date: 2026-08-21``).
+ # Only the known response labels above end the active section;
+ # treating every colon as a provider label loses real image text
+ # or the continuation of a detailed caption.
+ fields[multiline_field].append(_strip_outer_markdown_emphasis(line))
if not fields["TEXT"] and not fields["CAPTION"]:
raise ImageDescriptionParseError("vision response had no usable TEXT or CAPTION content")
@@ -289,7 +294,7 @@ def __init__(
api_key: str,
model: str | None = None,
*,
- timeout: float = 180.0,
+ timeout: float = 600.0,
allow_insecure_http: bool = False,
) -> None:
parsed = urlparse(base_url)
diff --git a/lineageweave/post_content_normalization.py b/lineageweave/post_content_normalization.py
index 196b7e7b4..2e2fe2c42 100644
--- a/lineageweave/post_content_normalization.py
+++ b/lineageweave/post_content_normalization.py
@@ -40,8 +40,8 @@
# what chunk_by_dom already splits on, plus the inline/replaced tags
# that carry images or wrap rich-text fragments.
_HTML_OPEN_TAG = re.compile(
- r"<\s*/?\s*(?:article|section|nav|aside|header|footer|div|p|li|td|th|tr|"
- r"table|blockquote|h[1-6]|img|br|hr|ul|ol|span|strong|em|b|i|u|a|"
+ r"<\s*/?\s*(?:article|section|nav|aside|header|footer|div|p|li|td|th|tr|sup|"
+ r"table|blockquote|h[1-6]|img|br|hr|ul|ol|oi|span|strong|em|b|i|u|a|"
r"html|body|head|style|script|font|center|pre)\b",
re.IGNORECASE,
)
@@ -262,6 +262,11 @@ def normalize_post_body(
vision_client = NullImageContentClient()
if not _looks_like_html(body):
+ markdown_chunks = chunk_by_dom(body)
+ if any(chunk.label == "markdown_tr" for chunk in markdown_chunks):
+ return NormalizedPostContent(
+ text="\n\n".join(chunk.text for chunk in markdown_chunks if chunk.text)
+ )
return NormalizedPostContent(text=normalize_semantic_text(body))
chunks: list[Chunk] = chunk_by_dom(body)
diff --git a/lineageweave/post_content_persistence.py b/lineageweave/post_content_persistence.py
index 32caa944b..745e15599 100644
--- a/lineageweave/post_content_persistence.py
+++ b/lineageweave/post_content_persistence.py
@@ -30,6 +30,10 @@
_LOGGER = logging.getLogger(__name__)
+class ImageOcrPreservationError(RuntimeError):
+ """A retry would erase stronger OCR already persisted for the same image."""
+
+
def _bounded_unit_batches( # noqa: UP047 - retain Python 3.10 compatibility.
units: list[tuple[_BatchKey, str]],
) -> list[list[tuple[_BatchKey, str]]]:
@@ -84,12 +88,24 @@ async def persist_post_content(
Provider calls happen before the short database transaction. A failed or
unavailable embedding call writes no vector row; it never writes a zero or
- guessed vector. The raw body remains in ``source_post`` for future retry.
+ guessed vector. A same-image retry cannot replace non-empty persisted OCR
+ with an empty result. The raw body remains in ``source_post`` for future
+ retry.
"""
normalized = normalized_result or normalize_post_body(body, vision_client)
chunks = chunk_by_source_body(body)
image_results = {result.chunk_index: result for result in normalized.image_results}
formatting = {hint.chunk_index: hint.style for hint in normalized.formatting_hints}
+ image_ocr_by_sha256: dict[str, bool] = {}
+ for chunk in chunks:
+ if chunk.unit_type != "image" or chunk.image_data is None:
+ continue
+ result = image_results.get(chunk.index)
+ description = result.description if result else None
+ content_sha256 = hashlib.sha256(chunk.image_data).hexdigest()
+ image_ocr_by_sha256[content_sha256] = image_ocr_by_sha256.get(
+ content_sha256, False
+ ) or bool(description and description.extracted_text.strip())
prepared: list[tuple[Chunk, str, str | None]] = []
for chunk in chunks:
@@ -226,6 +242,29 @@ async def persist_post_content(
)
async with conn.transaction():
+ if image_ocr_by_sha256:
+ await conn.fetchval(
+ "select post_id from source_post where post_id = $1 for update",
+ post_id,
+ )
+ previous_images = await conn.fetch(
+ """
+ select image.content_sha256, image.extracted_text
+ from post_content_unit unit
+ join post_content_image image using (post_content_unit_id)
+ where unit.post_id = $1
+ and nullif(btrim(image.extracted_text), '') is not null
+ """,
+ post_id,
+ )
+ if any(
+ row["content_sha256"] in image_ocr_by_sha256
+ and not image_ocr_by_sha256[row["content_sha256"]]
+ for row in previous_images
+ ):
+ raise ImageOcrPreservationError(
+ "refusing to replace non-empty image OCR with an empty retry result"
+ )
await conn.execute("delete from post_content_unit where post_id = $1", post_id)
unit_ids: dict[int, str] = {}
for chunk, unit_text, style in prepared:
diff --git a/scripts/backfill_post_content.py b/scripts/backfill_post_content.py
index 0e566129b..7be9c3ab6 100644
--- a/scripts/backfill_post_content.py
+++ b/scripts/backfill_post_content.py
@@ -27,7 +27,10 @@
from lineageweave.image_content import NullImageContentClient, orchestrator_vision_client
from lineageweave.llm_context import build_post_llm_metadata, use_llm_metadata
from lineageweave.post_content_normalization import normalize_post_body
-from lineageweave.post_content_persistence import persist_post_content
+from lineageweave.post_content_persistence import (
+ ImageOcrPreservationError,
+ persist_post_content,
+)
from lineageweave.post_structure import ContextualOrchestratorPostStructureClient, NullPostStructureClient
@@ -55,6 +58,15 @@ def _parser() -> argparse.ArgumentParser:
return parser
+async def _ensure_open_connection(
+ conn: asyncpg.Connection, target_dsn: str
+) -> asyncpg.Connection:
+ """Reconnect after a database restart without repeating VISION work."""
+ if not conn.is_closed():
+ return conn
+ return await asyncpg.connect(target_dsn)
+
+
async def backfill_post_content(
target_dsn: str,
raw_post_ids: list[str] | None,
@@ -217,17 +229,22 @@ async def backfill_post_content(
if described_images == 0 and not normalized.text.strip():
result["skipped_posts"] += 1
continue
- await persist_post_content(
- conn,
- str(row["post_id"]),
- row["post_body"],
- vision_client=vision_client,
- embedding_client=embedding_client,
- embedding_model_code=embedding_model or None,
- normalized_result=normalized,
- structure_client=structure_client,
- post_title=row["post_title"],
- )
+ conn = await _ensure_open_connection(conn, target_dsn)
+ try:
+ await persist_post_content(
+ conn,
+ str(row["post_id"]),
+ row["post_body"],
+ vision_client=vision_client,
+ embedding_client=embedding_client,
+ embedding_model_code=embedding_model or None,
+ normalized_result=normalized,
+ structure_client=structure_client,
+ post_title=row["post_title"],
+ )
+ except ImageOcrPreservationError:
+ result["skipped_posts"] += 1
+ continue
async with conn.transaction():
await record_post_content_backfill_success(
conn,
diff --git a/tests/test_backfill_post_content.py b/tests/test_backfill_post_content.py
new file mode 100644
index 000000000..d5f034297
--- /dev/null
+++ b/tests/test_backfill_post_content.py
@@ -0,0 +1,135 @@
+"""Operator backfill connection recovery contracts."""
+
+from __future__ import annotations
+
+import asyncio
+from types import SimpleNamespace
+
+from scripts import backfill_post_content
+from lineageweave.post_content_persistence import ImageOcrPreservationError
+
+
+def test_reconnects_only_after_database_connection_closes(monkeypatch) -> None:
+ replacement_connection = object()
+ connected_dsns: list[str] = []
+
+ class Connection:
+ def __init__(self, closed: bool) -> None:
+ self._closed = closed
+
+ def is_closed(self) -> bool:
+ return self._closed
+
+ async def connect(dsn: str):
+ connected_dsns.append(dsn)
+ return replacement_connection
+
+ monkeypatch.setattr(backfill_post_content.asyncpg, "connect", connect)
+
+ current_connection = Connection(False)
+ assert (
+ asyncio.run(backfill_post_content._ensure_open_connection(current_connection, "dsn"))
+ is current_connection
+ )
+ assert asyncio.run(backfill_post_content._ensure_open_connection(Connection(True), "dsn")) is replacement_connection
+ assert connected_dsns == ["dsn"]
+
+
+def test_backfill_skips_ocr_protected_post_and_continues(monkeypatch) -> None:
+ post_ids = [
+ "00505695-0000-1fd1-8000-000000000001",
+ "00505695-0000-1fd1-8000-000000000002",
+ ]
+
+ class Transaction:
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, exc_type, exc, traceback):
+ return False
+
+ class Connection:
+ def is_closed(self) -> bool:
+ return False
+
+ async def fetch(self, query, *args):
+ return [{"post_id": post_id} for post_id in post_ids]
+
+ async def fetchrow(self, query, post_id):
+ return {
+ "post_id": post_id,
+ "post_title": "Synthetic title",
+ "post_body": "Synthetic body",
+ "author_account_id": None,
+ "source_process_unit_code": None,
+ "source_author_code": None,
+ "source_company_code": None,
+ "source_customer_code": None,
+ "source_project_code": None,
+ "source_sales_pool_code": None,
+ "corporate_entity_code": None,
+ }
+
+ def transaction(self):
+ return Transaction()
+
+ async def fetchval(self, query, *args):
+ return 0
+
+ async def close(self):
+ return None
+
+ connection = Connection()
+ persisted: list[str] = []
+
+ async def connect(_dsn):
+ return connection
+
+ async def persist(conn, post_id, body, **kwargs):
+ persisted.append(post_id)
+ if post_id == post_ids[0]:
+ raise ImageOcrPreservationError("protected")
+ return 1
+
+ async def record_success(conn, post_id, body):
+ return None
+
+ class MetadataContext:
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc, traceback):
+ return False
+
+ monkeypatch.setattr(backfill_post_content.asyncpg, "connect", connect)
+ monkeypatch.setattr(backfill_post_content, "persist_post_content", persist)
+ monkeypatch.setattr(
+ backfill_post_content,
+ "record_post_content_backfill_success",
+ record_success,
+ )
+ monkeypatch.setattr(
+ backfill_post_content,
+ "normalize_post_body",
+ lambda body, vision_client: SimpleNamespace(image_results=(), text="text"),
+ )
+ monkeypatch.setattr(
+ backfill_post_content,
+ "build_post_llm_metadata",
+ lambda post_id, row: {},
+ )
+ monkeypatch.setattr(
+ backfill_post_content,
+ "use_llm_metadata",
+ lambda metadata: MetadataContext(),
+ )
+
+ result = asyncio.run(
+ backfill_post_content.backfill_post_content(
+ "dsn", post_ids, limit=None, normalize_only=True
+ )
+ )
+
+ assert persisted == post_ids
+ assert result["processed_posts"] == 1
+ assert result["skipped_posts"] == 1
diff --git a/tests/test_chunking.py b/tests/test_chunking.py
index fe8792a07..b3ecc3d67 100644
--- a/tests/test_chunking.py
+++ b/tests/test_chunking.py
@@ -2,6 +2,8 @@
from lineageweave.chunking import (
ConversationTurn,
+ _length_to_indent_units,
+ _shorthand_left_value,
chunk_by_conversation_turn,
chunk_by_dom,
chunk_by_paragraph,
@@ -12,6 +14,7 @@
def test_chunk_by_paragraph_splits_on_blank_lines() -> None:
+ """Blank lines delimit ordered paragraph chunks."""
text = "First paragraph about budgets.\n\nSecond paragraph about logistics.\n\nThird."
chunks = chunk_by_paragraph(text)
@@ -25,17 +28,20 @@ def test_chunk_by_paragraph_splits_on_blank_lines() -> None:
def test_chunk_by_paragraph_ignores_extra_blank_lines_and_whitespace() -> None:
+ """Extra blank lines and outer whitespace do not create chunks."""
text = " A. \n\n\n\n B. "
chunks = chunk_by_paragraph(text)
assert [c.text for c in chunks] == ["A.", "B."]
def test_chunk_by_paragraph_empty_text_yields_no_chunks() -> None:
+ """Empty paragraph input produces no semantic units."""
assert chunk_by_paragraph("") == []
assert chunk_by_paragraph(" \n\n ") == []
def test_chunk_by_sentence_splits_on_sentence_boundaries() -> None:
+ """Sentence punctuation followed by a new sentence creates a boundary."""
text = "This is one sentence. This is another! Is this a third?"
chunks = chunk_by_sentence(text)
@@ -47,7 +53,22 @@ def test_chunk_by_sentence_splits_on_sentence_boundaries() -> None:
assert all(c.unit_type == "sentence" for c in chunks)
+def test_chunk_by_sentence_empty_text_yields_no_chunks() -> None:
+ """Whitespace alone has no sentence-level semantic unit."""
+ assert chunk_by_sentence(" \n ") == []
+
+
+def test_indent_helpers_cover_invalid_lengths_and_css_shorthand_shapes() -> None:
+ """Invalid and non-positive lengths stay flat; shorthand picks the left side."""
+ assert _length_to_indent_units("auto") == 0
+ assert _length_to_indent_units("-8px") == 0
+ assert _shorthand_left_value("") == ""
+ assert _shorthand_left_value("8px") == "8px"
+ assert _shorthand_left_value("8px 16px") == "16px"
+
+
def test_chunk_by_dom_splits_on_block_element_boundaries() -> None:
+ """Sibling DOM blocks remain separate semantic units."""
html = (
" used to push its own independent
chunk with no row grouping, so a real table (headers + N data rows)
@@ -98,13 +130,33 @@ def test_chunk_by_dom_groups_table_cells_by_row_instead_of_flattening() -> None:
def test_chunk_by_dom_keeps_nested_table_cell_blocks_in_their_row() -> None:
+ """Nested cell blocks retain their table-row grouping."""
chunks = chunk_by_dom(
"
"
)
assert [(chunk.label, chunk.text) for chunk in chunks] == [("tr", "No. | Company")]
+def test_chunk_by_dom_keeps_cell_lists_inside_their_table_row() -> None:
+ """A list inside a cell stays readable and grouped with its row."""
+ chunks = chunk_by_dom(
+ "
"
+ )
+
+ assert [(chunk.label, chunk.text) for chunk in chunks] == [
+ ("tr", "Items: A B | Owner")
+ ]
+
+ leading_list = chunk_by_dom(
+ "
"
+ )
+ assert [chunk.text for chunk in leading_list] == ["A B | Owner"]
+
+
def test_chunk_by_dom_labels_markerless_footnotes() -> None:
+ """A leading footnote marker assigns the footnote label."""
chunks = chunk_by_dom("First item Second item 1 Acme Corp Word
Nested paragraph text.
Parent text
")
+
+ assert [(chunk.label, chunk.text) for chunk in chunks] == [
+ ("div", "Parent text"),
+ ("p", "Child text"),
+ ]
+
+
def test_chunk_by_dom_groups_table_cells_by_row_instead_of_flattening() -> None:
"""Live bug (2026-08-19): each Child text
No. | Company |
Items:
| "
+ "Owner |
| "
+ "Owner |
Body text
*Tier 2: follow-up note
") assert [(chunk.label, chunk.text) for chunk in chunks] == [ ("p", "Body text"), @@ -112,6 +164,154 @@ def test_chunk_by_dom_labels_markerless_footnotes() -> None: ] +def test_chunk_by_dom_labels_numeric_superscript_footnotes() -> None: + """A leading numeric superscript assigns the footnote label.""" + chunks = chunk_by_dom("1 Source note attached to the record.
") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("footnote", "1 Source note attached to the record."), + ] + + +def test_chunk_by_dom_labels_numeric_superscript_after_body_text() -> None: + """A numeric superscript anywhere in a paragraph marks its evidence role.""" + chunks = chunk_by_dom("Body claim1 source note.
") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("footnote", "Body claim1 source note."), + ] + + +def test_chunk_by_dom_does_not_treat_non_numeric_superscript_as_footnote() -> None: + """A formula superscript remains ordinary prose.""" + chunks = chunk_by_dom("Formula xn remains prose.
") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("p", "Formula xn remains prose."), + ] + + +def test_chunk_by_dom_preserves_explicit_metric_superscripts() -> None: + """A unit exponent remains searchable mathematical evidence.""" + chunks = chunk_by_dom("Volume: 5m3.
") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("p", "Volume: 5m³."), + ] + + +def test_chunk_by_dom_preserves_explicit_metric_subscripts() -> None: + """A unit subscript is retained without changing ordinary footnotes.""" + chunks = chunk_by_dom("Index m3 is measured.
") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("p", "Index m₃ is measured."), + ] + + +def test_chunk_by_source_body_normalizes_plain_metric_scripts() -> None: + """Plain-text metric scripts retain searchable exponent/index semantics.""" + chunks = chunk_by_source_body("Volume: 5m^3; index m_3; braced m^{2}.") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("", "Volume: 5m³; index m₃; braced m²."), + ] + + +def test_chunk_by_source_body_normalizes_metric_scripts_in_markdown_table_cells() -> None: + """Markdown table cells retain the same searchable metric semantics as prose.""" + chunks = chunk_by_source_body( + "| Metric | Index |\n| --- | --- |\n| 5m^3 | m_3 |" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("tr", "Metric | Index"), + ("tr", "5m³ | m₃"), + ] + + +def test_chunk_by_dom_preserves_nested_list_order_and_depth() -> None: + """Nested list items retain source order and increasing depth.""" + chunks = chunk_by_dom( + "- Parent item
- Child item
Body text
" @@ -173,6 +373,7 @@ def test_chunk_by_dom_preserves_explicit_metric_subscripts() -> None: def test_chunk_by_dom_word_table_rows_also_group_cells() -> None: + """WordprocessingML table cells group by their source row.""" html = "Level one
Level two
" chunks = chunk_by_dom(html) @@ -190,6 +392,7 @@ def test_chunk_by_dom_keeps_indentation_as_metadata_not_embedding_text() -> None def test_chunk_by_dom_reads_html_and_word_indentation_declarations() -> None: + """HTML and Word indentation declarations map to comparable units.""" html = ( 'HTML
' '- Outer
- Nested
HTML evidence
") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [("p", "HTML evidence")] + + def test_chunk_by_dom_joins_visual_continuation_lines_but_keeps_list_items() -> None: + """Visual wraps join while authored list starts retain boundaries.""" html = ( '1. 배경
'
" 1) 기존 대차는 이전이 필요함
"
@@ -269,6 +499,7 @@ def test_chunk_by_dom_joins_visual_continuation_lines_but_keeps_list_items() ->
def test_normalize_semantic_text_removes_visual_hanging_indent_breaks() -> None:
+ """Hanging-indent line wraps normalize without flattening list items."""
text = (
"1. 배경\n\n"
" 1) 기존 대차는 이전이 필요함\n"
@@ -284,16 +515,19 @@ def test_normalize_semantic_text_removes_visual_hanging_indent_breaks() -> None:
def test_normalize_semantic_text_preserves_blank_paragraph_boundaries() -> None:
+ """Blank lines continue to separate authored paragraphs."""
assert normalize_semantic_text("첫 문단\n\n둘째 문단") == "첫 문단\n\n둘째 문단"
def test_normalize_semantic_text_does_not_embed_visual_indentation_markers() -> None:
+ """Presentation-only non-breaking spaces do not enter semantic text."""
assert normalize_semantic_text("\xa0\xa0계속되는 문장\n\xa0\xa0\xa0\xa0다음 줄") == (
"계속되는 문장 다음 줄"
)
def test_chunk_by_dom_does_not_infer_marker_depth_without_source_whitespace() -> None:
+ """Marker shape alone cannot invent indentation depth."""
chunks = chunk_by_dom("
1. Root
1) Child
- Detail
") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [("", "1")] + + +def test_chunk_by_dom_decodes_deeply_escaped_entities_with_a_bounded_loop() -> None: + """Nested HTML entity escapes decode without an unbounded parser loop.""" + assert [chunk.text for chunk in chunk_by_dom("
&amp;amp;
")] == ["&"] + + def test_chunk_by_dom_falls_back_for_inline_only_markup() -> None: + """Inline-only markup falls back to one plain-text unit.""" chunks = chunk_by_dom("First inline block.Second inline block.") assert len(chunks) == 1 @@ -314,6 +563,7 @@ def test_chunk_by_dom_falls_back_for_inline_only_markup() -> None: def test_chunk_by_dom_flushes_unclosed_block_at_end_of_document() -> None: + """EOF flushes content from an unclosed source block.""" chunks = chunk_by_dom("Unclosed source fragment.")
assert len(chunks) == 1
@@ -322,6 +572,7 @@ def test_chunk_by_dom_flushes_unclosed_block_at_end_of_document() -> None:
def test_chunk_by_dom_interleaves_images_with_text_in_document_order() -> None:
+ """Embedded images retain their exact position between text blocks."""
tiny_png_b64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="
html = (
f"
'
chunks = chunk_by_dom(html)
assert [c.unit_type for c in chunks] == ["dom"]
+def test_chunk_by_dom_skips_images_without_embedded_base64_data() -> None:
+ """Missing, external, and non-base64 image sources are not embedded images."""
+ html = '![]()
'
+ assert chunk_by_dom(html) == []
+
+
def test_chunk_by_conversation_turn_labels_each_chunk_with_its_sender() -> None:
+ """Conversation units retain their sender labels and order."""
turns = [
ConversationTurn(sender="alice@example.com", text="Can we move the meeting?"),
ConversationTurn(sender="bob@example.com", text="Sure, how about Thursday?"),
@@ -366,6 +626,7 @@ def test_chunk_by_conversation_turn_labels_each_chunk_with_its_sender() -> None:
def test_chunk_by_conversation_turn_skips_empty_turns() -> None:
+ """Empty turns are removed before contiguous chunk indexing."""
turns = [
ConversationTurn(sender="alice@example.com", text="Hello."),
ConversationTurn(sender="bob@example.com", text=" "),
@@ -408,11 +669,13 @@ def test_chunk_by_dom_captures_style_as_separate_metadata_not_embedded_text() ->
def test_chunk_by_dom_style_is_none_when_element_has_no_style_attribute() -> None:
+ """A missing style attribute remains distinct from an empty style value."""
chunks = chunk_by_dom("Parent 
'
+ image_index = chunk_by_dom(body)[0].index
+ normalized = NormalizedPostContent(
+ text="[image: updated caption]",
+ image_results=(
+ ImageContentResult(
+ image_index,
+ "image/png",
+ "described",
+ SimpleNamespace(caption="updated caption", extracted_text="", tags=()),
+ ),
+ ),
+ )
+ conn = _Connection(
+ (
+ {
+ "content_sha256": hashlib.sha256(b"hello").hexdigest(),
+ "description_status_code": "described",
+ "extracted_text": "prior OCR",
+ "caption": "prior caption",
+ },
+ )
+ )
+
+ with pytest.raises(RuntimeError, match="refusing to replace non-empty image OCR"):
+ _persist(conn, "post-4", body, normalized_result=normalized)
+
+ assert not any("delete from post_content_unit" in query for query, _args in conn.executed)
+
+
def test_legacy_embed_and_malformed_vectors_never_write_vectors() -> None:
conn = _Connection()
legacy = _LegacyEmbed([float("nan")])
Before the picture.
" @@ -339,6 +590,7 @@ def test_chunk_by_dom_interleaves_images_with_text_in_document_order() -> None: def test_chunk_by_dom_labels_text_chunks_with_their_tag_name() -> None: + """DOM text chunks expose their source tag as the unit label.""" html = "A paragraph.
Text.
'
chunks = chunk_by_dom(html)
assert [c.unit_type for c in chunks] == ["dom"]
+def test_chunk_by_dom_skips_images_without_embedded_base64_data() -> None:
+ """Missing, external, and non-base64 image sources are not embedded images."""
+ html = ''
+ assert chunk_by_dom(html) == []
+
+
def test_chunk_by_conversation_turn_labels_each_chunk_with_its_sender() -> None:
+ """Conversation units retain their sender labels and order."""
turns = [
ConversationTurn(sender="alice@example.com", text="Can we move the meeting?"),
ConversationTurn(sender="bob@example.com", text="Sure, how about Thursday?"),
@@ -366,6 +626,7 @@ def test_chunk_by_conversation_turn_labels_each_chunk_with_its_sender() -> None:
def test_chunk_by_conversation_turn_skips_empty_turns() -> None:
+ """Empty turns are removed before contiguous chunk indexing."""
turns = [
ConversationTurn(sender="alice@example.com", text="Hello."),
ConversationTurn(sender="bob@example.com", text=" "),
@@ -408,11 +669,13 @@ def test_chunk_by_dom_captures_style_as_separate_metadata_not_embedded_text() ->
def test_chunk_by_dom_style_is_none_when_element_has_no_style_attribute() -> None:
+ """A missing style attribute remains distinct from an empty style value."""
chunks = chunk_by_dom("Plain paragraph.
") assert chunks[0].style is None def test_chunk_by_dom_splits_on_heading_boundaries_and_labels_the_level() -> None: + """Heading boundaries preserve their source level label.""" html = "Quarterly Review
Body text follows.
" chunks = chunk_by_dom(html) diff --git a/tests/test_contextual_orchestrator_start.py b/tests/test_contextual_orchestrator_start.py index e3618aa82..a4f6bcfad 100644 --- a/tests/test_contextual_orchestrator_start.py +++ b/tests/test_contextual_orchestrator_start.py @@ -52,6 +52,15 @@ def test_gateway_api_key_accepts_local_compatibility_alias(monkeypatch) -> None: assert module._pop_first_env("LLM_GATEWAY_API_KEY", "LLM_API_KEY") == "compatibility-key" +def test_env_file_quotes_are_not_part_of_transport_values(monkeypatch) -> None: + module = _load_start_module() + monkeypatch.setenv("LLM_GATEWAY_API_KEY", "'provider-key'") + monkeypatch.setenv("LLM_GATEWAY_API_URL", '"https://gateway.example/v1"') + + assert module._pop_first_env("LLM_GATEWAY_API_KEY") == "provider-key" + assert module._pop_first_env("LLM_GATEWAY_API_URL") == "https://gateway.example/v1" + + def test_bootstrap_registers_embedding_agent_before_deleting_secrets(monkeypatch) -> None: module = _load_start_module() captured: dict[str, object] = {} @@ -90,9 +99,11 @@ def serve() -> None: monkeypatch.setattr(module, "Path", FakePath) monkeypatch.setattr(sys, "argv", ["start.py"]) monkeypatch.setenv("LLM_GATEWAY_API_KEY", "provider-key") - monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_TOKEN", "orchestrator-token") + monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_TOKEN", "'orchestrator-token'") monkeypatch.setenv("LLM_GATEWAY_API_URL", "https://gateway.example") - monkeypatch.setenv("LLM_GATEWAY_EMBEDDING_MODEL", "embedding-model") + monkeypatch.setenv("LLM_GATEWAY_EMBEDDING_MODEL", "'embedding-model'") + monkeypatch.setenv("LLM_GATEWAY_MAX_OUTPUT_TOKENS", "'2048'") + monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_MAX_BODY_BYTES", '"65536"') module.main() @@ -100,6 +111,9 @@ def serve() -> None: assert isinstance(argv, list) assert "--embedding-provider-url" not in argv assert "--embedding-model" not in argv + assert argv[argv.index("--auth-token") + 1] == "orchestrator-token" + assert argv[argv.index("--max-output-tokens") + 1] == "2048" + assert argv[argv.index("--max-body-bytes") + 1] == "65536" assert captured["credentials"] == [ ("NVIDIA_NIM_API_KEY", "provider-key"), ("LLM_GATEWAY_API_KEY", "provider-key"), diff --git a/tests/test_image_content.py b/tests/test_image_content.py index de7fc49b5..148a979c5 100644 --- a/tests/test_image_content.py +++ b/tests/test_image_content.py @@ -85,6 +85,35 @@ def test_parse_description_preserves_multiline_ocr_text() -> None: assert description.caption == "A scanned page." +def test_parse_description_preserves_multiline_caption_evidence() -> None: + """Detailed VISION captions remain complete when providers wrap lines.""" + content = ( + "CAPTION: A project status table for the customer meeting.\n" + "The left column lists workstreams and the right column lists owners.\n" + "TEXT: Workstream | Owner\nAlpha | Team A\nTAGS: table, assignment" + ) + + description = _parse_description(content) + + assert description.caption == ( + "A project status table for the customer meeting.\n" + "The left column lists workstreams and the right column lists owners." + ) + assert description.extracted_text == "Workstream | Owner\nAlpha | Team A" + + +def test_parse_description_preserves_ocr_lines_that_contain_colons() -> None: + """A colon in a scanned field is OCR content, not a new response field.""" + content = ( + "TEXT: Invoice\nDate: 2026-08-21\nTotal: 100\n" + "CAPTION: A synthetic invoice.\nTAGS: invoice" + ) + + description = _parse_description(content) + + assert description.extracted_text == "Invoice\nDate: 2026-08-21\nTotal: 100" + + def test_parse_description_preserves_table_row_structure_in_ocr_text() -> None: """Live gap (2026-08-19): an image containing a table used to have its text flattened into an unstructured word list on OCR, the same @@ -202,6 +231,14 @@ def test_orchestrator_vision_client_does_not_double_v1() -> None: assert client._base_url == "https://gateway.example/v1" +def test_orchestrator_vision_client_allows_deep_agent_runtime() -> None: + """A valid VISION result must not be cut off by the former 180s limit.""" + client = orchestrator_vision_client("https://gateway.example", "key") + + assert isinstance(client, OpenAiCompatibleVisionClient) + assert client._timeout == 600.0 + + def test_orchestrator_vision_client_is_null_when_unconfigured() -> None: client = orchestrator_vision_client("", "") assert isinstance(client, NullImageContentClient) @@ -226,6 +263,17 @@ def test_ocr_prompt_asks_for_table_row_structure() -> None: assert "table" in _RESPONSE_FORMAT.lower() +def test_ocr_prompt_allows_multiline_tables_and_requests_semantic_detail() -> None: + """Table rows and ontology-ready captions must fit the response contract.""" + prompt = _RESPONSE_FORMAT.lower() + + assert "text may span multiple lines" in prompt + assert "separator row" in prompt + assert "named entities" in prompt + assert "relationships" in prompt + assert "exactly three lines" not in prompt + + def test_region_prompt_requires_full_image_coverage() -> None: """Live gap (2026-08-19): "distinct meaningful visual regions" alone let the model describe only the most visually striking part of an diff --git a/tests/test_person_mention_projection.py b/tests/test_person_mention_projection.py index 03d6fcf82..1d1b96ed4 100644 --- a/tests/test_person_mention_projection.py +++ b/tests/test_person_mention_projection.py @@ -103,6 +103,9 @@ _SOURCE_ORG_NAMED_HINTS_MIGRATION = ( Path(__file__).resolve().parents[1] / "migrations" / "0039_source_org_named_hints.sql" ) +_MAJOR_EVENT_ACTION_MIGRATION = ( + Path(__file__).resolve().parents[1] / "migrations" / "0100_major_event_action.sql" +) def _postgres_available() -> bool: diff --git a/tests/test_post_content_normalization.py b/tests/test_post_content_normalization.py index 18e046b8b..0007df6c9 100644 --- a/tests/test_post_content_normalization.py +++ b/tests/test_post_content_normalization.py @@ -122,6 +122,20 @@ def test_plain_text_visual_continuation_breaks_are_normalized_for_embeddings() - assert result.text == "- 요청 사항 후속 설명은 같은 항목에 속한다.\n· 다음 항목" +def test_markdown_table_rows_remain_separate_in_normalized_evidence() -> None: + result = normalize_post_body( + "| Project | Status |\n| --- | --- |\n| Alpha | Ready |" + ) + + assert result.text == "Project | Status\n\nAlpha | Ready" + + +def test_exporter_oi_lists_are_normalized_as_html_evidence() -> None: + result = normalize_post_body("- Child
Confirm delivery by Friday.
'
+ image_index = chunk_by_dom(body)[0].index
+ normalized = NormalizedPostContent(
+ text="[image: updated caption]",
+ image_results=(
+ ImageContentResult(
+ image_index,
+ "image/png",
+ "described",
+ SimpleNamespace(caption="updated caption", extracted_text="", tags=()),
+ ),
+ ),
+ )
+ conn = _Connection(
+ (
+ {
+ "content_sha256": hashlib.sha256(b"hello").hexdigest(),
+ "description_status_code": "described",
+ "extracted_text": "prior OCR",
+ "caption": "prior caption",
+ },
+ )
+ )
+
+ with pytest.raises(RuntimeError, match="refusing to replace non-empty image OCR"):
+ _persist(conn, "post-4", body, normalized_result=normalized)
+
+ assert not any("delete from post_content_unit" in query for query, _args in conn.executed)
+
+
def test_legacy_embed_and_malformed_vectors_never_write_vectors() -> None:
conn = _Connection()
legacy = _LegacyEmbed([float("nan")])