diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md new file mode 100644 index 00000000..d181f082 --- /dev/null +++ b/THIRD_PARTY_NOTICES.md @@ -0,0 +1,14 @@ +# Third-party notices + +## RAGFlow Markdown parser + +- Project: RAGFlow +- Upstream repository: +- Pinned commit: `4391e03886b996201f3b8818f671b19eb24d0f7b` +- Registered source: `deepdoc/parser/markdown_parser.py` +- License: Apache License 2.0 + +The verbatim upstream license and the complete path-level registration ship in +`third_party/ragflow/`. The registered parser imports Python-Markdown, which is +distributed under the BSD 3-Clause License; its verbatim license ships beside +the registration as `LICENSE.python-markdown`. diff --git a/adapters/parsers/ragflow_markdown.py b/adapters/parsers/ragflow_markdown.py new file mode 100644 index 00000000..15b358fc --- /dev/null +++ b/adapters/parsers/ragflow_markdown.py @@ -0,0 +1,752 @@ +"""Rich Markdown compiler built around the registered RAGFlow parser region.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any, Final, Protocol, cast + +from engine.supply.markdown import ( + MARKDOWN_CODE_LANGUAGE_MAX_LENGTH, + MARKDOWN_COMPILER_V3_VERSION, + MARKDOWN_RICH_CANONICALIZATION_PROFILE, + MARKDOWN_RICH_COMPILATION_DIGEST_PROFILE, + CompilationFailure, + CompilationFailureCode, + CompilationOutcome, + CompilationProvenance, + CompiledFragment, + MarkdownCompilerConfig, + ParsedDocument, + ParsedSection, + SectionKind, + SourcePoint, + SourceSpan, + StructuralPath, + UnsupportedConstruct, + is_markdown_control_character, + unsupported_rich_markdown_inline, +) +from third_party.ragflow.deepdoc.parser.markdown_parser import MarkdownElementExtractor + +_UTF8_BOM: Final = b"\xef\xbb\xbf" +_ATX_HEADING: Final = re.compile(r"^ {0,3}(#{1,6})[ \t]+(.+?)[ \t]*#*[ \t]*$") +_SETEXT: Final = re.compile(r"^ {0,3}(=+|-+)[ \t]*$") +_LIST_ITEM: Final = re.compile(r"^([ \t]*)(?:[-+*]|[0-9]+[.)])[ \t]+(.+)$") +_TOKEN: Final = re.compile(r"\S+") +_CALLOUT: Final = re.compile(r"^ {0,3}>[ \t]*\[![A-Za-z0-9_-]+][+-]?(?: .*)?$") +_HTML_OPEN: Final = re.compile( + r"^ {0,3}<(?Psection|div|details|summary|table|thead|tbody|tr|" + r"th|td|p|ul|ol|li)\b[^>]*>", + re.IGNORECASE, +) +_ANGLE_LITERAL: Final = re.compile(r"<[^<>\r\n]+>") +_FRONTMATTER_MAPPING: Final = re.compile( + r"^[ \t]*(?:[A-Za-z0-9_.-]+|\"[^\"\r\n]+\"|'[^'\r\n]+')" + r"[ \t]*:[ \t]*(?:.*)?$" +) +_FRONTMATTER_SEQUENCE: Final = re.compile(r"^[ \t]*-[ \t]+\S.*$") + + +@dataclass(frozen=True, slots=True) +class _Block: + kind: SectionKind + start: int + end: int + indivisible: bool = False + level: int | None = None + list_ordered: bool | None = None + list_items: tuple[str, ...] = () + code_language: str | None = None + code_body: str | None = None + table_header: tuple[str, ...] = () + table_rows: tuple[tuple[str, ...], ...] = () + + +class _ElementExtractor(Protocol): + def _get_fence_marker(self, line: str) -> tuple[str, int] | None: ... + + def _is_closing_fence( + self, + line: str, + fence_char: str, + fence_len: int, + ) -> bool: ... + + def _table_cells(self, line: str) -> list[str]: ... + + def _is_table_row(self, line: str) -> bool: ... + + def _is_table_separator_row(self, line: str) -> bool: ... + + +def rich_token_count(value: str) -> int: + """Count deterministic representation tokens for the v3 hard bound.""" + + if type(value) is not str: + raise TypeError("rich Markdown token counting requires exact text") + return sum(1 for _ in _TOKEN.finditer(value)) + + +def _failure( + code: CompilationFailureCode, + text: str, + offset: int, + construct: UnsupportedConstruct | None = None, +) -> CompilationFailure: + return CompilationFailure( + code=code, + position=_point(text, offset), + construct=construct, + ) + + +def _point(text: str, offset: int) -> SourcePoint: + prefix = text[:offset] + logical_prefix = prefix.replace("\r\n", "\n").replace("\r", "\n") + last_newline = logical_prefix.rfind("\n") + return SourcePoint( + line=logical_prefix.count("\n") + 1, + column=len(logical_prefix[last_newline + 1 :]) + 1, + byte_offset=len(prefix.encode("utf-8")), + ) + + +def _span(text: str, start: int, end: int) -> SourceSpan: + return SourceSpan(start=_point(text, start), end=_point(text, end)) + + +def _normalize(source: bytes) -> str | CompilationFailure: + try: + decoded = source.decode("utf-8", errors="strict") + except UnicodeDecodeError as error: + safe_prefix = source[: error.start].decode("utf-8", errors="strict") + return _failure( + CompilationFailureCode.INVALID_UTF8, + safe_prefix, + len(safe_prefix), + ) + if any(is_markdown_control_character(character) for character in decoded): + offset = next( + index + for index, character in enumerate(decoded) + if is_markdown_control_character(character) + ) + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + decoded, + offset, + UnsupportedConstruct.CONTROL_CHARACTER, + ) + return decoded + + +def _line_layout(text: str) -> tuple[list[str], list[int]]: + raw_lines = text.splitlines(keepends=True) + if not raw_lines: + raw_lines = [""] + lines: list[str] = [] + starts: list[int] = [] + offset = 0 + for index, raw_line in enumerate(raw_lines): + content = raw_line.removesuffix("\n").removesuffix("\r") + bom_width = 1 if index == 0 and content.startswith("\ufeff") else 0 + lines.append(content[bom_width:]) + starts.append(offset + bom_width) + offset += len(raw_line) + return lines, starts + + +def _line_end(lines: list[str], starts: list[int], index: int) -> int: + return starts[index] + len(lines[index]) + + +def _table_cells(extractor: _ElementExtractor, line: str) -> tuple[str, ...]: + return tuple(extractor._table_cells(line)) + + +def _is_table_source_line(line: str) -> bool: + """Return whether a nonblank line remains part of an opened pipe table.""" + + return "|" in line + + +def _frontmatter_end(lines: list[str]) -> int | None: + if not lines or lines[0] != "---": + return None + for index in range(1, len(lines)): + if lines[index] == "---": + first_payload_line = next( + (line for line in lines[1:index] if line.strip()), + None, + ) + return ( + index + if first_payload_line is not None + and ( + _FRONTMATTER_MAPPING.fullmatch(first_payload_line) is not None + or _FRONTMATTER_SEQUENCE.fullmatch(first_payload_line) is not None + ) + else None + ) + return None + + +def _blocks(text: str) -> tuple[_Block, ...] | CompilationFailure: + lines, starts = _line_layout(text) + extractor = cast( + _ElementExtractor, + cast(Any, MarkdownElementExtractor)(text.removesuffix("\n")), + ) + blocks: list[_Block] = [] + index = 0 + frontmatter_end = _frontmatter_end(lines) + if frontmatter_end is not None: + blocks.append( + _Block( + kind=SectionKind.PARAGRAPH, + start=starts[0], + end=_line_end(lines, starts, frontmatter_end), + ) + ) + index = frontmatter_end + 1 + + while index < len(lines): + line = lines[index] + if not line.strip(): + index += 1 + continue + + atx = _ATX_HEADING.fullmatch(line) + if atx is not None: + construct = unsupported_rich_markdown_inline(atx.group(2).strip()) + if construct is not None: + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + text, + starts[index], + construct, + ) + blocks.append( + _Block( + kind=SectionKind.HEADING, + start=starts[index], + end=_line_end(lines, starts, index), + level=len(atx.group(1)), + ) + ) + index += 1 + continue + fence = extractor._get_fence_marker(line) + if fence is not None: + fence_character, fence_length = fence + language = line.lstrip()[fence_length:].strip() or None + closing = index + 1 + while closing < len(lines) and not extractor._is_closing_fence( + lines[closing], fence_character, fence_length + ): + closing += 1 + if closing >= len(lines): + if language is None and any( + candidate.strip() + for candidate in lines[index + 1 :] + ): + blocks.append( + _Block( + kind=SectionKind.PARAGRAPH, + start=starts[index], + end=_line_end(lines, starts, len(lines) - 1), + ) + ) + index = len(lines) + continue + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + text, + starts[index], + UnsupportedConstruct.CODE_BLOCK, + ) + if language is not None and ( + len(language) > MARKDOWN_CODE_LANGUAGE_MAX_LENGTH + or any(character.isspace() for character in language) + ): + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + text, + starts[index], + UnsupportedConstruct.CODE_BLOCK, + ) + body = "\n".join(lines[index + 1 : closing]) + if not body or body.isspace(): + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + text, + starts[index], + UnsupportedConstruct.CODE_BLOCK, + ) + blocks.append( + _Block( + kind=SectionKind.FENCED_CODE, + start=starts[index], + end=_line_end(lines, starts, closing), + code_language=language, + code_body=body, + ) + ) + index = closing + 1 + continue + + if re.fullmatch( + r" {0,3}(?:(?:\*[ \t]*){3,}|(?:_[ \t]*){3,}|(?:-[ \t]*){3,})", + line, + ): + blocks.append( + _Block( + kind=SectionKind.PARAGRAPH, + start=starts[index], + end=_line_end(lines, starts, index), + ) + ) + index += 1 + continue + + if index + 1 < len(lines) and _SETEXT.fullmatch(lines[index + 1]): + construct = unsupported_rich_markdown_inline(line.strip()) + if construct is not None: + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + text, + starts[index], + construct, + ) + blocks.append( + _Block( + kind=SectionKind.HEADING, + start=starts[index], + end=_line_end(lines, starts, index + 1), + level=1 if lines[index + 1].lstrip().startswith("=") else 2, + ) + ) + index += 2 + continue + + if ( + index + 1 < len(lines) + and extractor._is_table_row(line) + and extractor._is_table_separator_row(lines[index + 1]) + ): + header = _table_cells(extractor, line) + separator = _table_cells(extractor, lines[index + 1]) + width = len(header) + rows: list[tuple[str, ...]] = [] + end = index + 2 + while end < len(lines) and _is_table_source_line(lines[end]): + row = _table_cells(extractor, lines[end]) + rows.append(row) + end += 1 + if ( + not rows + or width < 2 + or len(separator) != width + or any( + len(row) != width or any(not cell for cell in row) + for row in (header, *rows) + ) + ): + for table_index in range(index, end): + construct = unsupported_rich_markdown_inline( + lines[table_index] + ) + if construct is not None: + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + text, + starts[table_index], + construct, + ) + blocks.append( + _Block( + kind=SectionKind.PARAGRAPH, + start=starts[index], + end=_line_end(lines, starts, end - 1), + indivisible=True, + ) + ) + index = end + continue + blocks.append( + _Block( + kind=SectionKind.TABLE, + start=starts[index], + end=_line_end(lines, starts, end - 1), + table_header=header, + table_rows=tuple(rows), + ) + ) + index = end + continue + + list_match = _LIST_ITEM.fullmatch(line) + if list_match is not None: + items: list[str] = [] + end = index + ordered = re.match(r"[0-9]+[.)]", line.lstrip()) is not None + while end < len(lines): + candidate = lines[end] + match = _LIST_ITEM.fullmatch(candidate) + if match is not None: + item = match.group(2).rstrip(" \t") + construct = unsupported_rich_markdown_inline(item) + if construct not in {None, UnsupportedConstruct.LIST}: + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + text, + starts[end], + construct, + ) + items.append(item) + end += 1 + continue + if candidate.strip() and candidate.startswith((" ", "\t")): + construct = unsupported_rich_markdown_inline( + candidate.lstrip() + ) + if construct is not None: + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + text, + starts[end], + construct, + ) + end += 1 + continue + break + blocks.append( + _Block( + kind=SectionKind.LIST, + start=starts[index], + end=_line_end(lines, starts, end - 1), + list_ordered=ordered, + list_items=tuple(items), + ) + ) + index = end + continue + + if line.lstrip().startswith("<"): + if _ANGLE_LITERAL.fullmatch(line.strip()) is not None: + blocks.append( + _Block( + kind=SectionKind.PARAGRAPH, + start=starts[index], + end=_line_end(lines, starts, index), + ) + ) + index += 1 + continue + html_open = _HTML_OPEN.match(line) + if html_open is None: + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + text, + starts[index], + UnsupportedConstruct.HTML, + ) + tag = html_open.group("tag") + end = index + 1 + while end < len(lines) and lines[end].strip(): + end += 1 + html_source = text[starts[index] : _line_end(lines, starts, end - 1)] + closing_tag = re.search( + rf"", html_source, re.IGNORECASE + ) + if closing_tag is None: + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + text, + starts[index], + UnsupportedConstruct.HTML, + ) + blocks.append( + _Block( + kind=SectionKind.PARAGRAPH, + start=starts[index], + end=_line_end(lines, starts, end - 1), + ) + ) + index = end + continue + + if line.lstrip().startswith(">"): + end = index + 1 + while end < len(lines) and lines[end].lstrip().startswith(">"): + end += 1 + for quote_index in range(index, end): + quoted = lines[quote_index].lstrip()[1:].lstrip() + callout = _CALLOUT.fullmatch(lines[quote_index]) is not None + construct = ( + None + if callout + else unsupported_rich_markdown_inline(quoted) + ) + if construct is not None: + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + text, + starts[quote_index], + construct, + ) + blocks.append( + _Block( + kind=SectionKind.PARAGRAPH, + start=starts[index], + end=_line_end(lines, starts, end - 1), + ) + ) + index = end + continue + + if re.match( + r"^ {0,3}#{1,6}[ \t]*$", + line, + ): + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + text, + starts[index], + UnsupportedConstruct.NESTED_HEADING, + ) + + end = index + 1 + while end < len(lines) and lines[end].strip(): + if ( + _ATX_HEADING.fullmatch(lines[end]) + or extractor._get_fence_marker(lines[end]) is not None + or _LIST_ITEM.fullmatch(lines[end]) + or lines[end].lstrip().startswith((">", "<")) + or re.fullmatch( + r" {0,3}(?:(?:\*[ \t]*){3,}|(?:_[ \t]*){3,}|" + r"(?:-[ \t]*){3,})", + lines[end], + ) + or ( + end + 1 < len(lines) + and extractor._is_table_row(lines[end]) + and extractor._is_table_separator_row(lines[end + 1]) + ) + ): + break + end += 1 + for paragraph_index in range(index, end): + construct = unsupported_rich_markdown_inline( + lines[paragraph_index] + ) + if construct is not None: + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + text, + starts[paragraph_index], + construct, + ) + blocks.append( + _Block( + kind=SectionKind.PARAGRAPH, + start=starts[index], + end=_line_end(lines, starts, end - 1), + ) + ) + index = end + return tuple(blocks) + + +def _heading_text(source: str, level: int) -> str: + lines = source.splitlines() + if len(lines) == 2 and _SETEXT.fullmatch(lines[1]): + return lines[0].strip() + match = _ATX_HEADING.fullmatch(source) + if match is None: + raise ValueError("rich heading source must match its declared syntax") + assert len(match.group(1)) == level + return match.group(2).strip() + + +def _split_ranges(source: str, capacity: int) -> tuple[tuple[int, int], ...]: + tokens = tuple(_TOKEN.finditer(source)) + if not tokens or capacity < 1: + return () + return tuple( + (tokens[index].start(), tokens[min(index + capacity, len(tokens)) - 1].end()) + for index in range(0, len(tokens), capacity) + ) + + +def _section( + block: _Block, + source: str, + path: StructuralPath, + position: SourceSpan, +) -> ParsedSection: + return ParsedSection( + kind=block.kind, + text=( + _heading_text(source, block.level) + if block.kind is SectionKind.HEADING and block.level is not None + else source + ), + path=path, + position=position, + level=block.level, + list_ordered=block.list_ordered, + list_items=block.list_items, + code_language=block.code_language, + code_body=block.code_body, + table_header=block.table_header, + table_rows=block.table_rows, + ) + + +def compile_rich_markdown( + source: bytes, + config: MarkdownCompilerConfig, +) -> CompilationOutcome: + """Compile exact bytes through the explicit rich v3 representation.""" + + if type(source) is not bytes: + raise TypeError("rich Markdown compiler source must be exact bytes") + if type(config) is not MarkdownCompilerConfig: + raise TypeError("rich Markdown compiler config must be exact") + if config.version != "markdown-config-v3": + raise ValueError("rich Markdown compiler requires markdown-config-v3") + assert config.token_ceiling is not None + token_ceiling = config.token_ceiling + normalized = _normalize(source) + if isinstance(normalized, CompilationFailure): + return normalized + try: + blocks = _blocks(normalized) + except Exception: + return _failure( + CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE, + normalized, + 0, + ) + if isinstance(blocks, CompilationFailure): + return blocks + if not blocks: + return _failure( + CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE, + normalized, + 0, + ) + + headings: list[ParsedSection] = [] + counters: dict[tuple[tuple[str, ...], SectionKind], int] = {} + ordinals: dict[SectionKind, int] = {} + sections: list[ParsedSection] = [] + fragments: list[CompiledFragment] = [] + for block in blocks: + full_source = normalized[block.start : block.end] + if block.kind is SectionKind.HEADING: + assert block.level is not None + headings = [ + heading + for heading in headings + if heading.level is not None and heading.level < block.level + ] + parents = tuple(headings) + parent_path = parents[-1].path.segments if parents else ("document",) + ancestry = "\n\n".join( + f"{'#' * heading.level} {heading.text}" + for heading in parents + if heading.level is not None + ) + capacity = token_ceiling - rich_token_count(ancestry) + if block.kind is SectionKind.HEADING: + ranges: tuple[tuple[int, int], ...] = ((0, len(full_source)),) + if rich_token_count(full_source) > capacity: + return _failure( + CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE, + normalized, + block.start, + ) + elif block.indivisible or block.kind in { + SectionKind.LIST, + SectionKind.FENCED_CODE, + SectionKind.TABLE, + }: + ranges = ((0, len(full_source)),) + if rich_token_count(full_source) > capacity: + return _failure( + CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE, + normalized, + block.start, + ) + else: + ranges = _split_ranges(full_source, capacity) + if not ranges: + return _failure( + CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE, + normalized, + block.start, + ) + for relative_start, relative_end in ranges: + source_text = full_source[relative_start:relative_end] + key = (parent_path, block.kind) + counters[key] = counters.get(key, 0) + 1 + try: + path = StructuralPath( + parent_path + (f"{block.kind.value}[{counters[key]}]",) + ) + position = _span( + normalized, + block.start + relative_start, + block.start + relative_end, + ) + section = _section(block, source_text, path, position) + ordinals[block.kind] = ordinals.get(block.kind, 0) + 1 + contextual = ( + f"{ancestry}\n\n{source_text}" if ancestry else source_text + ) + phrases = tuple(dict.fromkeys((source_text, section.text))) + fragment = CompiledFragment( + fragment_ref=( + f"fragment:{block.kind.value}:{ordinals[block.kind]}" + ), + kind=block.kind, + path=path, + position=position, + source_text=source_text, + contextual_text=contextual, + parent_headings=parents, + search_phrases=phrases, + ) + except Exception: + return _failure( + CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE, + normalized, + block.start + relative_start, + ) + fragments.append(fragment) + sections.append(section) + if block.kind is SectionKind.HEADING: + headings.append(section) + try: + provenance = CompilationProvenance( + compiler_version=MARKDOWN_COMPILER_V3_VERSION, + config_version=config.version, + canonicalization_profile=MARKDOWN_RICH_CANONICALIZATION_PROFILE, + compilation_digest_profile=MARKDOWN_RICH_COMPILATION_DIGEST_PROFILE, + token_ceiling=token_ceiling, + ) + return ParsedDocument.rich_v3( + canonical_text=normalized, + sections=tuple(sections), + fragments=tuple(fragments), + provenance=provenance, + ) + except Exception: + return _failure( + CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE, + normalized, + 0, + ) diff --git a/applications/compiler_runner.py b/applications/compiler_runner.py new file mode 100644 index 00000000..7fa35652 --- /dev/null +++ b/applications/compiler_runner.py @@ -0,0 +1,372 @@ +"""Pure subprocess boundary for the registered rich Markdown compiler.""" + +from __future__ import annotations + +import argparse +import base64 +import hashlib +import json +import re +import subprocess +import sys +from pathlib import Path, PurePath +from typing import Final, Protocol, cast + +from adapters.parsers.ragflow_markdown import compile_rich_markdown, rich_token_count +from engine.supply import ( + MARKDOWN_RICH_TOKEN_CEILING, + CompilationFailure, + CompilationFailureCode, + CompilationOutcome, + MarkdownCompilerConfig, + ParsedDocument, + SourcePoint, + UnsupportedConstruct, + canonicalize_parsed_document, + deserialize_parsed_document, +) +from eval._compiler_acceptance import ( + _AcceptanceContext, + acceptance_context, + is_acceptance_context, +) + +_RUNNER_MODULE: Final = "applications.compiler_runner" +COMPILER_RUNNER_TIMEOUT_SECONDS: Final = 30.0 + + +class _AcceptanceEntryPoint(Protocol): + def __call__( + self, + source: bytes, + config: MarkdownCompilerConfig, + *, + acceptance_context: _AcceptanceContext, + ) -> CompilationOutcome: ... + + +def _boundary_failure() -> CompilationFailure: + return CompilationFailure( + code=CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE, + position=None, + ) + + +def _failure_document(failure: CompilationFailure) -> dict[str, object]: + return { + "code": failure.code.value, + "construct": failure.construct.value if failure.construct is not None else None, + "position": ( + { + "line": failure.position.line, + "column": failure.position.column, + "byteOffset": failure.position.byte_offset, + } + if failure.position is not None + else None + ), + } + + +def _failure_from_document(value: object) -> CompilationFailure: + if type(value) is not dict: + raise ValueError("runner failure must be an object") + document = cast(dict[str, object], value) + position_value = document["position"] + position = None + if type(position_value) is dict: + point = cast(dict[str, object], position_value) + position = SourcePoint( + line=cast(int, point["line"]), + column=cast(int, point["column"]), + byte_offset=cast(int, point["byteOffset"]), + ) + construct_value = document["construct"] + return CompilationFailure( + code=CompilationFailureCode(cast(str, document["code"])), + position=position, + construct=( + UnsupportedConstruct(cast(str, construct_value)) + if construct_value is not None + else None + ), + ) + + +def _require_acceptance_context( + entry_point: _AcceptanceEntryPoint, +) -> _AcceptanceEntryPoint: + def guarded( + source: bytes, + config: MarkdownCompilerConfig, + *, + acceptance_context: _AcceptanceContext | None = None, + ) -> CompilationOutcome: + if is_acceptance_context(acceptance_context): + return entry_point( + source, + config, + acceptance_context=cast(_AcceptanceContext, acceptance_context), + ) + return _boundary_failure() + + return cast(_AcceptanceEntryPoint, guarded) + + +@_require_acceptance_context +def compile_in_local_compiler_runner( + source: bytes, + config: MarkdownCompilerConfig, + *, + acceptance_context: _AcceptanceContext, +) -> CompilationOutcome: + """Compile in an unleased local process that production must never call.""" + + assert is_acceptance_context(acceptance_context) + if type(source) is not bytes: + raise TypeError("compiler-runner source must be exact bytes") + if type(config) is not MarkdownCompilerConfig: + raise TypeError("compiler-runner config must be exact") + if config.token_ceiling is None: + raise ValueError("compiler-runner requires rich Markdown config") + try: + completed = subprocess.run( + [ + sys.executable, + "-m", + _RUNNER_MODULE, + "--compile", + "--config", + config.version, + "--token-ceiling", + str(config.token_ceiling), + ], + input=source, + capture_output=True, + check=False, + timeout=COMPILER_RUNNER_TIMEOUT_SECONDS, + ) + except Exception: + return _boundary_failure() + if completed.returncode != 0: + return _boundary_failure() + try: + envelope = json.loads(completed.stdout) + except (UnicodeDecodeError, json.JSONDecodeError): + return _boundary_failure() + if type(envelope) is not dict: + return _boundary_failure() + document = cast(dict[str, object], envelope) + if document.get("outcome") == "parsed": + encoded = document.get("document") + if type(encoded) is not str: + return _boundary_failure() + try: + return deserialize_parsed_document( + base64.b64decode(encoded, validate=True) + ) + except Exception: + return _boundary_failure() + if document.get("outcome") == "failure": + try: + return _failure_from_document(document.get("failure")) + except Exception: + return _boundary_failure() + return _boundary_failure() + + +def _emit( + source: bytes, + config: MarkdownCompilerConfig, + *, + acceptance_context: _AcceptanceContext | None = None, +) -> None: + if not is_acceptance_context(acceptance_context): + outcome: CompilationOutcome = _boundary_failure() + else: + try: + outcome = compile_rich_markdown(source, config) + except Exception: + outcome = _boundary_failure() + if type(outcome) is ParsedDocument: + envelope: dict[str, object] = { + "outcome": "parsed", + "document": base64.b64encode( + canonicalize_parsed_document(outcome) + ).decode("ascii"), + } + else: + assert type(outcome) is CompilationFailure + envelope = {"outcome": "failure", "failure": _failure_document(outcome)} + sys.stdout.write(json.dumps(envelope, sort_keys=True, separators=(",", ":"))) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--compile", action="store_true") + parser.add_argument("--config", default="markdown-config-v3") + parser.add_argument( + "--token-ceiling", + type=int, + default=MARKDOWN_RICH_TOKEN_CEILING, + ) + parser.add_argument("--acceptance-report", action="store_true") + parser.add_argument("--root", type=Path) + parser.add_argument( + "--output", + type=Path, + default=Path(".context-engine/compiler-runner-acceptance.json"), + ) + return parser + + +_CONSTRUCT_PATTERNS: Final[dict[str, re.Pattern[str]]] = { + "atxHeadings": re.compile(r"(?m)^ {0,3}#{1,6}[ \t]+"), + "setextHeadings": re.compile(r"(?m)^.+\n {0,3}(?:=+|-+)[ \t]*$"), + "lists": re.compile(r"(?m)^[ \t]*(?:[-+*]|[0-9]+[.)])[ \t]+"), + "fencedCode": re.compile(r"(?m)^ {0,3}(?:`{3,}|~{3,})"), + "tables": re.compile(r"(?m)^.*\|.*\n[ \t]*\|?[ :|-]+\|"), + "wikilinks": re.compile(r"(?[ \t]*\[![A-Za-z0-9_-]+]"), + "inlineMath": re.compile(r"(? tuple[Path, ...]: + if not root.is_dir(): + raise ValueError("acceptance root must be a directory") + return tuple( + sorted( + (path for path in root.rglob("*.md") if path.is_file()), + key=lambda path: PurePath(*path.relative_to(root).parts).as_posix(), + ) + ) + + +def _acceptance_report( + root: Path, + token_ceiling: int, + *, + acceptance_context: _AcceptanceContext, +) -> dict[str, object]: + if not is_acceptance_context(acceptance_context): + raise ValueError("acceptance report requires its private context") + accepted = 0 + refused = 0 + digests: list[str] = [] + maximum = 0 + histogram = {name: 0 for name in _CONSTRUCT_PATTERNS} + refusal_histogram: dict[str, int] = {} + for path in _safe_markdown_files(root): + try: + source = path.read_bytes() + except OSError: + refused += 1 + category = CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE.value + refusal_histogram[category] = refusal_histogram.get(category, 0) + 1 + continue + try: + inspected = source.removeprefix(b"\xef\xbb\xbf").decode("utf-8") + except UnicodeDecodeError: + inspected = "" + normalized = inspected.replace("\r\n", "\n").replace("\r", "\n") + for name, pattern in _CONSTRUCT_PATTERNS.items(): + histogram[name] += len(pattern.findall(normalized)) + outcome = compile_rich_markdown( + source, + MarkdownCompilerConfig( + "markdown-config-v3", + token_ceiling=token_ceiling, + ), + ) + if type(outcome) is ParsedDocument: + accepted += 1 + digests.append(outcome.compilation_digest) + maximum = max( + maximum, + *( + rich_token_count(fragment.contextual_text) + for fragment in outcome.fragments + ), + ) + else: + refused += 1 + assert type(outcome) is CompilationFailure + category = outcome.code.value + if outcome.construct is not None: + category = f"{category}:{outcome.construct.value}" + refusal_histogram[category] = refusal_histogram.get(category, 0) + 1 + aggregate = hashlib.sha256() + for digest in sorted(digests): + aggregate.update(bytes.fromhex(digest)) + total = accepted + refused + acceptance_rate = f"{accepted / total:.6f}" if total else "0.000000" + return { + "schemaVersion": "compiler-runner-acceptance-v1", + "compilerVersion": "context-engine-markdown-v3", + "configVersion": "markdown-config-v3", + "documents": { + "accepted": accepted, + "acceptanceRate": acceptance_rate, + "refused": refused, + "total": total, + }, + "constructHistogram": histogram, + "refusalHistogram": refusal_histogram, + "tokenCeiling": token_ceiling, + "maxFragmentTokenCount": maximum, + "aggregateCompilationDigest": aggregate.hexdigest(), + } + + +def _write_acceptance_report( + root: Path, + output: Path, + token_ceiling: int, + *, + acceptance_context: _AcceptanceContext, +) -> None: + if ".context-engine" not in output.parts: + raise ValueError("acceptance reports must be written under .context-engine") + report = _acceptance_report( + root, + token_ceiling, + acceptance_context=acceptance_context, + ) + serialized = json.dumps(report, sort_keys=True, separators=(",", ":")) + "\n" + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(serialized, encoding="utf-8") + sys.stdout.write(serialized) + + +def main() -> None: + args = _parser().parse_args() + if args.compile: + _emit( + sys.stdin.buffer.read(), + MarkdownCompilerConfig( + args.config, + token_ceiling=cast(int, args.token_ceiling), + ), + acceptance_context=acceptance_context(), + ) + return + if args.acceptance_report: + if args.root is None: + raise SystemExit("--acceptance-report requires --root") + _write_acceptance_report( + cast(Path, args.root), + cast(Path, args.output), + cast(int, args.token_ceiling), + acceptance_context=acceptance_context(), + ) + return + raise SystemExit("one runner operation is required") + + +if __name__ == "__main__": + main() diff --git a/docs/contracts/compiler-runner-acceptance-v1.schema.json b/docs/contracts/compiler-runner-acceptance-v1.schema.json new file mode 100644 index 00000000..7af71130 --- /dev/null +++ b/docs/contracts/compiler-runner-acceptance-v1.schema.json @@ -0,0 +1,83 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://context-engine.invalid/schemas/compiler-runner-acceptance-v1.json", + "title": "CompilerRunnerAcceptanceReportV1", + "type": "object", + "additionalProperties": false, + "required": [ + "aggregateCompilationDigest", + "compilerVersion", + "configVersion", + "constructHistogram", + "documents", + "maxFragmentTokenCount", + "refusalHistogram", + "schemaVersion", + "tokenCeiling" + ], + "properties": { + "aggregateCompilationDigest": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "compilerVersion": {"const": "context-engine-markdown-v3"}, + "configVersion": {"const": "markdown-config-v3"}, + "constructHistogram": { + "type": "object", + "additionalProperties": false, + "required": [ + "atxHeadings", + "callouts", + "embeds", + "fencedCode", + "footnotes", + "frontmatter", + "htmlBlocks", + "inlineMath", + "lists", + "setextHeadings", + "tables", + "wikilinks" + ], + "properties": { + "atxHeadings": {"type": "integer", "minimum": 0}, + "callouts": {"type": "integer", "minimum": 0}, + "embeds": {"type": "integer", "minimum": 0}, + "fencedCode": {"type": "integer", "minimum": 0}, + "footnotes": {"type": "integer", "minimum": 0}, + "frontmatter": {"type": "integer", "minimum": 0}, + "htmlBlocks": {"type": "integer", "minimum": 0}, + "inlineMath": {"type": "integer", "minimum": 0}, + "lists": {"type": "integer", "minimum": 0}, + "setextHeadings": {"type": "integer", "minimum": 0}, + "tables": {"type": "integer", "minimum": 0}, + "wikilinks": {"type": "integer", "minimum": 0} + } + }, + "documents": { + "type": "object", + "additionalProperties": false, + "required": ["accepted", "acceptanceRate", "refused", "total"], + "properties": { + "accepted": {"type": "integer", "minimum": 0}, + "acceptanceRate": { + "type": "string", + "pattern": "^(?:0|1)\\.[0-9]{6}$" + }, + "refused": {"type": "integer", "minimum": 0}, + "total": {"type": "integer", "minimum": 0} + } + }, + "refusalHistogram": { + "type": "object", + "propertyNames": { + "type": "string", + "pattern": "^(?:invalid_utf8|unsupported_document_shape|unsupported_construct:(?:atx_closing_sequence|blockquote|code_block|control_character|emphasis|entity|escape|frontmatter_or_rule|hard_break|html|inline_code|link_or_image|list|nested_heading|strikethrough|table))$" + }, + "additionalProperties": {"type": "integer", "minimum": 1} + }, + "maxFragmentTokenCount": {"type": "integer", "minimum": 0}, + "schemaVersion": {"const": "compiler-runner-acceptance-v1"}, + "tokenCeiling": {"type": "integer", "minimum": 1} + } +} diff --git a/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md b/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md new file mode 100644 index 00000000..fb0267eb --- /dev/null +++ b/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md @@ -0,0 +1,182 @@ +--- +name: adr-0079-compile-rich-markdown-in-an-owned-runner +version: "1.0.0" +description: > + Add an explicit rich-Markdown v3 representation behind a pure owned + compiler-runner while keeping frozen v1/v2 publication inactive. +--- + +# 0079. Compile rich Markdown in an owned runner + +- Status: accepted +- Date: 2026-07-29 +- Refines: ADR-0038, ADR-0074, ADR-0075 + +## Context + +The frozen Markdown v1 and v2 representations deliberately reject syntax used +by the measured File corpus. ADR-0038 requires a new decision before accepting +another construct or adding size-based splitting. It also requires any +replacement to retain version-explicit compilation, exact source provenance, +all-or-nothing failure, atomic immutable publication, budget-visible context, +and the sealed Runtime authorization order. + +V3 accepts both nonempty `---`-delimited frontmatter and thematic breaks, so it +must define how an initial `---` without a valid closing delimiter is +classified. Treating every leading `---` as frontmatter would make the accepted +thematic-break grammar depend on document position. Independent construction +testing also showed that v3 profile labels alone do not bind the exact +compiler/configuration identity, and a runner subprocess without a deadline +could fail to produce either an accepted document or a typed refusal. + +ADR-0074 permits a pinned, registered copy of RAGFlow's Apache-2.0 Markdown +parser region after path-level license and nested-dependency verification. +ADR-0075 permits Supply work in a ContextEngine-owned runner subprocess only +when the runner is a pure transform: it owns no persistence, cache, corpus, or +index and executes under the exact parent WorkerLease binding. + +## Decision + +1. **Explicit v3 representation.** `markdown-config-v3` selects + `context-engine-markdown-v3` with its own canonicalization and compilation + digest profiles. V1 and v2 remain frozen and byte-reproducible; v3 never + silently reinterprets either version. +2. **Closed rich grammar.** V3 accepts UTF-8 Markdown containing nonempty, + `---`-delimited YAML frontmatter, ATX and setext headings, nested ordered or + unordered lists, + backtick or tilde fenced code blocks whose content may contain shorter fence + runs, pipe tables, wikilinks, embeds, footnotes, HTML blocks, Obsidian + callouts, ordinary blockquotes, inline math, emphasis/strong text, inline + code, inline or reference links/images and reference definitions, + strikethrough, angle-bracket literals, hard line breaks, thematic breaks, + and ordinary paragraphs. Pipe tables retain + ragged or empty rows as one exact atomic source block rather than silently + truncating the document; typed cell metadata remains best-effort within + that exact source block. + A leading `---` opens frontmatter only when the first later `---` closes a + nonempty YAML-shaped mapping or sequence payload. The complete delimiter + matrix is closed as follows: a bare + `---` is one thematic break; adjacent delimiters are two thematic breaks; + delimiters separated only by blank lines remain thematic breaks; a nonempty + YAML-shaped mapping or sequence payload with a closing delimiter is + frontmatter; the same payload without a closing delimiter is ordinary + content after the leading thematic break; ordinary prose between delimiters + is a leading thematic break followed by ordinary Markdown (so a closing + `---` may serve as its setext underline); and adjacent delimiters followed + by text remain two thematic breaks followed by ordinary content. CRLF and LF + forms have identical grammar, and an optional leading BOM changes none of + these classifications. The mapping/sequence check is lexical rather than + semantic YAML interpretation. This delimiter-complete rule preserves both + constructs and does not infer metadata from empty payloads or ordinary + following prose. + A bare unmatched fence-marker line plus following nonblank text is retained + as one exact literal paragraph block; an empty or language-bearing + unterminated fence remains a typed refusal. + These additional bounded constructs are explicit because the real-corpus + acceptance gate showed they are required to close the measured v1 gap; raw + syntax remains exact source text and is not re-rendered. CRLF, lone CR, and + LF are treated uniformly for grammar recognition and line/column + calculation, but exact decoded source bytes are retained so every byte span + round-trips against the original UTF-8 input and representation digests + distinguish distinct inputs. Frontmatter payload bytes are retained but not + interpreted or used as authority; semantic YAML validation, CommonMark + compatibility, and unrestricted HTML compatibility are not claimed. An + optional leading UTF-8 BOM is retained for representation identity and + treated only as a transport marker preceding the first Fragment. +3. **Existing output contracts.** The runner emits and deserializes only the + existing `ParsedDocument`, `CompiledFragment`, `SourceSpan`, + `StructuralPath`, `CompilationProvenance`, or typed `CompilationFailure` + contracts. Rich constructs are projected into those existing section kinds; + no parallel document or Fragment model is introduced. +4. **Exact provenance.** Every Fragment carries an end-exclusive UTF-8 byte, + line, and column span. Its `source_text` is exactly the original input byte + slice at that span. Non-whitespace source bytes cannot be omitted. Table + Fragments obey the same round-trip rule. Heading ancestry is derived during + compilation and copied into the same budget-visible Fragment. The v3 domain + constructor independently re-derives the closed source grammar, typed + section metadata, coordinates, paths, parent headings, stable Fragment + references, search phrases, contextual text, and the provenance-bound + ceiling; parser-supplied metadata cannot bypass those checks. + Rich provenance binds the exact `context-engine-markdown-v3` compiler and + `markdown-config-v3` configuration identifiers as well as the v3 profiles. + The self-validating domain constructor rejects older or arbitrary identities. + The constructor and parser share only the closed control-character + classifier: C0 controls other than tab and line endings, DEL, and C1 controls + are refused. The constructor invokes it independently over exact source, so + fenced-code metadata cannot bypass a parser-ingress check. +5. **Hard splitting gate.** V3 uses a representation-bound 2,048-token default + ceiling, recorded in both configuration and compilation provenance. The + deterministic counter treats each non-whitespace run as one token. A block + whose contextual text would exceed the ceiling is split at source token + boundaries into ordered Fragments; every part retains the same preceding + heading ancestry, exact source span, and structural lineage. If heading + ancestry alone leaves no capacity, or an indivisible construct cannot be + represented within the ceiling, compilation refuses all-or-nothing. +6. **Owned pure process.** This issue delivers an unleased local/acceptance + process only. It receives exact source bytes and configuration, executes the + registered RAGFlow element-recognition helpers plus the ContextEngine-owned + grammar, hierarchy, raw-span, and bounds kernel, and emits deterministic + bytes. The copied upstream file remains deliberately unmodified so its hash + stays independently auditable; the adapter calls its fence and table + recognition methods directly. ContextEngine rewrites rich construct + classification, exact raw-byte position mapping, ancestry, typed output, + and splitting because the upstream return shape cannot express those + contracts. The unleased entry point requires an exact process-local + acceptance context from a module-private capability that is not exported, + configurable, or environment-derived; omitting or forging that context is a + typed refusal. A sound static gate additionally forbids every other + production module from directly importing the raw compiler, local runner, + or private capability. The process performs no network or database I/O and + retains no independent state, cache, index, or checkpoint. Both the direct + compiler seam and subprocess envelope convert unexpected parser or + domain-constructor rejection into a typed all-or-nothing + `CompilationFailure`; no partial document or raw exception crosses the + runner boundary. +7. **Activation remains deferred.** This decision proves the v3 pure transform + and local acceptance reporting only. The active File import configuration, + database publication functions, immutable Revision schemas, embeddings, + and existing v1 Revision migration remain unchanged. Activating v3 for + production publication requires a separate decision and complete atomic + publication and re-embedding evidence. The Supply execution bridge in issue + #125 owns the future production invocation and must bind it to the exact + parent WorkerLease. Production activation cannot reuse or manufacture this + issue's acceptance context. +8. **Runtime remains sealed.** Compilation changes no Runtime composition. + After any future v3 activation, each published Fragment is still only a + candidate until it crosses the exact + `CandidateRef -> AuthorizationKernel -> AuthorizedProjection` path. + +## Consequences + +- Delimiter completeness makes the two already accepted `---` meanings + deterministic without changing v1 or v2. A document beginning with an + unclosed `---` compiles that line as a thematic break; complete nonempty + frontmatter retains its exact raw span. +- Exact identity binding keeps stored provenance interpretable. Forged v3 + documents cannot substitute an older or arbitrary compiler/config identity + while retaining rich profiles. Sharing the finite character predicate + prevents drift without trusting any parser-provided structure, span, or + construct metadata. +- The local compiler-runner call has a fixed positive timeout. Child timeout or + launch failure produces the same content-free typed boundary failure as any + other subprocess-boundary failure; an unbounded wait is not an outcome. A + positive deadline completes the all-or-nothing process contract even when a + child wedges before emitting bytes. +- Rich-note and hard-bound behavior can be verified without creating durable + content or adding another persistence truth. +- Deterministic source spans and heading context remain part of the same + Fragment and therefore visible to existing provenance and Package budgets. +- The first controlled RAGFlow reuse pays its own pinned registration and + dependency-audit cost; artifact-wide aggregation remains owned by its + separately accepted work. +- Production File imports continue to use their current frozen representation + until activation and migration are decided explicitly. + +## Revisit trigger + +Revisit before accepting a construct outside the closed v3 grammar, changing +the token counter or ceiling, splitting an indivisible construct differently, +activating v3 publication, migrating an existing Revision, adding runner-local +state, changing the same-Fragment ancestry rule, changing leading delimiter +disambiguation or v3 identity binding, or making the fixed subprocess timeout a +caller-controlled runtime parameter. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 4da67764..4a1fb07c 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -42,6 +42,7 @@ kernel, capability separation, and publication visibility model. | First File source registration | [0035 — Trusted File source registration](0035-register-file-sources-through-context-control.md) | One operation-bound trusted Control call atomically creates an Organization-owned source plus immutable active first version; all acquisition carriers remain unavailable | Caller-authored Organization/mode, host paths, registration-time File I/O, future capability claims, or cross-tenant idempotency | | First Markdown compiler | [0036 — Deterministic narrow Markdown compilation](0036-compile-narrow-markdown-deterministically.md) | Exact bytes compile purely into one canonical heading-plus-paragraph ParsedDocument with normalized coordinates and versioned content/compilation identities | Path-coupled parsing, silent unsupported syntax, partial documents, unversioned derived identities, or parser-side I/O | | Structural Markdown compiler | [0038 — Structural Markdown units](0038-compile-and-publish-structural-markdown.md) | Explicit v2 compilation publishes one coherent Fragment per heading, paragraph, list, fenced code block, or table with exact provenance and same-Fragment heading ancestry | Silent v1 reinterpretation, item/cell fragmentation, post-authorization parent expansion, or unbudgeted context | +| Rich Markdown compiler-runner | [0079 — Rich Markdown in an owned runner](0079-compile-rich-markdown-in-an-owned-runner.md) | Explicit v3 pure compilation accepts the delimiter-complete closed rich grammar with exact source spans, exact provenance identity, same-Fragment ancestry, a hard token ceiling, and bounded typed runner termination while production activation stays deferred | Silent v1/v2 reinterpretation, position-dependent thematic-break refusal, relabeled v3 provenance, runner-local state, unbounded waits or Fragments, or activating publication without a separate decision | | Unchanged File acquisition | [0039 — File acquisition no-op](0039-deduplicate-unchanged-file-acquisitions.md) | Tenant/source/resource-scoped canonical identity and one PostgreSQL guard lock classify a complete active artifact before publication; each observation retains an immutable digest-only outcome | Cross-tenant/global deduplication, process-local locking, partial-artifact reuse, silent version reuse, or no-op by bypassing publication validation | | File Resource deletion | [0042 — Tombstone before cleanup](0042-tombstone-file-resources-before-cleanup.md) | One trusted Control transaction tombstones the active File Resource, advances its Organization Policy Epoch, and records immutable pending cleanup lineage before any physical deletion | Cleanup-defined visibility, caller-authored tenant/epoch/cleanup identity, index deletion as authorization, restore, or native watcher claims | | File Source progress | [0043 — Separate acquisition and publication progress](0043-separate-file-acquisition-progress-from-publication-progress.md), [0072 — Report File source status with closed refusals](0072-report-file-source-status-with-closed-refusals.md) | Append accepted changes separately from contiguous Runtime-visibility completion and expose content-free operational status through an Organization/Source-scoped Control read | One ambiguous checkpoint, skipped publication gaps, retained source/compiler diagnostics, Runtime authorization from status or watermarks, or false standard ProviderPort capability claims | @@ -182,5 +183,6 @@ touched: - [0076 — Rejoin rank evidence after authorization](0076-rejoin-rank-evidence-after-authorization.md) - [0077 — Fix the Article as the content authorization atom](0077-fix-the-article-as-the-content-authorization-atom.md) - [0078 — Narrow the contract-kit gate to per-connector twins](0078-narrow-the-contract-kit-gate-to-per-connector-twins.md) +- [0079 — Compile rich Markdown in an owned runner](0079-compile-rich-markdown-in-an-owned-runner.md) - [0080 — Refuse authoritative evaluation without an executor](0080-refuse-authoritative-evaluation-without-an-executor.md) - [0081 — Seal candidate discovery behind a data-only session](0081-seal-candidate-discovery-behind-a-data-only-session.md) diff --git a/engine/supply/__init__.py b/engine/supply/__init__.py index b46af33e..b8f234ab 100644 --- a/engine/supply/__init__.py +++ b/engine/supply/__init__.py @@ -57,8 +57,12 @@ MARKDOWN_COMPILATION_DIGEST_PROFILE, MARKDOWN_COMPILATION_DIGEST_V1_PROFILE, MARKDOWN_COMPILER_V1_VERSION, + MARKDOWN_COMPILER_V3_VERSION, MARKDOWN_COMPILER_VERSION, MARKDOWN_CONTENT_HASH_PROFILE, + MARKDOWN_RICH_CANONICALIZATION_PROFILE, + MARKDOWN_RICH_COMPILATION_DIGEST_PROFILE, + MARKDOWN_RICH_TOKEN_CEILING, CompilationFailure, CompilationFailureCode, CompilationOutcome, @@ -75,6 +79,7 @@ StructuralPath, UnsupportedConstruct, canonicalize_parsed_document, + deserialize_parsed_document, ) __all__ = [ @@ -86,8 +91,12 @@ "MARKDOWN_COMPILATION_DIGEST_PROFILE", "MARKDOWN_COMPILATION_DIGEST_V1_PROFILE", "MARKDOWN_COMPILER_VERSION", + "MARKDOWN_COMPILER_V3_VERSION", "MARKDOWN_COMPILER_V1_VERSION", "MARKDOWN_CONTENT_HASH_PROFILE", + "MARKDOWN_RICH_CANONICALIZATION_PROFILE", + "MARKDOWN_RICH_COMPILATION_DIGEST_PROFILE", + "MARKDOWN_RICH_TOKEN_CEILING", "WORKER_LEASE_ACTOR_KIND", "WORKER_LEASE_OPERATION", "SUPPLY_CONNECTOR_WORKER_LEASE_OPERATION", @@ -141,6 +150,7 @@ "WorkerLeaseToken", "generate_worker_lease_nonce", "canonicalize_parsed_document", + "deserialize_parsed_document", "worker_lease_digest", "worker_lease_nonce_digest", "validate_embedding_batch", diff --git a/engine/supply/markdown.py b/engine/supply/markdown.py index 588b3a62..a1573534 100644 --- a/engine/supply/markdown.py +++ b/engine/supply/markdown.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import re from dataclasses import dataclass from enum import StrEnum @@ -12,18 +13,34 @@ MARKDOWN_COMPILER_V1_VERSION: Final = "context-engine-markdown-v1" MARKDOWN_COMPILER_VERSION: Final = "context-engine-markdown-v2" +MARKDOWN_COMPILER_V3_VERSION: Final = "context-engine-markdown-v3" ACTIVE_FILE_IMPORT_MARKDOWN_CONFIG_VERSION: Final = "markdown-config-v1" MARKDOWN_CANONICALIZATION_V1_PROFILE: Final = "markdown-heading-paragraph-v1" MARKDOWN_CANONICALIZATION_PROFILE: Final = "markdown-structural-units-v2" MARKDOWN_CONTENT_HASH_PROFILE: Final = "sha256-canonical-utf8-v1" MARKDOWN_COMPILATION_DIGEST_V1_PROFILE: Final = "rfc8785-sha256-v1" MARKDOWN_COMPILATION_DIGEST_PROFILE: Final = "rfc8785-sha256-v2" +MARKDOWN_RICH_CANONICALIZATION_PROFILE: Final = "markdown-rich-structural-v3" +MARKDOWN_RICH_COMPILATION_DIGEST_PROFILE: Final = "rfc8785-sha256-v3" MARKDOWN_CODE_LANGUAGE_MAX_LENGTH: Final = 64 +MARKDOWN_RICH_TOKEN_CEILING: Final = 2048 _COMPILATION_DIGEST_V1_DOMAIN: Final = b"context-engine.markdown-compilation.v1\x00" _COMPILATION_DIGEST_DOMAIN: Final = b"context-engine.markdown-compilation.v2\x00" +_COMPILATION_DIGEST_V3_DOMAIN: Final = b"context-engine.markdown-compilation.v3\x00" _MAX_VERSION_LENGTH: Final = 128 +def is_markdown_control_character(character: str) -> bool: + """Return whether one character is outside the closed Markdown grammar.""" + + if type(character) is not str or len(character) != 1: + raise TypeError("Markdown control classification requires one character") + codepoint = ord(character) + return (codepoint < 0x20 and character not in "\t\n\r") or ( + 0x7F <= codepoint <= 0x9F + ) + + def _require_version(value: object) -> str: if ( type(value) is not str @@ -54,9 +71,17 @@ class MarkdownCompilerConfig: """Explicit representation-affecting compiler configuration identity.""" version: str + token_ceiling: int | None = None def __post_init__(self) -> None: _require_version(self.version) + if self.version == "markdown-config-v3": + if self.token_ceiling is None: + object.__setattr__(self, "token_ceiling", MARKDOWN_RICH_TOKEN_CEILING) + elif type(self.token_ceiling) is not int or self.token_ceiling < 1: + raise ValueError("rich Markdown token ceiling must be positive") + elif self.token_ceiling is not None: + raise ValueError("only rich Markdown config records a token ceiling") @dataclass(frozen=True, slots=True) @@ -78,7 +103,7 @@ def __post_init__(self) -> None: @dataclass(frozen=True, slots=True) class SourceSpan: - """End-exclusive source span over canonical normalized UTF-8 text.""" + """End-exclusive source span over the compiler's UTF-8 representation.""" start: SourcePoint end: SourcePoint @@ -147,12 +172,7 @@ class ParsedSection: def __post_init__(self) -> None: if type(self.kind) is not SectionKind: raise TypeError("parsed section kind must be SectionKind") - if ( - type(self.text) is not str - or not self.text - or self.text.isspace() - or self.text != self.text.strip() - ): + if type(self.text) is not str or not self.text or self.text.isspace(): raise ValueError("parsed section text must be exact nonblank text") if type(self.path) is not StructuralPath: raise TypeError("parsed section path must be StructuralPath") @@ -188,9 +208,8 @@ def __post_init__(self) -> None: if self.kind is SectionKind.TABLE: if not self.table_header or not self.table_rows: raise ValueError("table sections require a header and rows") - width = len(self.table_header) - if width < 1 or any(len(row) != width for row in self.table_rows): - raise ValueError("table rows must match the header width") + if len(self.table_header) < 1: + raise ValueError("table header must carry at least one cell") if any( type(cell) is not str or not cell or cell.isspace() for row in (self.table_header, *self.table_rows) @@ -279,6 +298,7 @@ class CompilationProvenance: canonicalization_profile: str = MARKDOWN_CANONICALIZATION_V1_PROFILE content_hash_profile: str = MARKDOWN_CONTENT_HASH_PROFILE compilation_digest_profile: str = MARKDOWN_COMPILATION_DIGEST_V1_PROFILE + token_ceiling: int | None = None def __post_init__(self) -> None: _require_version(self.compiler_version) @@ -293,15 +313,35 @@ def __post_init__(self) -> None: MARKDOWN_COMPILATION_DIGEST_V1_PROFILE, ), (MARKDOWN_CANONICALIZATION_PROFILE, MARKDOWN_COMPILATION_DIGEST_PROFILE), + ( + MARKDOWN_RICH_CANONICALIZATION_PROFILE, + MARKDOWN_RICH_COMPILATION_DIGEST_PROFILE, + ), }: raise ValueError("Markdown canonicalization and digest profiles must match") if self.content_hash_profile != MARKDOWN_CONTENT_HASH_PROFILE: raise ValueError("content hash profile must use the active version") + if self.is_rich_v3: + if ( + self.compiler_version != MARKDOWN_COMPILER_V3_VERSION + or self.config_version != "markdown-config-v3" + ): + raise ValueError( + "rich provenance identity requires the v3 compiler and config" + ) + if type(self.token_ceiling) is not int or self.token_ceiling < 1: + raise ValueError("rich provenance requires a positive token ceiling") + elif self.token_ceiling is not None: + raise ValueError("only rich provenance records a token ceiling") @property def is_structural_v2(self) -> bool: return self.canonicalization_profile == MARKDOWN_CANONICALIZATION_PROFILE + @property + def is_rich_v3(self) -> bool: + return self.canonicalization_profile == MARKDOWN_RICH_CANONICALIZATION_PROFILE + @dataclass(frozen=True, slots=True) class ParsedDocument: @@ -316,10 +356,8 @@ class ParsedDocument: warnings: tuple[CompilationWarning, ...] = () def __post_init__(self) -> None: - if type(self.canonical_text) is not str or not self.canonical_text.endswith( - "\n" - ): - raise ValueError("parsed document requires final-newline canonical text") + if type(self.canonical_text) is not str or not self.canonical_text: + raise ValueError("parsed document requires nonempty canonical text") if type(self.sections) is not tuple or any( type(section) is not ParsedSection for section in self.sections ): @@ -332,13 +370,23 @@ def __post_init__(self) -> None: _require_sha256("compilation digest", self.compilation_digest) if type(self.provenance) is not CompilationProvenance: raise TypeError("parsed document provenance must be CompilationProvenance") + if not self.provenance.is_rich_v3 and not self.canonical_text.endswith("\n"): + raise ValueError("parsed document requires final-newline canonical text") if type(self.warnings) is not tuple or any( type(warning) is not CompilationWarning for warning in self.warnings ): raise TypeError("parsed document warnings must be typed immutable values") if self.warnings: raise ValueError("the active Markdown compiler emits no warnings") - if self.provenance.is_structural_v2: + if self.provenance.is_rich_v3: + assert self.provenance.token_ceiling is not None + _validate_rich_content( + self.canonical_text, + self.sections, + self.fragments, + self.provenance.token_ceiling, + ) + elif self.provenance.is_structural_v2: _validate_structural_content( self.canonical_text, self.sections, @@ -426,6 +474,37 @@ def structural_v2( fragments=fragments, ) + @classmethod + def rich_v3( + cls, + *, + canonical_text: str, + sections: tuple[ParsedSection, ...], + fragments: tuple[CompiledFragment, ...], + provenance: CompilationProvenance, + ) -> ParsedDocument: + """Build one self-validating rich Markdown compilation result.""" + + if cls is not ParsedDocument or not provenance.is_rich_v3: + raise TypeError("rich ParsedDocument construction requires v3 provenance") + content_hash = sha256(canonical_text.encode("utf-8")).hexdigest() + compilation_digest = _compilation_digest( + canonical_text=canonical_text, + sections=sections, + fragments=fragments, + content_hash=content_hash, + provenance=provenance, + warnings=(), + ) + return ParsedDocument( + canonical_text=canonical_text, + sections=sections, + content_hash=content_hash, + compilation_digest=compilation_digest, + provenance=provenance, + fragments=fragments, + ) + class CompilationFailureCode(StrEnum): INVALID_UTF8 = "invalid_utf8" @@ -473,6 +552,39 @@ class UnsupportedConstruct(StrEnum): r"(?`{3,}|~{3,})(?:.*)$" +) +_RICH_WIKILINK_PATTERN: Final = re.compile(r"!?\[\[[^]\r\n]+]]") +_RICH_FOOTNOTE_PATTERN: Final = re.compile(r"\[\^[^]\r\n]+](?::)?") +_RICH_INLINE_MATH_PATTERN: Final = re.compile( + r"(?\s]+>", + re.IGNORECASE, +) +_RICH_INLINE_LINK_PATTERN: Final = re.compile( + r"!?\[[^]\r\n]*]\([^()\r\n]+\)" +) +_RICH_REFERENCE_LINK_PATTERN: Final = re.compile( + r"(?:!?\[[^]\r\n]*]\[[^]\r\n]*]|\[[^]\r\n]+]:[ \t]*\S+)" +) +_RICH_STRIKETHROUGH_PATTERN: Final = re.compile(r"~~(?=\S).+?(?<=\S)~~") +_RICH_ANGLE_LITERAL_PATTERN: Final = re.compile(r"<[^<>\r\n]+>") +_RICH_HTML_OPEN_PATTERN: Final = re.compile( + r"^ {0,3}<(?Psection|div|details|summary|table|thead|tbody|tr|" + r"th|td|p|ul|ol|li)\b[^>]*>", + re.IGNORECASE, +) def unsupported_markdown_construct( @@ -525,6 +637,31 @@ def unsupported_markdown_construct( return None +def unsupported_rich_markdown_inline(line: str) -> UnsupportedConstruct | None: + """Classify inline syntax outside the accepted rich-v3 grammar.""" + + if type(line) is not str: + raise TypeError("rich Markdown inline classification requires exact text") + masked = line.rstrip(" \t") + if masked.endswith("\\"): + masked = masked[:-1] + "x" + for pattern in ( + _RICH_WIKILINK_PATTERN, + _RICH_FOOTNOTE_PATTERN, + _RICH_INLINE_MATH_PATTERN, + _RICH_INLINE_CODE_PATTERN, + _EMPHASIS_PATTERN, + _RICH_AUTOLINK_PATTERN, + _RICH_INLINE_LINK_PATTERN, + _RICH_REFERENCE_LINK_PATTERN, + _RICH_STRIKETHROUGH_PATTERN, + _RICH_ANGLE_LITERAL_PATTERN, + ): + masked = pattern.sub(lambda match: "x" * len(match.group()), masked) + construct = unsupported_markdown_construct(masked, supported_heading=False) + return None if construct is UnsupportedConstruct.LIST else construct + + @dataclass(frozen=True, slots=True) class CompilationFailure: """Typed all-or-nothing failure; it never carries partial ParsedDocument data.""" @@ -649,7 +786,11 @@ def _compilation_document( for warning in warnings ], } - if provenance.is_structural_v2: + if provenance.is_rich_v3: + provenance_value = document["provenance"] + assert isinstance(provenance_value, dict) + provenance_value["tokenCeiling"] = provenance.token_ceiling + if provenance.is_structural_v2 or provenance.is_rich_v3: document["fragments"] = [ _fragment_document(fragment) for fragment in fragments ] @@ -684,11 +825,12 @@ def _compilation_digest( provenance=provenance, warnings=warnings, ) - domain = ( - _COMPILATION_DIGEST_DOMAIN - if provenance.is_structural_v2 - else _COMPILATION_DIGEST_V1_DOMAIN - ) + if provenance.is_rich_v3: + domain = _COMPILATION_DIGEST_V3_DOMAIN + elif provenance.is_structural_v2: + domain = _COMPILATION_DIGEST_DOMAIN + else: + domain = _COMPILATION_DIGEST_V1_DOMAIN return sha256(domain + rfc8785.dumps(cast(Any, document))).hexdigest() @@ -742,6 +884,520 @@ def _expected_search_phrases( return tuple(dict.fromkeys(values)) +def _expected_rich_search_phrases( + section: ParsedSection, + source_text: str, +) -> tuple[str, ...]: + return tuple(dict.fromkeys((source_text, section.text))) + + +def _rich_table_cells(line: str) -> tuple[str, ...]: + stripped = line.strip() + if stripped.startswith("|"): + stripped = stripped[1:] + if stripped.endswith("|"): + stripped = stripped[:-1] + return tuple(cell.strip() for cell in stripped.split("|")) + + +def _rich_table_source_ranges(source: str) -> tuple[tuple[int, int], ...]: + raw_lines = source.splitlines(keepends=True) or (source,) + lines: list[str] = [] + starts: list[int] = [] + offset = 0 + for index, raw_line in enumerate(raw_lines): + content = raw_line.removesuffix("\n").removesuffix("\r") + bom_width = 1 if index == 0 and content.startswith("\ufeff") else 0 + lines.append(content[bom_width:]) + starts.append(offset + bom_width) + offset += len(raw_line) + ranges: list[tuple[int, int]] = [] + index = 0 + while index + 1 < len(lines): + header = _rich_table_cells(lines[index]) + separator = _rich_table_cells(lines[index + 1]) + if ( + "|" not in lines[index] + or len(header) < 2 + or not any(header) + or len(separator) < 2 + or not all( + re.fullmatch(r":?-+:?", cell.replace(" ", "")) is not None + for cell in separator + ) + ): + index += 1 + continue + end = index + 2 + while end < len(lines) and "|" in lines[end]: + end += 1 + ranges.append( + ( + starts[index], + starts[end - 1] + len(lines[end - 1]), + ) + ) + index = end + return tuple(ranges) + + +@dataclass(frozen=True, slots=True) +class _RichSourceBlock: + kind: SectionKind + start: int + end: int + indivisible: bool = False + heading_level: int | None = None + heading_text: str | None = None + + +def _rich_source_line_layout(source: str) -> tuple[list[str], list[int]]: + raw_lines = source.splitlines(keepends=True) or [source] + lines: list[str] = [] + starts: list[int] = [] + offset = 0 + for index, raw_line in enumerate(raw_lines): + content = raw_line.removesuffix("\n").removesuffix("\r") + bom_width = 1 if index == 0 and content.startswith("\ufeff") else 0 + lines.append(content[bom_width:]) + starts.append(offset + bom_width) + offset += len(raw_line) + return lines, starts + + +def _rich_source_line_end( + lines: list[str], starts: list[int], index: int +) -> int: + return starts[index] + len(lines[index]) + + +def _rich_table_starts_at(lines: list[str], index: int) -> bool: + if index + 1 >= len(lines) or "|" not in lines[index]: + return False + header = _rich_table_cells(lines[index]) + separator = _rich_table_cells(lines[index + 1]) + return ( + len(header) >= 2 + and any(header) + and len(separator) >= 2 + and all( + re.fullmatch(r":?-+:?", cell.replace(" ", "")) is not None + for cell in separator + ) + ) + + +def _rich_source_blocks(source: str) -> tuple[_RichSourceBlock, ...]: + """Independently derive v3 block boundaries from the canonical source.""" + + lines, starts = _rich_source_line_layout(source) + blocks: list[_RichSourceBlock] = [] + index = 0 + if lines and lines[0] == "---": + closing = next( + ( + candidate + for candidate in range(1, len(lines)) + if lines[candidate] == "---" + ), + None, + ) + first_payload_line = next( + (line for line in lines[1:closing] if line.strip()), + None, + ) if closing is not None else None + mapping_payload = ( + first_payload_line is not None + and re.fullmatch( + r"^[ \t]*(?:[A-Za-z0-9_.-]+|\"[^\"\r\n]+\"|'[^'\r\n]+')" + r"[ \t]*:[ \t]*(?:.*)?$", + first_payload_line, + ) + is not None + ) + sequence_payload = ( + first_payload_line is not None + and re.fullmatch(r"^[ \t]*-[ \t]+\S.*$", first_payload_line) + is not None + ) + if closing is not None and (mapping_payload or sequence_payload): + blocks.append( + _RichSourceBlock( + kind=SectionKind.PARAGRAPH, + start=starts[0], + end=_rich_source_line_end(lines, starts, closing), + ) + ) + index = closing + 1 + + while index < len(lines): + line = lines[index] + if not line.strip(): + index += 1 + continue + atx = _RICH_ATX_HEADING_PATTERN.fullmatch(line) + if atx is not None: + blocks.append( + _RichSourceBlock( + kind=SectionKind.HEADING, + start=starts[index], + end=_rich_source_line_end(lines, starts, index), + heading_level=len(atx.group(1)), + heading_text=atx.group(2).strip(), + ) + ) + index += 1 + continue + fence = _RICH_FENCE_PATTERN.match(line) + if fence is not None: + marker = fence.group("fence") + closing = index + 1 + while closing < len(lines) and re.fullmatch( + rf"[ \t]{{0,3}}{re.escape(marker[0])}{{{len(marker)},}}[ \t]*", + lines[closing], + ) is None: + closing += 1 + if closing >= len(lines): + language = line.lstrip()[len(marker) :].strip() + if not language and any( + candidate.strip() for candidate in lines[index + 1 :] + ): + blocks.append( + _RichSourceBlock( + kind=SectionKind.PARAGRAPH, + start=starts[index], + end=_rich_source_line_end(lines, starts, len(lines) - 1), + ) + ) + index = len(lines) + continue + raise ValueError("rich fenced source must match the closed grammar") + blocks.append( + _RichSourceBlock( + kind=SectionKind.FENCED_CODE, + start=starts[index], + end=_rich_source_line_end(lines, starts, closing), + indivisible=True, + ) + ) + index = closing + 1 + continue + if _THEMATIC_BREAK_PATTERN.fullmatch(line) is not None: + blocks.append( + _RichSourceBlock( + kind=SectionKind.PARAGRAPH, + start=starts[index], + end=_rich_source_line_end(lines, starts, index), + ) + ) + index += 1 + continue + if index + 1 < len(lines) and _RICH_SETEXT_PATTERN.fullmatch( + lines[index + 1] + ): + underline = lines[index + 1].lstrip() + blocks.append( + _RichSourceBlock( + kind=SectionKind.HEADING, + start=starts[index], + end=_rich_source_line_end(lines, starts, index + 1), + heading_level=1 if underline.startswith("=") else 2, + heading_text=line.strip(), + ) + ) + index += 2 + continue + if _rich_table_starts_at(lines, index): + end = index + 2 + while end < len(lines) and "|" in lines[end]: + end += 1 + header = _rich_table_cells(lines[index]) + separator = _rich_table_cells(lines[index + 1]) + rows = tuple( + _rich_table_cells(candidate) + for candidate in lines[index + 2 : end] + ) + valid = ( + bool(rows) + and len(header) >= 2 + and len(separator) == len(header) + and all( + len(row) == len(header) and all(row) + for row in (header, *rows) + ) + ) + blocks.append( + _RichSourceBlock( + kind=SectionKind.TABLE if valid else SectionKind.PARAGRAPH, + start=starts[index], + end=_rich_source_line_end(lines, starts, end - 1), + indivisible=not valid, + ) + ) + index = end + continue + if _RICH_LIST_ITEM_PATTERN.fullmatch(line) is not None: + end = index + 1 + while end < len(lines): + candidate = lines[end] + if _RICH_LIST_ITEM_PATTERN.fullmatch(candidate) is not None: + end += 1 + continue + if candidate.strip() and candidate.startswith((" ", "\t")): + end += 1 + continue + break + blocks.append( + _RichSourceBlock( + kind=SectionKind.LIST, + start=starts[index], + end=_rich_source_line_end(lines, starts, end - 1), + indivisible=True, + ) + ) + index = end + continue + if line.lstrip().startswith("<"): + if _RICH_ANGLE_LITERAL_PATTERN.fullmatch(line.strip()) is not None: + end = index + 1 + else: + end = index + 1 + while end < len(lines) and lines[end].strip(): + end += 1 + blocks.append( + _RichSourceBlock( + kind=SectionKind.PARAGRAPH, + start=starts[index], + end=_rich_source_line_end(lines, starts, end - 1), + ) + ) + index = end + continue + if line.lstrip().startswith(">"): + end = index + 1 + while end < len(lines) and lines[end].lstrip().startswith(">"): + end += 1 + blocks.append( + _RichSourceBlock( + kind=SectionKind.PARAGRAPH, + start=starts[index], + end=_rich_source_line_end(lines, starts, end - 1), + ) + ) + index = end + continue + end = index + 1 + while end < len(lines) and lines[end].strip(): + if ( + _RICH_ATX_HEADING_PATTERN.fullmatch(lines[end]) is not None + or _RICH_FENCE_PATTERN.match(lines[end]) is not None + or _RICH_LIST_ITEM_PATTERN.fullmatch(lines[end]) is not None + or lines[end].lstrip().startswith((">", "<")) + or _THEMATIC_BREAK_PATTERN.fullmatch(lines[end]) is not None + or _rich_table_starts_at(lines, end) + ): + break + end += 1 + blocks.append( + _RichSourceBlock( + kind=SectionKind.PARAGRAPH, + start=starts[index], + end=_rich_source_line_end(lines, starts, end - 1), + ) + ) + index = end + return tuple(blocks) + + +def _expected_rich_fragment_layout( + canonical_text: str, + token_ceiling: int, +) -> tuple[tuple[SectionKind, int, int], ...]: + expected: list[tuple[SectionKind, int, int]] = [] + headings: list[tuple[int, str]] = [] + for block in _rich_source_blocks(canonical_text): + if block.kind is SectionKind.HEADING: + assert block.heading_level is not None + headings = [ + heading for heading in headings if heading[0] < block.heading_level + ] + source = canonical_text[block.start : block.end] + ancestry = "\n\n".join( + f"{'#' * level} {text}" for level, text in headings + ) + capacity = token_ceiling - len(re.findall(r"\S+", ancestry)) + indivisible = block.indivisible or block.kind in { + SectionKind.HEADING, + SectionKind.LIST, + SectionKind.FENCED_CODE, + SectionKind.TABLE, + } + ranges: tuple[tuple[int, int], ...] + if indivisible: + if len(re.findall(r"\S+", source)) > capacity: + raise ValueError("rich indivisible source exceeds its ceiling") + ranges = ((0, len(source)),) + else: + tokens = tuple(re.finditer(r"\S+", source)) + if not tokens or capacity < 1: + raise ValueError("rich paragraph cannot fit its ceiling") + ranges = tuple( + ( + tokens[index].start(), + tokens[min(index + capacity, len(tokens)) - 1].end(), + ) + for index in range(0, len(tokens), capacity) + ) + for relative_start, relative_end in ranges: + start = len(canonical_text[: block.start + relative_start].encode("utf-8")) + end = len(canonical_text[: block.start + relative_end].encode("utf-8")) + expected.append((block.kind, start, end)) + if block.kind is SectionKind.HEADING: + assert block.heading_level is not None and block.heading_text is not None + headings.append((block.heading_level, block.heading_text)) + return tuple(expected) + + +def _expected_rich_section_kind(source: str) -> SectionKind: + lines = source.splitlines() + if len(lines) >= 2 and lines[0] == "---" and lines[-1] == "---": + return SectionKind.PARAGRAPH + if len(lines) == 1 and _THEMATIC_BREAK_PATTERN.fullmatch(lines[0]) is not None: + return SectionKind.PARAGRAPH + if ( + _RICH_ATX_HEADING_PATTERN.fullmatch(source) is not None + or len(lines) == 2 + and _RICH_SETEXT_PATTERN.fullmatch(lines[1]) is not None + ): + return SectionKind.HEADING + fence = _RICH_FENCE_PATTERN.match(lines[0]) if lines else None + if fence is not None and len(lines) >= 3: + marker = fence.group("fence") + if re.fullmatch( + rf"[ \t]{{0,3}}{re.escape(marker[0])}{{{len(marker)},}}[ \t]*", + lines[-1], + ): + return SectionKind.FENCED_CODE + if lines and _RICH_LIST_ITEM_PATTERN.fullmatch(lines[0]) is not None: + return SectionKind.LIST + table_ranges = _rich_table_source_ranges(source) + if table_ranges == ((0, len(source)),): + header_cells = _rich_table_cells(lines[0]) + separator_cells = _rich_table_cells(lines[1]) + header_width = len(header_cells) + source_rows = tuple(_rich_table_cells(line) for line in lines[2:]) + if ( + source_rows + and header_width >= 2 + and len(separator_cells) == header_width + and all(header_cells) + and all(len(row) == header_width and all(row) for row in source_rows) + ): + return SectionKind.TABLE + return SectionKind.PARAGRAPH + + +def _validate_rich_closed_grammar(section: ParsedSection, source: str) -> None: + if section.kind is not _expected_rich_section_kind(source): + raise ValueError("rich section kind must match its source grammar") + lines = source.splitlines() + metadata_valid = True + if section.kind is SectionKind.LIST: + item_matches = tuple( + match + for line in lines + if (match := _RICH_LIST_ITEM_PATTERN.fullmatch(line)) is not None + ) + metadata_valid = ( + bool(item_matches) + and section.list_ordered + is (re.match(r"[0-9]+[.)]", lines[0].lstrip()) is not None) + and section.list_items + == tuple(match.group(2).rstrip(" \t") for match in item_matches) + ) + elif section.kind is SectionKind.FENCED_CODE: + fence = _RICH_FENCE_PATTERN.match(lines[0]) if lines else None + assert fence is not None + marker = fence.group("fence") + expected_language = lines[0].lstrip()[len(marker) :].strip() or None + expected_body = "\n".join(lines[1:-1]) + metadata_valid = ( + section.code_language == expected_language + and section.code_body == expected_body + ) + elif section.kind is SectionKind.TABLE: + cells = tuple(_rich_table_cells(line) for line in lines) + metadata_valid = ( + len(cells) >= 3 + and section.table_header == cells[0] + and section.table_rows == cells[2:] + ) + if not metadata_valid: + raise ValueError("rich section metadata must match its source grammar") + if section.kind is SectionKind.FENCED_CODE: + return + if len(lines) >= 2 and lines[0] == "---" and lines[-1] == "---": + return + html_open = _RICH_HTML_OPEN_PATTERN.match(lines[0]) if lines else None + if ( + html_open is not None + and re.search( + rf"", + source, + re.IGNORECASE, + ) + ): + return + if ( + section.kind is SectionKind.PARAGRAPH + and len(lines) == 1 + and _THEMATIC_BREAK_PATTERN.fullmatch(lines[0]) is not None + ): + return + if ( + section.kind is SectionKind.PARAGRAPH + and lines + and _RICH_FENCE_PATTERN.match(lines[0]) is not None + ): + marker = _RICH_FENCE_PATTERN.match(lines[0]) + assert marker is not None + fence_text = marker.group("fence") + language = lines[0].lstrip()[len(fence_text) :].strip() + if language or not any(line.strip() for line in lines[1:]): + raise ValueError("rich section source must match the closed grammar") + return + if ( + section.kind is SectionKind.PARAGRAPH + and _rich_table_source_ranges(source) == ((0, len(source)),) + ): + if any( + unsupported_rich_markdown_inline(line) is not None + for line in lines + ): + raise ValueError("rich section source must match the closed grammar") + return + for line in lines: + inspected = line + if section.kind is SectionKind.HEADING: + match = _RICH_ATX_HEADING_PATTERN.fullmatch(line) + if match is not None: + inspected = match.group(2).strip() + elif _RICH_SETEXT_PATTERN.fullmatch(line) is not None: + continue + elif section.kind is SectionKind.LIST: + match = _RICH_LIST_ITEM_PATTERN.fullmatch(line) + if match is not None: + inspected = match.group(2) + elif line.startswith((" ", "\t")): + inspected = line.lstrip() + elif line.lstrip().startswith(">"): + inspected = line.lstrip()[1:].lstrip() + if inspected.startswith("[!"): + inspected = _RICH_FOOTNOTE_PATTERN.sub("x", inspected, count=1) + if unsupported_rich_markdown_inline(inspected) is not None: + raise ValueError("rich section source must match the closed grammar") + + def _validate_section_source(section: ParsedSection, source_text: str) -> None: lines = source_text.split("\n") if section.kind is SectionKind.HEADING: @@ -928,6 +1584,163 @@ def _validate_structural_content( raise ValueError("structural sections cannot omit trailing canonical content") +def _validate_rich_content( + canonical_text: str, + sections: tuple[ParsedSection, ...], + fragments: tuple[CompiledFragment, ...], + token_ceiling: int, +) -> None: + if not sections or not fragments or len(sections) != len(fragments): + raise ValueError("rich Markdown requires one Fragment per parsed section") + if any(is_markdown_control_character(character) for character in canonical_text): + raise ValueError("rich Markdown cannot contain a control character") + canonical_bytes = canonical_text.encode("utf-8") + prior_end = 0 + refs: set[str] = set() + headings: list[ParsedSection] = [] + counters: dict[tuple[tuple[str, ...], SectionKind], int] = {} + kind_ordinals: dict[SectionKind, int] = {} + for table_start_character, table_end_character in _rich_table_source_ranges( + canonical_text + ): + table_start = len( + canonical_text[:table_start_character].encode("utf-8") + ) + table_end = len(canonical_text[:table_end_character].encode("utf-8")) + overlapping_sections = tuple( + section + for section in sections + if section.position.start.byte_offset < table_end + and section.position.end.byte_offset > table_start + ) + if ( + len(overlapping_sections) != 1 + or overlapping_sections[0].position.start.byte_offset != table_start + or overlapping_sections[0].position.end.byte_offset != table_end + ): + raise ValueError("rich table source must remain atomic") + expected_layout = tuple( + (start, end) + for _, start, end in _expected_rich_fragment_layout( + canonical_text, token_ceiling + ) + ) + actual_layout = tuple( + ( + section.position.start.byte_offset, + section.position.end.byte_offset, + ) + for section in sections + ) + if actual_layout != expected_layout: + raise ValueError("rich block splitting must be exact") + for section, fragment in zip(sections, fragments, strict=True): + if ( + fragment.kind is not section.kind + or fragment.path != section.path + or fragment.position != section.position + or fragment.fragment_ref in refs + or section.position.start.byte_offset < prior_end + ): + raise ValueError("rich Fragment lineage must match source order") + if _expected_rich_point( + canonical_text, section.position.start.byte_offset + ) != ( + section.position.start + ) or _expected_rich_point( + canonical_text, section.position.end.byte_offset + ) != ( + section.position.end + ): + raise ValueError("rich source coordinates must match UTF-8 offsets") + source = canonical_bytes[ + section.position.start.byte_offset : section.position.end.byte_offset + ].decode("utf-8") + if source != fragment.source_text: + raise ValueError("rich Fragment source text must match its span") + if section.text != source and section.kind is not SectionKind.HEADING: + raise ValueError("rich section text must match its span") + _validate_rich_closed_grammar(section, source) + if section.kind is SectionKind.HEADING: + assert section.level is not None + heading_lines = source.splitlines() + setext = ( + len(heading_lines) == 2 + and re.fullmatch(r"^ {0,3}(=+|-+)[ \t]*$", heading_lines[1]) + is not None + ) + atx = re.fullmatch( + r"^ {0,3}(#{1,6})[ \t]+(.+?)[ \t]*#*[ \t]*$", + source, + ) + expected_heading_text = ( + heading_lines[0].strip() + if setext + else atx.group(2).strip() + if atx is not None + else None + ) + expected_level = ( + 1 + if setext and heading_lines[1].lstrip().startswith("=") + else 2 + if setext + else len(atx.group(1)) + if atx is not None + else None + ) + if section.text != expected_heading_text or section.level != expected_level: + raise ValueError("rich heading metadata must match its source") + headings = [ + heading + for heading in headings + if heading.level is not None and heading.level < section.level + ] + parents = tuple(headings) + parent_path = parents[-1].path.segments if parents else ("document",) + counter_key = (parent_path, section.kind) + counters[counter_key] = counters.get(counter_key, 0) + 1 + expected_path = StructuralPath( + parent_path + (f"{section.kind.value}[{counters[counter_key]}]",) + ) + kind_ordinals[section.kind] = kind_ordinals.get(section.kind, 0) + 1 + expected_ref = f"fragment:{section.kind.value}:{kind_ordinals[section.kind]}" + if ( + section.path != expected_path + or fragment.parent_headings != parents + or fragment.fragment_ref != expected_ref + or fragment.search_phrases + != _expected_rich_search_phrases(section, source) + ): + raise ValueError("rich Fragment derivation must be exact") + omitted = canonical_bytes[ + prior_end : section.position.start.byte_offset + ] + if any(byte not in b" \t\r\n\xef\xbb\xbf" for byte in omitted): + raise ValueError("rich sections cannot omit non-whitespace source") + if fragment.contextual_text != _expected_contextual_text(fragment): + raise ValueError("rich Fragment context must be exact heading ancestry") + if len(re.findall(r"\S+", fragment.contextual_text)) > token_ceiling: + raise ValueError("rich Fragment exceeds its provenance-bound ceiling") + if section.kind is SectionKind.HEADING: + headings.append(section) + refs.add(fragment.fragment_ref) + prior_end = section.position.end.byte_offset + if any(byte not in b" \t\r\n" for byte in canonical_bytes[prior_end:]): + raise ValueError("rich sections cannot omit trailing source") + + +def _expected_rich_point(source_text: str, byte_offset: int) -> SourcePoint: + prefix = source_text.encode("utf-8")[:byte_offset].decode("utf-8") + logical = prefix.replace("\r\n", "\n").replace("\r", "\n") + last_newline = logical.rfind("\n") + return SourcePoint( + line=logical.count("\n") + 1, + column=len(logical[last_newline + 1 :]) + 1, + byte_offset=byte_offset, + ) + + def _validate_issue_22_content( canonical_text: str, sections: tuple[ParsedSection, ...], @@ -987,3 +1800,128 @@ def canonicalize_parsed_document(document: ParsedDocument) -> bytes: canonical = _document_without_digest(document) canonical["compilationDigest"] = document.compilation_digest return rfc8785.dumps(cast(Any, canonical)) + + +def _source_point_from_document(value: object) -> SourcePoint: + if type(value) is not dict: + raise ValueError("source point document must be an object") + document = cast(dict[str, object], value) + if set(document) != {"line", "column", "byteOffset"}: + raise ValueError("source point document has unexpected fields") + return SourcePoint( + line=cast(int, document["line"]), + column=cast(int, document["column"]), + byte_offset=cast(int, document["byteOffset"]), + ) + + +def _source_span_from_document(value: object) -> SourceSpan: + if type(value) is not dict: + raise ValueError("source span document must be an object") + document = cast(dict[str, object], value) + if set(document) != {"start", "end"}: + raise ValueError("source span document has unexpected fields") + return SourceSpan( + start=_source_point_from_document(document["start"]), + end=_source_point_from_document(document["end"]), + ) + + +def _section_from_document(value: object) -> ParsedSection: + if type(value) is not dict: + raise ValueError("section document must be an object") + document = cast(dict[str, object], value) + kind = SectionKind(cast(str, document["kind"])) + path = StructuralPath(tuple(cast(list[str], document["path"]))) + return ParsedSection( + kind=kind, + text=cast(str, document["text"]), + path=path, + position=_source_span_from_document(document["position"]), + level=cast(int | None, document.get("level")), + list_ordered=cast(bool | None, document.get("ordered")), + list_items=tuple(cast(list[str], document.get("items", []))), + code_language=cast(str | None, document.get("language")), + code_body=cast(str | None, document.get("code")), + table_header=tuple(cast(list[str], document.get("header", []))), + table_rows=tuple( + tuple(row) for row in cast(list[list[str]], document.get("rows", [])) + ), + ) + + +def _fragment_from_document( + value: object, + heading_by_key: dict[tuple[str, ...], ParsedSection], +) -> CompiledFragment: + if type(value) is not dict: + raise ValueError("Fragment document must be an object") + document = cast(dict[str, object], value) + parents: list[ParsedSection] = [] + for parent_value in cast(list[object], document["parentHeadings"]): + if type(parent_value) is not dict: + raise ValueError("parent heading document must be an object") + parent = cast(dict[str, object], parent_value) + key = tuple(cast(list[str], parent["path"])) + heading = heading_by_key.get(key) + if heading is None: + raise ValueError("Fragment parent heading must name a parsed section") + parents.append(heading) + return CompiledFragment( + fragment_ref=cast(str, document["fragmentRef"]), + kind=SectionKind(cast(str, document["kind"])), + path=StructuralPath(tuple(cast(list[str], document["path"]))), + position=_source_span_from_document(document["position"]), + source_text=cast(str, document["sourceText"]), + contextual_text=cast(str, document["contextualText"]), + parent_headings=tuple(parents), + search_phrases=tuple(cast(list[str], document["searchPhrases"])), + ) + + +def deserialize_parsed_document(payload: bytes) -> ParsedDocument: + """Deserialize runner bytes into the existing self-validating contract.""" + + if type(payload) is not bytes: + raise TypeError("parsed document payload must be exact bytes") + raw = json.loads(payload) + if type(raw) is not dict: + raise ValueError("parsed document payload must contain one object") + document = cast(dict[str, object], raw) + provenance_value = document["provenance"] + if type(provenance_value) is not dict: + raise ValueError("parsed document provenance must be an object") + provenance_document = cast(dict[str, object], provenance_value) + provenance = CompilationProvenance( + compiler_version=cast(str, provenance_document["compilerVersion"]), + config_version=cast(str, provenance_document["configVersion"]), + canonicalization_profile=cast( + str, provenance_document["canonicalizationProfile"] + ), + content_hash_profile=cast(str, provenance_document["contentHashProfile"]), + compilation_digest_profile=cast( + str, provenance_document["compilationDigestProfile"] + ), + token_ceiling=cast(int | None, provenance_document.get("tokenCeiling")), + ) + sections = tuple( + _section_from_document(value) + for value in cast(list[object], document["sections"]) + ) + heading_by_key = { + section.path.segments: section + for section in sections + if section.kind is SectionKind.HEADING + } + fragments = tuple( + _fragment_from_document(value, heading_by_key) + for value in cast(list[object], document.get("fragments", [])) + ) + return ParsedDocument( + canonical_text=cast(str, document["canonicalText"]), + sections=sections, + content_hash=cast(str, document["contentHash"]), + compilation_digest=cast(str, document["compilationDigest"]), + provenance=provenance, + fragments=fragments, + ) diff --git a/eval/_compiler_acceptance.py b/eval/_compiler_acceptance.py new file mode 100644 index 00000000..ad499950 --- /dev/null +++ b/eval/_compiler_acceptance.py @@ -0,0 +1,31 @@ +"""Private capability for local compiler acceptance and tests only.""" + +from __future__ import annotations + +from typing import Final + +_CONSTRUCTION_TOKEN: Final = object() + + +class _AcceptanceContext: + __slots__ = () + + def __new__(cls, token: object) -> _AcceptanceContext: + if cls is not _AcceptanceContext or token is not _CONSTRUCTION_TOKEN: + raise TypeError("compiler acceptance context cannot be constructed") + return super().__new__(cls) + + +_CONTEXT: Final = _AcceptanceContext(_CONSTRUCTION_TOKEN) + + +def acceptance_context() -> _AcceptanceContext: + """Return the process-local capability for explicit acceptance work.""" + + return _CONTEXT + + +def is_acceptance_context(value: object) -> bool: + """Return whether the exact private process-local capability was supplied.""" + + return value is _CONTEXT diff --git a/eval/embedding_benchmark.py b/eval/embedding_benchmark.py index b349eff6..620b79c8 100644 --- a/eval/embedding_benchmark.py +++ b/eval/embedding_benchmark.py @@ -882,7 +882,7 @@ def _validate_report(document: object, schema: object) -> None: def validate_json_schema_document(value: object, schema: object) -> None: - """Validate documents against the tracked schema vocabulary used by eval.""" + """Validate documents against the tracked schema vocabulary used by reports.""" _validate_bounded_json_value(value) _validate_bounded_json_value(schema) @@ -928,7 +928,10 @@ def _validate_json_schema( raise BenchmarkUnavailable("benchmark report schema is unavailable") properties = _object(schema.get("properties", {})) additional = schema.get("additionalProperties", True) + property_names = schema.get("propertyNames") for key, item in document.items(): + if property_names is not None: + _validate_json_schema(key, _object(property_names), root) child_schema = properties.get(key) if child_schema is not None: _validate_json_schema(item, _object(child_schema), root) diff --git a/pyproject.toml b/pyproject.toml index 92953fa3..1a0c5385 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,13 +8,20 @@ version = "0.1.0" description = "Permission-aware context delivery engine" readme = "README.md" license = "Apache-2.0" -license-files = ["LICENSE", "NOTICE"] +license-files = [ + "LICENSE", + "NOTICE", + "THIRD_PARTY_NOTICES.md", + "third_party/ragflow/LICENSE.upstream", + "third_party/ragflow/LICENSE.python-markdown", +] requires-python = ">=3.13,<3.14" dependencies = [ "alembic>=1.16,<1.17", "cryptography>=49,<50", "fastapi>=0.116,<0.117", "jsonschema>=4.25,<5", + "markdown>=3.6,<3.7", "pydantic>=2.13,<2.14", "psycopg[binary]>=3.2,<3.3", "rfc8785>=0.1.4,<0.2", @@ -48,7 +55,14 @@ benchmark = [ ] [tool.hatch.build.targets.wheel] -packages = ["engine", "adapters", "applications", "eval", "migrations"] +packages = [ + "engine", + "adapters", + "applications", + "eval", + "migrations", + "third_party", +] [tool.hatch.build.targets.wheel.force-include] "eval/golden/v1/schema.json" = "eval/golden/v1/schema.json" @@ -82,3 +96,6 @@ line-length = 88 [tool.ruff.lint] select = ["E", "F", "I", "UP", "B", "SIM"] + +[tool.ruff.lint.per-file-ignores] +"third_party/**/*.py" = ["E501", "SIM103", "UP009", "UP032"] diff --git a/scripts/run_m0_security_gate.py b/scripts/run_m0_security_gate.py index 06fddbbd..43bc10c5 100644 --- a/scripts/run_m0_security_gate.py +++ b/scripts/run_m0_security_gate.py @@ -4,8 +4,11 @@ from __future__ import annotations import argparse +import io +import logging import sys from collections.abc import Sequence +from contextlib import redirect_stderr, redirect_stdout from pathlib import Path from scripts.security_gate.runner import GatePaths, run_gate @@ -25,15 +28,20 @@ def build_parser() -> argparse.ArgumentParser: def main(argv: Sequence[str] | None = None) -> int: arguments = build_parser().parse_args(argv) paths = GatePaths.defaults(arguments.output_dir.resolve()) + prior_logging_disable = logging.root.manager.disable try: - report = run_gate(paths) + logging.disable(logging.CRITICAL) + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + report = run_gate(paths) except Exception as error: print(f"M0 security gate failed: {type(error).__name__}", file=sys.stderr) return 1 + finally: + logging.disable(prior_logging_disable) if report.get("m0SecurityDecision") != "pass": print("M0 SECURITY FAIL", file=sys.stderr) return 1 - print(f"M0 SECURITY PASS ({paths.output_directory})") + print("M0 SECURITY PASS") return 0 diff --git a/scripts/security_gate/runner.py b/scripts/security_gate/runner.py index 9d2803e5..b52b2fda 100644 --- a/scripts/security_gate/runner.py +++ b/scripts/security_gate/runner.py @@ -260,10 +260,31 @@ def build_pytest_command( ) +def _report_execution_command( + selectors: Sequence[str], +) -> list[str]: + """Describe gate execution without retaining machine-local absolute paths.""" + + return list( + build_pytest_command( + selectors, + raw_path=Path(".context-engine/security-gate") / RAW_ARTIFACT_NAME, + python_executable="python", + ) + ) + + def _execute_pytest( command: Sequence[str], *, cwd: Path, env: Mapping[str, str] ) -> int: - return subprocess.run(command, cwd=cwd, env=env, check=False).returncode + return subprocess.run( + command, + cwd=cwd, + env=env, + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ).returncode def _audit_rls( @@ -450,12 +471,7 @@ def _provenance( return { "runnerVersion": RUNNER_VERSION, **_git_state(paths.repository_root), - "executionCommand": list( - build_pytest_command( - selectors, - raw_path=paths.output_directory / RAW_ARTIFACT_NAME, - ) - ), + "executionCommand": _report_execution_command(selectors), "catalogDigest": _file_digest(paths.catalog), "fixtureDigest": canonical_digest(fixtures), "catalogSchemaDigest": _file_digest(paths.catalog_schema), @@ -536,12 +552,7 @@ def _best_effort_provenance( if live_database_revision is not None: provenance["liveDatabaseRevision"] = live_database_revision if selectors: - provenance["executionCommand"] = list( - build_pytest_command( - selectors, - raw_path=paths.output_directory / RAW_ARTIFACT_NAME, - ) - ) + provenance["executionCommand"] = _report_execution_command(selectors) configuration_fields = { field: provenance[field] for field in ( diff --git a/tests/catalog/test_m0_security_gate.py b/tests/catalog/test_m0_security_gate.py index a2b5ab46..42a94ac4 100644 --- a/tests/catalog/test_m0_security_gate.py +++ b/tests/catalog/test_m0_security_gate.py @@ -656,6 +656,18 @@ def test_provenance_has_exact_nonsecret_config_migration_and_fixture_digests( assert len(cast(str, provenance["fixtureDigest"])) == 64 assert len(cast(str, provenance["configurationDigest"])) == 64 assert "database.env" not in json.dumps(provenance) + assert provenance["executionCommand"] == [ + "python", + "-m", + "pytest", + "-p", + "scripts.security_gate.pytest_plugin", + "--security-gate-raw", + ".context-engine/security-gate/raw-evidence.json", + "--strict-markers", + "--strict-config", + "tests/test_gate_sample.py::test_fixture", + ] def test_git_state_hashes_staged_unstaged_and_untracked_nonignored_content( diff --git a/tests/fixtures/markdown/rich-code-tables.md b/tests/fixtures/markdown/rich-code-tables.md new file mode 100644 index 00000000..323bd36a --- /dev/null +++ b/tests/fixtures/markdown/rich-code-tables.md @@ -0,0 +1,12 @@ +# Technical notes + +````python +print("outer") +``` +print("still outer") +```` + +| Name | State | +| :--- | ---: | +| alpha | ready | +| beta | held | diff --git a/tests/fixtures/markdown/rich-expanded.md b/tests/fixtures/markdown/rich-expanded.md new file mode 100644 index 00000000..d5351122 --- /dev/null +++ b/tests/fixtures/markdown/rich-expanded.md @@ -0,0 +1,27 @@ +# Expanded grammar + +> Ordinary blockquote with **strong**, *emphasis*, `inline code`, $x^2$, and ~~strike~~. + +[inline link](https://example.test) and ![inline image](image.png). + +[reference link][reference] and ![reference image][reference]. + +[reference]: https://example.test + + + +Next line uses a hard break.\ +Final line. + +* * * + +| A | B | +| --- | --- | +| ragged | + +| A | B | +| --- | --- | +| empty | | + +``` +Literal unmatched fence content. diff --git a/tests/fixtures/markdown/rich-frontmatter.md b/tests/fixtures/markdown/rich-frontmatter.md new file mode 100644 index 00000000..78f6ab7e --- /dev/null +++ b/tests/fixtures/markdown/rich-frontmatter.md @@ -0,0 +1,8 @@ +--- +category: handbook +aliases: + - field guide +--- +# Handbook + +Opening paragraph with [[guide|the guide]] and ![[diagram]]. diff --git a/tests/fixtures/markdown/rich-headings-lists.md b/tests/fixtures/markdown/rich-headings-lists.md new file mode 100644 index 00000000..fd3ae313 --- /dev/null +++ b/tests/fixtures/markdown/rich-headings-lists.md @@ -0,0 +1,12 @@ +Handbook +======== + +## Repeated + +- first item + - nested item + 1. ordered child + +## Repeated + +Closing paragraph. diff --git a/tests/fixtures/markdown/rich-inline-html.md b/tests/fixtures/markdown/rich-inline-html.md new file mode 100644 index 00000000..3bebfc43 --- /dev/null +++ b/tests/fixtures/markdown/rich-inline-html.md @@ -0,0 +1,12 @@ +# Reference + +> [!NOTE] +> A callout with inline math $x^2 + y^2$. + +Term with a footnote.[^1] + +[^1]: Footnote body. + +
+HTML block body. +
diff --git a/tests/fixtures/markdown/rich-mixed-newlines.hex b/tests/fixtures/markdown/rich-mixed-newlines.hex new file mode 100644 index 00000000..6095309b --- /dev/null +++ b/tests/fixtures/markdown/rich-mixed-newlines.hex @@ -0,0 +1 @@ +23204d697865640d0a0d0a4669727374206c696e652e0a5365636f6e64206c696e652e0d0a diff --git a/tests/integration/test_zzz_security_gate_cli_privacy.py b/tests/integration/test_zzz_security_gate_cli_privacy.py new file mode 100644 index 00000000..7a0a970b --- /dev/null +++ b/tests/integration/test_zzz_security_gate_cli_privacy.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import re +import subprocess +import sys +from pathlib import Path + +import pytest + +REPOSITORY_ROOT = Path(__file__).parents[2] +_ABSOLUTE_PATHS = ( + re.compile(r"(? None: + completed = subprocess.run( + ( + sys.executable, + "scripts/run_m0_security_gate.py", + "--output-dir", + str(tmp_path / "gate-evidence"), + ), + cwd=REPOSITORY_ROOT, + check=False, + capture_output=True, + text=True, + timeout=300, + ) + output = completed.stdout + completed.stderr + + assert completed.returncode == 0 + assert output == "M0 SECURITY PASS\n" + assert all(pattern.search(output) is None for pattern in _ABSOLUTE_PATHS) diff --git a/tests/unit/test_compiler_runner_acceptance.py b/tests/unit/test_compiler_runner_acceptance.py new file mode 100644 index 00000000..3797efe6 --- /dev/null +++ b/tests/unit/test_compiler_runner_acceptance.py @@ -0,0 +1,873 @@ +from __future__ import annotations + +from dataclasses import replace +from pathlib import Path + +import pytest + +from adapters.parsers.ragflow_markdown import compile_rich_markdown +from engine.supply import ( + MARKDOWN_COMPILER_V3_VERSION, + MARKDOWN_RICH_CANONICALIZATION_PROFILE, + MARKDOWN_RICH_COMPILATION_DIGEST_PROFILE, + CompilationFailure, + CompilationProvenance, + CompiledFragment, + MarkdownCompilerConfig, + ParsedDocument, + ParsedSection, + SectionKind, + SourcePoint, + SourceSpan, + StructuralPath, +) + +FIXTURES = Path(__file__).parents[1] / "fixtures/markdown" +CONFIG = MarkdownCompilerConfig(version="markdown-config-v3") +RICH_FIXTURES = ( + "rich-frontmatter.md", + "rich-headings-lists.md", + "rich-code-tables.md", + "rich-expanded.md", + "rich-inline-html.md", + "rich-mixed-newlines.hex", +) + + +def _source(name: str) -> bytes: + path = FIXTURES / name + if path.suffix == ".hex": + return bytes.fromhex(path.read_text(encoding="ascii")) + return path.read_bytes() + + +@pytest.mark.parametrize("fixture", RICH_FIXTURES) +def test_tracked_rich_construct_corpus_compiles_all_or_nothing(fixture: str) -> None: + outcome = compile_rich_markdown(_source(fixture), CONFIG) + + assert not isinstance(outcome, CompilationFailure) + assert type(outcome) is ParsedDocument + assert outcome.fragments + + +@pytest.mark.parametrize("fixture", RICH_FIXTURES) +def test_every_fragment_span_round_trips_to_exact_original_utf8( + fixture: str, +) -> None: + source = _source(fixture) + outcome = compile_rich_markdown(source, CONFIG) + + assert type(outcome) is ParsedDocument + for fragment in outcome.fragments: + span = fragment.position + assert source[span.start.byte_offset : span.end.byte_offset] == ( + fragment.source_text.encode("utf-8") + ) + + +def test_nested_setext_and_duplicate_headings_have_stable_ancestry() -> None: + outcome = compile_rich_markdown(_source("rich-headings-lists.md"), CONFIG) + + assert type(outcome) is ParsedDocument + headings = [ + fragment + for fragment in outcome.fragments + if fragment.kind is SectionKind.HEADING + ] + assert [fragment.path.segments for fragment in headings] == [ + ("document", "heading[1]"), + ("document", "heading[1]", "heading[1]"), + ("document", "heading[1]", "heading[2]"), + ] + closing = outcome.fragments[-1] + assert tuple(heading.text for heading in closing.parent_headings) == ( + "Handbook", + "Repeated", + ) + assert closing.path.segments == ( + "document", + "heading[1]", + "heading[2]", + "paragraph[1]", + ) + + +def test_lone_cr_setext_heading_compiles_with_exact_span_and_ancestry() -> None: + source = "标题🙂\r====\r\r段落\r".encode() + + outcome = compile_rich_markdown(source, CONFIG) + + assert type(outcome) is ParsedDocument + heading, paragraph = outcome.fragments + assert heading.kind is SectionKind.HEADING + assert heading.source_text == "标题🙂\r====" + assert paragraph.parent_headings == (outcome.sections[0],) + for fragment in outcome.fragments: + span = fragment.position + assert source[span.start.byte_offset : span.end.byte_offset] == ( + fragment.source_text.encode("utf-8") + ) + + +@pytest.mark.parametrize( + "source", + ( + b"# T\n\n- body \n", + b"# T\n\n```\nbody\n``` \n", + b"# T\n\n - body \n", + b"# T\n\n ```\nbody\n ``` \n", + ), +) +def test_valid_blocks_retain_trailing_whitespace_without_crashing( + source: bytes, +) -> None: + outcome = compile_rich_markdown(source, CONFIG) + + assert type(outcome) is ParsedDocument + fragment = outcome.fragments[-1] + span = fragment.position + assert source[span.start.byte_offset : span.end.byte_offset] == ( + fragment.source_text.encode("utf-8") + ) + + +def test_fence_precedes_setext_recognition_when_first_body_line_is_rule_like() -> None: + source = b"# T\n\n```text\n---\nbody\n```\n" + + outcome = compile_rich_markdown(source, CONFIG) + + assert type(outcome) is ParsedDocument + fence = outcome.fragments[-1] + assert fence.kind is SectionKind.FENCED_CODE + span = fence.position + assert source[span.start.byte_offset : span.end.byte_offset] == ( + fence.source_text.encode("utf-8") + ) + + +@pytest.mark.parametrize( + "source", + ( + b"---\nkey: value\n---\n# T\n", + b"T\n===\n\n## Child\n", + b"# T\n\n- parent\n 1. child\n", + b"# T\n\n~~~text\nbody\n~~~\n", + b"# T\n\n| A | B |\n| --- | --- |\n| x | y |\n", + b"# T\n\n[[target|label]] and ![[asset]]\n", + b"# T\n\nText[^1].\n\n[^1]: Note.\n", + b"# T\n\n
body
\n", + b"# T\n\n> [!NOTE]\n> callout body\n", + b"# T\n\n> quoted paragraph\n", + b"# T\n\n$x^2$\n", + b"# T\n\n**strong** and *emphasis*\n", + b"# T\n\nUse `inline code`.\n", + b"# T\n\n[link](https://example.test) and ![image](image.png)\n", + b"# T\n\n[link][ref] and ![image][ref]\n\n[ref]: https://example.test\n", + b"# T\n\n~~strikethrough~~\n", + b"# T\n\n\n", + b"# T\n\nhard break \nnext line\n", + b"# T\n\nhard break\\\nnext line\n", + b"# T\n\n---\n", + b"# T\n\n* * *\n", + b"# T\n\n_ _ _\n", + b"# T\n\nordinary paragraph\n", + b"# T\n\n| A | B |\n| --- | --- |\n| x |\n", + b"# T\n\n| A | B |\n| --- | --- |\n| x | |\n", + b"# T\n\n```\nliteral unmatched fence\n", + ), +) +def test_every_adr_listed_rich_construct_is_explicitly_accepted( + source: bytes, +) -> None: + outcome = compile_rich_markdown(source, CONFIG) + + assert type(outcome) is ParsedDocument + assert outcome.fragments + + +@pytest.mark.parametrize("source", (b"---\n", b"---\n\ntext\n")) +def test_leading_unclosed_dash_rule_is_an_accepted_thematic_break( + source: bytes, +) -> None: + outcome = compile_rich_markdown(source, CONFIG) + + assert type(outcome) is ParsedDocument + assert outcome.fragments[0].source_text == "---" + + +@pytest.mark.parametrize("newline", (b"\n", b"\r\n")) +@pytest.mark.parametrize("bom", (b"", b"\xef\xbb\xbf")) +@pytest.mark.parametrize( + ("lines", "expected_fragments"), + ( + ((b"---",), ("---",)), + ((b"---", b"---"), ("---", "---")), + ((b"---", b"", b"---"), ("---", "---")), + ((b"---", b"key: value", b"---"), ("frontmatter",)), + ((b"---", b"ordinary prose", b"---"), ("---", "ordinary-prose")), + ((b"---", b"key: value"), ("---", "key: value")), + ((b"---", b"---", b"text"), ("---", "---", "text")), + ), +) +def test_leading_dash_delimiter_matrix_is_closed_and_exact( + newline: bytes, + bom: bytes, + lines: tuple[bytes, ...], + expected_fragments: tuple[str, ...], +) -> None: + source = bom + newline.join(lines) + newline + + outcome = compile_rich_markdown(source, CONFIG) + + assert type(outcome) is ParsedDocument + expected = tuple( + ( + newline.join(lines).decode("utf-8") + if value == "frontmatter" + else newline.join(lines[1:]).decode("utf-8") + if value == "ordinary-prose" + else value + ) + for value in expected_fragments + ) + assert tuple(fragment.source_text for fragment in outcome.fragments) == expected + for fragment in outcome.fragments: + span = fragment.position + assert source[span.start.byte_offset : span.end.byte_offset] == ( + fragment.source_text.encode("utf-8") + ) + + +def test_rich_constructor_rejects_forged_lineage_and_ancestry() -> None: + compiled = compile_rich_markdown(b"# Root\n\nBody\n", CONFIG) + assert type(compiled) is ParsedDocument + heading, body = compiled.sections + heading_fragment, body_fragment = compiled.fragments + forged_heading = replace( + heading, + text="WRONG", + path=type(heading.path)(("invented", "heading[7]")), + ) + forged_body = replace( + body, + path=type(body.path)(("invented", "paragraph[9]")), + ) + forged_fragments = ( + replace( + heading_fragment, + fragment_ref="fragment:heading:9", + path=forged_heading.path, + search_phrases=("poison-heading",), + ), + replace( + body_fragment, + fragment_ref="fragment:paragraph:99", + path=forged_body.path, + parent_headings=(), + contextual_text=body_fragment.source_text, + search_phrases=("poison-body",), + ), + ) + + expected_error = "rich (?:heading metadata|Fragment derivation)" + with pytest.raises(ValueError, match=expected_error): + ParsedDocument.rich_v3( + canonical_text=compiled.canonical_text, + sections=(forged_heading, forged_body), + fragments=forged_fragments, + provenance=compiled.provenance, + ) + + +def _point_for(canonical_text: str, byte_offset: int) -> SourcePoint: + prefix = canonical_text.encode("utf-8")[:byte_offset].decode("utf-8") + logical = prefix.replace("\r\n", "\n").replace("\r", "\n") + return SourcePoint( + line=logical.count("\n") + 1, + column=len(logical.rsplit("\n", maxsplit=1)[-1]) + 1, + byte_offset=byte_offset, + ) + + +def _forged_sections( + canonical_text: str, + parts: tuple[tuple[SectionKind, str, tuple[str, ...]], ...], +) -> tuple[tuple[ParsedSection, ...], tuple[CompiledFragment, ...]]: + sections: list[ParsedSection] = [] + fragments: list[CompiledFragment] = [] + search_start = 0 + kind_ordinals: dict[SectionKind, int] = {} + for kind, source, list_items in parts: + start = canonical_text.index(source, search_start) + end = start + len(source.encode("utf-8")) + search_start = end + kind_ordinals[kind] = kind_ordinals.get(kind, 0) + 1 + ordinal = kind_ordinals[kind] + path = StructuralPath(("document", f"{kind.value}[{ordinal}]")) + position = SourceSpan( + start=_point_for(canonical_text, start), + end=_point_for(canonical_text, end), + ) + section = ParsedSection( + kind=kind, + text=source, + path=path, + position=position, + list_ordered=False if kind is SectionKind.LIST else None, + list_items=list_items, + ) + sections.append(section) + fragments.append( + CompiledFragment( + fragment_ref=f"fragment:{kind.value}:{ordinal}", + kind=kind, + path=path, + position=position, + source_text=source, + contextual_text=source, + parent_headings=(), + search_phrases=(source,), + ) + ) + return tuple(sections), tuple(fragments) + + +def test_rich_constructor_rejects_split_of_undersize_paragraph() -> None: + canonical_text = "one two three\n" + sections, fragments = _forged_sections( + canonical_text, + ( + (SectionKind.PARAGRAPH, "one", ()), + (SectionKind.PARAGRAPH, "two three", ()), + ), + ) + + with pytest.raises(ValueError, match="rich block splitting must be exact"): + ParsedDocument.rich_v3( + canonical_text=canonical_text, + sections=sections, + fragments=fragments, + provenance=CompilationProvenance( + compiler_version=MARKDOWN_COMPILER_V3_VERSION, + config_version="markdown-config-v3", + canonicalization_profile=MARKDOWN_RICH_CANONICALIZATION_PROFILE, + compilation_digest_profile=MARKDOWN_RICH_COMPILATION_DIGEST_PROFILE, + token_ceiling=2048, + ), + ) + + +def test_rich_constructor_rejects_split_of_one_contiguous_list() -> None: + canonical_text = "- one\n- two\n" + sections, fragments = _forged_sections( + canonical_text, + ( + (SectionKind.LIST, "- one", ("one",)), + (SectionKind.LIST, "- two", ("two",)), + ), + ) + + with pytest.raises(ValueError, match="rich block splitting must be exact"): + ParsedDocument.rich_v3( + canonical_text=canonical_text, + sections=sections, + fragments=fragments, + provenance=CompilationProvenance( + compiler_version=MARKDOWN_COMPILER_V3_VERSION, + config_version="markdown-config-v3", + canonicalization_profile=MARKDOWN_RICH_CANONICALIZATION_PROFILE, + compilation_digest_profile=MARKDOWN_RICH_COMPILATION_DIGEST_PROFILE, + token_ceiling=2048, + ), + ) + + +def test_rich_constructor_rederives_section_kind_from_exact_source() -> None: + compiled = compile_rich_markdown(b"# Root\n", CONFIG) + assert type(compiled) is ParsedDocument + heading = compiled.sections[0] + fragment = compiled.fragments[0] + forged_path = StructuralPath(("document", "paragraph[1]")) + + with pytest.raises(ValueError, match="rich section kind"): + ParsedDocument.rich_v3( + canonical_text=compiled.canonical_text, + sections=( + replace( + heading, + kind=SectionKind.PARAGRAPH, + text=fragment.source_text, + path=forged_path, + level=None, + ), + ), + fragments=( + replace( + fragment, + fragment_ref="fragment:paragraph:1", + kind=SectionKind.PARAGRAPH, + path=forged_path, + search_phrases=(fragment.source_text,), + ), + ), + provenance=compiled.provenance, + ) + + +def test_rich_constructor_rederives_table_kind_from_exact_source() -> None: + compiled = compile_rich_markdown( + b"| A | B |\n| --- | --- |\n| x | y |\n", + CONFIG, + ) + assert type(compiled) is ParsedDocument + table = compiled.sections[0] + fragment = compiled.fragments[0] + forged_path = StructuralPath(("document", "paragraph[1]")) + + with pytest.raises(ValueError, match="rich section kind"): + ParsedDocument.rich_v3( + canonical_text=compiled.canonical_text, + sections=( + replace( + table, + kind=SectionKind.PARAGRAPH, + path=forged_path, + table_header=(), + table_rows=(), + ), + ), + fragments=( + replace( + fragment, + fragment_ref="fragment:paragraph:1", + kind=SectionKind.PARAGRAPH, + path=forged_path, + ), + ), + provenance=compiled.provenance, + ) + + +def test_rich_constructor_rejects_forged_table_kind_for_ragged_source() -> None: + compiled = compile_rich_markdown( + b"| A | B |\n| --- | --- |\n| x |\n", + CONFIG, + ) + assert type(compiled) is ParsedDocument + paragraph = compiled.sections[0] + fragment = compiled.fragments[0] + forged_path = StructuralPath(("document", "table[1]")) + + with pytest.raises(ValueError, match="rich section kind"): + ParsedDocument.rich_v3( + canonical_text=compiled.canonical_text, + sections=( + replace( + paragraph, + kind=SectionKind.TABLE, + path=forged_path, + table_header=("A", "B"), + table_rows=(("x",),), + ), + ), + fragments=( + replace( + fragment, + fragment_ref="fragment:table:1", + kind=SectionKind.TABLE, + path=forged_path, + ), + ), + provenance=compiled.provenance, + ) + + +def test_rich_constructor_rejects_language_bearing_unmatched_fence() -> None: + compiled = compile_rich_markdown(b"Plaintext\nbody\n", CONFIG) + assert type(compiled) is ParsedDocument + paragraph = compiled.sections[0] + fragment = compiled.fragments[0] + forged_source = "```python\nbody" + assert len(forged_source.encode("utf-8")) == fragment.position.end.byte_offset + + with pytest.raises(ValueError, match="closed grammar"): + ParsedDocument.rich_v3( + canonical_text=f"{forged_source}\n", + sections=(replace(paragraph, text=forged_source),), + fragments=( + replace( + fragment, + source_text=forged_source, + contextual_text=forged_source, + search_phrases=(forged_source,), + ), + ), + provenance=compiled.provenance, + ) + + +@pytest.mark.parametrize( + "control_character", + ("\x00", "\x07", "\x1b", "\x1f", "\x7f", "\x85"), +) +def test_rich_constructor_rejects_every_control_character_forged_inside_fence( + control_character: str, +) -> None: + compiled = compile_rich_markdown(b"```text\nbody\n```\n", CONFIG) + assert type(compiled) is ParsedDocument + section = compiled.sections[0] + fragment = compiled.fragments[0] + forged_body = f"bo{control_character}y" + forged_source = fragment.source_text.replace("body", forged_body) + + with pytest.raises(ValueError, match="control character"): + ParsedDocument.rich_v3( + canonical_text=f"{forged_source}\n", + sections=( + replace( + section, + text=forged_source, + code_body=forged_body, + ), + ), + fragments=( + replace( + fragment, + source_text=forged_source, + contextual_text=forged_source, + search_phrases=(forged_source,), + ), + ), + provenance=compiled.provenance, + ) + + +@pytest.mark.parametrize( + ("compiler_version", "config_version"), + ( + ("context-engine-markdown-v1", "markdown-config-v1"), + ("arbitrary-compiler", "arbitrary-config"), + ), +) +def test_rich_constructor_rejects_forged_v3_provenance_identity( + compiler_version: str, + config_version: str, +) -> None: + compiled = compile_rich_markdown(b"Plain\n", CONFIG) + assert type(compiled) is ParsedDocument + + with pytest.raises(ValueError, match="rich provenance identity"): + provenance = CompilationProvenance( + compiler_version=compiler_version, + config_version=config_version, + canonicalization_profile=MARKDOWN_RICH_CANONICALIZATION_PROFILE, + compilation_digest_profile=MARKDOWN_RICH_COMPILATION_DIGEST_PROFILE, + token_ceiling=2048, + ) + ParsedDocument.rich_v3( + canonical_text=compiled.canonical_text, + sections=compiled.sections, + fragments=compiled.fragments, + provenance=provenance, + ) + + +def test_rich_constructor_rejects_unlisted_construct_in_forged_document() -> None: + compiled = compile_rich_markdown(b"Plain\n", CONFIG) + assert type(compiled) is ParsedDocument + paragraph = compiled.sections[0] + fragment = compiled.fragments[0] + forged_source = "&" + + with pytest.raises(ValueError, match="closed grammar"): + ParsedDocument.rich_v3( + canonical_text=f"{forged_source}\n", + sections=(replace(paragraph, text=forged_source),), + fragments=( + replace( + fragment, + source_text=forged_source, + contextual_text=forged_source, + search_phrases=(forged_source,), + ), + ), + provenance=compiled.provenance, + ) + + +def test_rich_constructor_rejects_unlisted_construct_in_ragged_table() -> None: + compiled = compile_rich_markdown( + b"| A | B |\n| --- | --- |\n| x | value |\n| ragged |\n", + CONFIG, + ) + assert type(compiled) is ParsedDocument + paragraph = compiled.sections[0] + fragment = compiled.fragments[0] + forged_source = fragment.source_text.replace("value", "&") + + with pytest.raises(ValueError, match="closed grammar"): + ParsedDocument.rich_v3( + canonical_text=f"{forged_source}\n", + sections=(replace(paragraph, text=forged_source),), + fragments=( + replace( + fragment, + source_text=forged_source, + contextual_text=forged_source, + search_phrases=(forged_source,), + ), + ), + provenance=compiled.provenance, + ) + + +def test_rich_constructor_does_not_treat_arbitrary_pipe_lines_as_table() -> None: + compiled = compile_rich_markdown( + b"first | line\nsecond | line\nplain value\n", + CONFIG, + ) + assert type(compiled) is ParsedDocument + paragraph = compiled.sections[0] + fragment = compiled.fragments[0] + forged_source = fragment.source_text.replace("value", "&") + + with pytest.raises(ValueError, match="closed grammar"): + ParsedDocument.rich_v3( + canonical_text=f"{forged_source}\n", + sections=(replace(paragraph, text=forged_source),), + fragments=( + replace( + fragment, + source_text=forged_source, + contextual_text=forged_source, + search_phrases=(forged_source,), + ), + ), + provenance=compiled.provenance, + ) + + +def test_rich_constructor_rejects_split_fragments_of_one_atomic_ragged_table() -> None: + canonical_text = ( + "| A | B |\n" + "| --- | --- |\n" + "| one two three | four five six |\n" + "| ragged seven eight |\n" + ) + source_parts = ( + "| A | B |\n| --- |", + "--- |\n| one two three | four", + "five six |\n| ragged seven eight |", + ) + + def point(byte_offset: int) -> SourcePoint: + prefix = canonical_text.encode("utf-8")[:byte_offset].decode("utf-8") + return SourcePoint( + line=prefix.count("\n") + 1, + column=len(prefix.rsplit("\n", maxsplit=1)[-1]) + 1, + byte_offset=byte_offset, + ) + + sections: list[ParsedSection] = [] + fragments: list[CompiledFragment] = [] + search_start = 0 + for ordinal, source_part in enumerate(source_parts, start=1): + start = canonical_text.index(source_part, search_start) + end = start + len(source_part) + search_start = end + path = StructuralPath(("document", f"paragraph[{ordinal}]")) + position = SourceSpan(start=point(start), end=point(end)) + section = ParsedSection( + kind=SectionKind.PARAGRAPH, + text=source_part, + path=path, + position=position, + ) + sections.append(section) + fragments.append( + CompiledFragment( + fragment_ref=f"fragment:paragraph:{ordinal}", + kind=SectionKind.PARAGRAPH, + path=path, + position=position, + source_text=source_part, + contextual_text=source_part, + parent_headings=(), + search_phrases=(source_part,), + ) + ) + + with pytest.raises(ValueError, match="rich table source must remain atomic"): + ParsedDocument.rich_v3( + canonical_text=canonical_text, + sections=tuple(sections), + fragments=tuple(fragments), + provenance=CompilationProvenance( + compiler_version=MARKDOWN_COMPILER_V3_VERSION, + config_version="markdown-config-v3", + canonicalization_profile=MARKDOWN_RICH_CANONICALIZATION_PROFILE, + compilation_digest_profile=MARKDOWN_RICH_COMPILATION_DIGEST_PROFILE, + token_ceiling=8, + ), + ) + + +def test_rich_constructor_rejects_ragged_table_forged_as_table_then_paragraph() -> None: + table_source = "| A | B |\n| --- | --- |\n| x | y |" + ragged_source = "| ragged |" + canonical_text = f"{table_source}\n{ragged_source}\n" + table_document = compile_rich_markdown(table_source.encode("utf-8"), CONFIG) + ragged_document = compile_rich_markdown(ragged_source.encode("utf-8"), CONFIG) + assert type(table_document) is ParsedDocument + assert type(ragged_document) is ParsedDocument + table_section = table_document.sections[0] + table_fragment = table_document.fragments[0] + ragged_start = len(f"{table_source}\n".encode()) + ragged_end = ragged_start + len(ragged_source.encode("utf-8")) + ragged_path = StructuralPath(("document", "paragraph[1]")) + ragged_position = SourceSpan( + start=SourcePoint(line=4, column=1, byte_offset=ragged_start), + end=SourcePoint( + line=4, + column=len(ragged_source) + 1, + byte_offset=ragged_end, + ), + ) + ragged_section = replace( + ragged_document.sections[0], + path=ragged_path, + position=ragged_position, + ) + ragged_fragment = replace( + ragged_document.fragments[0], + path=ragged_path, + position=ragged_position, + ) + + with pytest.raises(ValueError, match="rich table source must remain atomic"): + ParsedDocument.rich_v3( + canonical_text=canonical_text, + sections=(table_section, ragged_section), + fragments=(table_fragment, ragged_fragment), + provenance=table_document.provenance, + ) + + +@pytest.mark.parametrize( + ("original", "forged_source"), + ( + (b"> plain\n", "> &"), + (b"- item\n plain\n", "- item\n &"), + ), +) +def test_rich_constructor_rejects_unlisted_construct_in_nested_context( + original: bytes, + forged_source: str, +) -> None: + compiled = compile_rich_markdown(original, CONFIG) + assert type(compiled) is ParsedDocument + section = compiled.sections[0] + fragment = compiled.fragments[0] + assert len(forged_source.encode()) == fragment.position.end.byte_offset + + with pytest.raises(ValueError, match="closed grammar"): + ParsedDocument.rich_v3( + canonical_text=f"{forged_source}\n", + sections=(replace(section, text=forged_source),), + fragments=( + replace( + fragment, + source_text=forged_source, + contextual_text=forged_source, + search_phrases=(forged_source,), + ), + ), + provenance=compiled.provenance, + ) + + +def test_rich_constructor_rederives_list_metadata() -> None: + compiled = compile_rich_markdown(b"- first\n 1. child\n", CONFIG) + assert type(compiled) is ParsedDocument + + with pytest.raises(ValueError, match="rich section metadata"): + ParsedDocument.rich_v3( + canonical_text=compiled.canonical_text, + sections=(replace(compiled.sections[0], list_items=("forged",)),), + fragments=compiled.fragments, + provenance=compiled.provenance, + ) + + +def test_rich_constructor_rederives_code_metadata() -> None: + compiled = compile_rich_markdown(b"```python\nbody\n```\n", CONFIG) + assert type(compiled) is ParsedDocument + + with pytest.raises(ValueError, match="rich section metadata"): + ParsedDocument.rich_v3( + canonical_text=compiled.canonical_text, + sections=( + replace( + compiled.sections[0], + code_language="forged", + code_body="forged", + ), + ), + fragments=compiled.fragments, + provenance=compiled.provenance, + ) + + +def test_rich_constructor_rederives_table_metadata() -> None: + compiled = compile_rich_markdown( + b"| A | B |\n| --- | --- |\n| x | y |\n", + CONFIG, + ) + assert type(compiled) is ParsedDocument + + with pytest.raises(ValueError, match="rich section metadata"): + ParsedDocument.rich_v3( + canonical_text=compiled.canonical_text, + sections=( + replace( + compiled.sections[0], + table_header=("forged",), + table_rows=(("forged",),), + ), + ), + fragments=compiled.fragments, + provenance=compiled.provenance, + ) + + +@pytest.mark.parametrize( + "frontmatter", + ( + ( + b"owner:\n" + b" name: compiler\n" + b" teams:\n" + b" - supply\n" + b"description: |\n" + b" Compiler metadata keeps\n" + b" ---\n" + b" its exact source lines.\n" + ), + b"- alpha\n- beta\n", + ), +) +def test_delimited_yaml_frontmatter_is_accepted_as_exact_source_fragment( + frontmatter: bytes, +) -> None: + source = b"---\n" + frontmatter + b"---\n# Handbook\n" + + outcome = compile_rich_markdown(source, CONFIG) + + assert type(outcome) is ParsedDocument + frontmatter_fragment = outcome.fragments[0] + expected = source[: source.index(b"#")].rstrip(b"\n").decode() + assert frontmatter_fragment.source_text == expected + span = frontmatter_fragment.position + assert source[span.start.byte_offset : span.end.byte_offset] == ( + frontmatter_fragment.source_text.encode("utf-8") + ) diff --git a/tests/unit/test_compiler_runner_acceptance_report.py b/tests/unit/test_compiler_runner_acceptance_report.py new file mode 100644 index 00000000..d7010849 --- /dev/null +++ b/tests/unit/test_compiler_runner_acceptance_report.py @@ -0,0 +1,297 @@ +from __future__ import annotations + +import json +import re +import subprocess +import sys +from copy import deepcopy +from pathlib import Path + +import pytest + +import applications.compiler_runner as compiler_runner +from eval.embedding_benchmark import ( + BenchmarkUnavailable, + validate_json_schema_document, +) + +REPOSITORY_ROOT = Path(__file__).parents[2] +REPORT_SCHEMA = ( + REPOSITORY_ROOT / "docs/contracts/compiler-runner-acceptance-v1.schema.json" +) +_PERSONAL_ROOT_PATTERNS = ( + re.compile(r"/" + "Users" + r"/[^/\s]+/"), + re.compile(r"/" + "home" + r"/[^/\s]+/"), + re.compile(r"[A-Za-z]:\\(?:Users|Documents and Settings)\\", re.IGNORECASE), +) +_SYNTHETIC_PRIVATE_ROOT_FRAGMENT = "-".join(("synthetic", "corpus", "canary")) +_PRIVACY_BEARING_SCHEMA_WORDS = frozenset( + {"excerpt", "file", "filename", "path", "root", "source", "text", "title"} +) +_PRIVACY_GUARD_PATHS = ( + Path(__file__), + REPOSITORY_ROOT / "tests/integration/test_zzz_security_gate_cli_privacy.py", +) + + +def _schema_property_names(value: object) -> set[str]: + if type(value) is dict: + document = value + names: set[str] = set() + properties = document.get("properties") + if type(properties) is dict: + names.update(str(key).casefold() for key in properties) + names.update( + nested + for item in document.values() + for nested in _schema_property_names(item) + ) + return names + if type(value) is list: + return { + nested for item in value for nested in _schema_property_names(item) + } + return set() + + +def _schema_name_words(name: str) -> set[str]: + return { + word.casefold() + for word in re.findall( + r"[A-Z]+(?=[A-Z][a-z]|\b)|[A-Z]?[a-z]+|[0-9]+", + name, + ) + } + + +def _contains_private_location(value: str) -> bool: + return any(pattern.search(value) for pattern in _PERSONAL_ROOT_PATTERNS) or ( + _SYNTHETIC_PRIVATE_ROOT_FRAGMENT in value + ) + + +def test_privacy_guards_do_not_embed_identifier_fingerprints() -> None: + fingerprint = re.compile(r"(? None: + assert _contains_private_location("/" + "Users" + "/person/corpus") + assert _contains_private_location(_SYNTHETIC_PRIVATE_ROOT_FRAGMENT) + assert not _contains_private_location("aggregate-counts-only") + + +def test_tracked_tree_and_acceptance_schema_cannot_carry_private_paths() -> None: + tracked = subprocess.run( + ["git", "ls-files", "-z"], + cwd=REPOSITORY_ROOT, + check=True, + capture_output=True, + ).stdout.split(b"\0") + leaks: list[str] = [] + for raw_path in tracked: + if not raw_path: + continue + path = REPOSITORY_ROOT / raw_path.decode("utf-8") + content = path.read_text(encoding="utf-8", errors="ignore") + if _contains_private_location(content): + leaks.append(path.relative_to(REPOSITORY_ROOT).as_posix()) + + schema_text = REPORT_SCHEMA.read_text(encoding="utf-8") + schema = json.loads(schema_text) + schema_names = _schema_property_names(schema) + assert leaks == [] + assert not any( + _PRIVACY_BEARING_SCHEMA_WORDS.intersection(_schema_name_words(name)) + for name in schema_names + ) + assert all( + pattern.search(schema_text) is None + for pattern in _PERSONAL_ROOT_PATTERNS + ) + + +def _count_only_report() -> dict[str, object]: + return { + "aggregateCompilationDigest": "a" * 64, + "compilerVersion": "context-engine-markdown-v3", + "configVersion": "markdown-config-v3", + "constructHistogram": { + "atxHeadings": 0, + "callouts": 0, + "embeds": 0, + "fencedCode": 0, + "footnotes": 0, + "frontmatter": 0, + "htmlBlocks": 0, + "inlineMath": 0, + "lists": 0, + "setextHeadings": 0, + "tables": 0, + "wikilinks": 0, + }, + "documents": { + "accepted": 1, + "acceptanceRate": "1.000000", + "refused": 0, + "total": 1, + }, + "maxFragmentTokenCount": 1, + "refusalHistogram": {}, + "schemaVersion": "compiler-runner-acceptance-v1", + "tokenCeiling": 2048, + } + + +def _private_path_shapes() -> tuple[str, ...]: + slash = chr(47) + backslash = chr(92) + return ( + slash.join(("", "Volumes", "private", "entry.md")), + slash.join(("", "var", "private", "entry.md")), + slash.join(("~", "private", "entry.md")), + "Z" + chr(58) + backslash + backslash.join(("private", "entry.md")), + backslash * 2 + backslash.join(("server", "share", "entry.md")), + "private-entry.md", + ) + + +@pytest.mark.parametrize("path_shaped_key", _private_path_shapes()) +def test_acceptance_schema_rejects_paths_and_filenames_in_histogram_keys( + path_shaped_key: str, +) -> None: + schema = json.loads(REPORT_SCHEMA.read_text(encoding="utf-8")) + report = deepcopy(_count_only_report()) + refusal_histogram = report["refusalHistogram"] + assert type(refusal_histogram) is dict + refusal_histogram[path_shaped_key] = 1 + + with pytest.raises(BenchmarkUnavailable, match="report schema"): + validate_json_schema_document(report, schema) + + +def test_acceptance_report_is_count_only_deterministic_and_written_under_ignore( + tmp_path: Path, +) -> None: + corpus = tmp_path / "private-corpus" + corpus.mkdir() + (corpus / "one.md").write_text("# One\n\nFirst.\n", encoding="utf-8") + (corpus / "two.md").write_text( + "# Two\n\n| A | B |\n| --- | --- |\n| x | y |\n", + encoding="utf-8", + ) + (corpus / "refused.md").write_text( + "# Refused\n\n&\n", + encoding="utf-8", + ) + output = tmp_path / ".context-engine/compiler-runner-acceptance.json" + command = [ + sys.executable, + "-m", + "applications.compiler_runner", + "--acceptance-report", + "--root", + str(corpus), + "--output", + str(output), + ] + + first = subprocess.run( + command, check=True, capture_output=True, text=True, timeout=30 + ) + first_bytes = output.read_bytes() + second = subprocess.run( + command, check=True, capture_output=True, text=True, timeout=30 + ) + + report = json.loads(first_bytes) + schema = json.loads(REPORT_SCHEMA.read_text(encoding="utf-8")) + validate_json_schema_document(report, schema) + assert report["schemaVersion"] == "compiler-runner-acceptance-v1" + assert report["documents"] == { + "accepted": 2, + "acceptanceRate": "0.666667", + "refused": 1, + "total": 3, + } + assert report["refusalHistogram"] == { + "unsupported_construct:entity": 1, + } + assert report["aggregateCompilationDigest"] + assert report["maxFragmentTokenCount"] <= report["tokenCeiling"] + assert report["constructHistogram"]["tables"] == 1 + assert str(corpus) not in first_bytes.decode("utf-8") + assert "one.md" not in first_bytes.decode("utf-8") + assert "refused.md" not in first_bytes.decode("utf-8") + assert first_bytes == output.read_bytes() + assert first.stdout == second.stdout + + +@pytest.mark.parametrize( + "failure_kind", + ("io", "permission", "vanished", "directory"), +) +def test_acceptance_cli_error_paths_emit_only_counted_private_safe_outcomes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + failure_kind: str, +) -> None: + corpus = tmp_path / _SYNTHETIC_PRIVATE_ROOT_FRAGMENT + corpus.mkdir() + note = corpus / "private-note-canary.md" + if failure_kind == "directory": + note.mkdir() + elif failure_kind != "vanished": + note.write_text("# Synthetic\n", encoding="utf-8") + output = tmp_path / ".context-engine/compiler-runner-acceptance.json" + monkeypatch.setattr( + compiler_runner, + "_safe_markdown_files", + lambda root: (note,), + ) + original_read_bytes = Path.read_bytes + + def read_bytes(path: Path) -> bytes: + if path != note: + return original_read_bytes(path) + if failure_kind == "io": + raise OSError(5, "synthetic I/O failure", str(path)) + if failure_kind == "permission": + raise PermissionError(13, "synthetic permission failure", str(path)) + return original_read_bytes(path) + + monkeypatch.setattr(Path, "read_bytes", read_bytes) + monkeypatch.setattr( + sys, + "argv", + [ + "compiler-runner", + "--acceptance-report", + "--root", + str(corpus), + "--output", + str(output), + ], + ) + + compiler_runner.main() + + captured = capsys.readouterr() + emitted = captured.out + captured.err + report = json.loads(output.read_text(encoding="utf-8")) + assert report["documents"] == { + "accepted": 0, + "acceptanceRate": "0.000000", + "refused": 1, + "total": 1, + } + assert report["refusalHistogram"] == {"unsupported_document_shape": 1} + assert captured.err == "" + assert not _contains_private_location(emitted) + assert str(tmp_path) not in emitted + assert note.name not in emitted diff --git a/tests/unit/test_compiler_runner_bounds.py b/tests/unit/test_compiler_runner_bounds.py new file mode 100644 index 00000000..f409f074 --- /dev/null +++ b/tests/unit/test_compiler_runner_bounds.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import pytest + +from adapters.parsers.ragflow_markdown import compile_rich_markdown, rich_token_count +from engine.supply import CompilationFailure, MarkdownCompilerConfig, ParsedDocument + +CONFIG = MarkdownCompilerConfig(version="markdown-config-v3") + + +def test_oversize_blocks_split_under_hard_bound_with_exact_spans_and_ancestry() -> None: + ceiling = 64 + source = ( + "# Root\n\n## Deep\n\n" + + " ".join(f"token{index}" for index in range(17000)) + + "\n" + ).encode() + + config = MarkdownCompilerConfig( + version="markdown-config-v3", + token_ceiling=ceiling, + ) + outcome = compile_rich_markdown(source, config) + + assert type(outcome) is ParsedDocument + assert outcome.provenance.token_ceiling == ceiling + paragraph_fragments = outcome.fragments[2:] + assert len(paragraph_fragments) > 1 + for fragment in paragraph_fragments: + assert rich_token_count(fragment.contextual_text) <= ceiling + assert tuple(heading.text for heading in fragment.parent_headings) == ( + "Root", + "Deep", + ) + span = fragment.position + assert source[span.start.byte_offset : span.end.byte_offset] == ( + fragment.source_text.encode("utf-8") + ) + assert " ".join(fragment.source_text for fragment in paragraph_fragments) == ( + outcome.canonical_text.split("\n\n", 2)[2].rstrip("\n") + ) + + +def test_oversize_indivisible_code_block_refuses_instead_of_token_splitting() -> None: + ceiling = 32 + code = " ".join(f"operation_{index}()" for index in range(200)) + source = f"# Root\r\n\r\n```python\r\n{code}\r\n```\r\n".encode() + + config = MarkdownCompilerConfig( + version="markdown-config-v3", + token_ceiling=ceiling, + ) + outcome = compile_rich_markdown(source, config) + + assert type(outcome) is CompilationFailure + + +def test_every_emitted_fragment_obeys_the_ceiling() -> None: + ceiling = 12 + source = ( + "# Root\n\n" + " ".join(f"word{index}" for index in range(80)) + "\n" + ).encode() + + config = MarkdownCompilerConfig( + version="markdown-config-v3", + token_ceiling=ceiling, + ) + outcome = compile_rich_markdown(source, config) + + assert type(outcome) is ParsedDocument + assert all( + rich_token_count(fragment.contextual_text) <= ceiling + for fragment in outcome.fragments + ) + + +def test_rich_token_ceiling_is_validated_by_configuration() -> None: + with pytest.raises(ValueError, match="token ceiling"): + MarkdownCompilerConfig(version="markdown-config-v3", token_ceiling=0) + + +def test_ceiling_is_serialized_as_representation_provenance() -> None: + source = b"# Root\n\n" + b"word " * 80 + narrow = compile_rich_markdown( + source, + MarkdownCompilerConfig(version="markdown-config-v3", token_ceiling=16), + ) + wide = compile_rich_markdown( + source, + MarkdownCompilerConfig(version="markdown-config-v3", token_ceiling=64), + ) + + assert type(narrow) is ParsedDocument + assert type(wide) is ParsedDocument + assert narrow.provenance.token_ceiling == 16 + assert wide.provenance.token_ceiling == 64 + assert narrow.provenance != wide.provenance + assert narrow.compilation_digest != wide.compilation_digest diff --git a/tests/unit/test_compiler_runner_determinism.py b/tests/unit/test_compiler_runner_determinism.py new file mode 100644 index 00000000..883c9a1a --- /dev/null +++ b/tests/unit/test_compiler_runner_determinism.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +import base64 +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +import applications.compiler_runner as compiler_runner +from adapters.parsers.ragflow_markdown import compile_rich_markdown +from engine.supply import ( + MarkdownCompilerConfig, + ParsedDocument, + deserialize_parsed_document, +) +from eval._compiler_acceptance import acceptance_context + +FIXTURES = Path(__file__).parents[1] / "fixtures/markdown" +CONFIG = MarkdownCompilerConfig(version="markdown-config-v3") +RICH_FIXTURES = ( + "rich-frontmatter.md", + "rich-headings-lists.md", + "rich-code-tables.md", + "rich-expanded.md", + "rich-inline-html.md", + "rich-mixed-newlines.hex", +) + + +def _source(name: str) -> bytes: + path = FIXTURES / name + if path.suffix == ".hex": + return bytes.fromhex(path.read_text(encoding="ascii")) + return path.read_bytes() + + +@pytest.mark.parametrize("fixture", RICH_FIXTURES) +def test_rich_compilation_digest_is_stable_in_process_and_across_runner( + fixture: str, +) -> None: + source = _source(fixture) + + first = compile_rich_markdown(source, CONFIG) + second = compile_rich_markdown(source, CONFIG) + command = [ + sys.executable, + "-m", + "applications.compiler_runner", + "--compile", + "--config", + CONFIG.version, + ] + first_process = subprocess.run( + command, + input=source, + capture_output=True, + check=True, + timeout=30, + ) + second_process = subprocess.run( + command, + input=source, + capture_output=True, + check=True, + timeout=30, + ) + assert first_process.stdout == second_process.stdout + envelope = json.loads(first_process.stdout) + subprocess_result = deserialize_parsed_document( + base64.b64decode(envelope["document"], validate=True) + ) + + assert type(first) is ParsedDocument + assert type(second) is ParsedDocument + assert type(subprocess_result) is ParsedDocument + assert first.compilation_digest == second.compilation_digest + assert first.compilation_digest == subprocess_result.compilation_digest + assert subprocess_result == first + + +def test_representation_digest_distinguishes_exact_trailing_newline_bytes() -> None: + variants = ( + b"# Exact\n\nBody", + b"# Exact\n\nBody\n", + b"# Exact\r\n\r\nBody\r\n", + b"# Exact\n\nBody\n\n", + ) + + outcomes = tuple(compile_rich_markdown(source, CONFIG) for source in variants) + + assert all(type(outcome) is ParsedDocument for outcome in outcomes) + documents = tuple( + outcome for outcome in outcomes if type(outcome) is ParsedDocument + ) + assert tuple(document.canonical_text.encode("utf-8") for document in documents) == ( + variants + ) + assert len({document.compilation_digest for document in documents}) == len(variants) + + +def test_local_runner_wrapper_cannot_be_substituted_with_a_direct_call( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def direct_call_must_not_run(*args: object, **kwargs: object) -> object: + raise AssertionError("wrapper bypassed its subprocess boundary") + + monkeypatch.setattr( + compiler_runner, + "compile_rich_markdown", + direct_call_must_not_run, + ) + + outcome = compiler_runner.compile_in_local_compiler_runner( + b"# T\n", + CONFIG, + acceptance_context=acceptance_context(), + ) + + assert type(outcome) is ParsedDocument diff --git a/tests/unit/test_compiler_runner_production_boundary.py b/tests/unit/test_compiler_runner_production_boundary.py new file mode 100644 index 00000000..73c9a069 --- /dev/null +++ b/tests/unit/test_compiler_runner_production_boundary.py @@ -0,0 +1,270 @@ +from __future__ import annotations + +import ast +import subprocess +import sys +from pathlib import Path + +import applications.compiler_runner as compiler_runner + +REPOSITORY_ROOT = Path(__file__).parents[2] +PRODUCTION_ROOTS = ("engine", "adapters", "applications") +_IGNORED_MODULES = frozenset( + { + "adapters.parsers.ragflow_markdown", + "applications.compiler_runner", + } +) +_FORBIDDEN_MODULES = frozenset( + { + "adapters.parsers.ragflow_markdown", + "applications.compiler_runner", + "eval._compiler_acceptance", + } +) + + +def _module_name(repository_root: Path, path: Path) -> str: + relative = path.relative_to(repository_root) + parts = relative.with_suffix("").parts + return ".".join(parts[:-1] if parts[-1] == "__init__" else parts) + + +def _resolve_import_module( + module: str, + imported: str | None, + level: int, + *, + is_package: bool, +) -> str: + if level == 0: + return imported or "" + package = module if is_package else module.rsplit(".", maxsplit=1)[0] + parts = package.split(".") if package else [] + retained = parts[: max(0, len(parts) - level + 1)] + imported_parts = imported.split(".") if imported else [] + return ".".join((*retained, *imported_parts)) + + +def _forbidden_imports( + module: str, + tree: ast.Module, + *, + is_package: bool, +) -> frozenset[str]: + violations: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + if any( + alias.name == forbidden + or alias.name.startswith(f"{forbidden}.") + for forbidden in _FORBIDDEN_MODULES + ): + violations.add(alias.name) + elif isinstance(node, ast.ImportFrom): + imported_module = _resolve_import_module( + module, + node.module, + node.level, + is_package=is_package, + ) + for alias in node.names: + qualified = ( + f"{imported_module}.{alias.name}" + if imported_module + else alias.name + ) + if any( + imported_module == forbidden + or imported_module.startswith(f"{forbidden}.") + or qualified == forbidden + or qualified.startswith(f"{forbidden}.") + for forbidden in _FORBIDDEN_MODULES + ): + violations.add(qualified) + return frozenset(violations) + + +def _production_import_violations( + repository_root: Path, + *, + production_roots: tuple[str, ...], + ignored_modules: frozenset[str], +) -> tuple[tuple[str, str], ...]: + violations: set[tuple[str, str]] = set() + for root_name in production_roots: + root = repository_root / root_name + if not root.exists(): + continue + for path in root.rglob("*.py"): + module = _module_name(repository_root, path) + if module in ignored_modules: + continue + tree = ast.parse(path.read_bytes(), filename=str(path)) + relative = path.relative_to(repository_root).as_posix() + violations.update( + (relative, imported) + for imported in _forbidden_imports( + module, + tree, + is_package=path.name == "__init__.py", + ) + ) + return tuple(sorted(violations)) + + +def test_unleased_subprocess_helper_is_explicitly_local_only() -> None: + assert not hasattr(compiler_runner, "compile_in_compiler_runner") + assert hasattr(compiler_runner, "compile_in_local_compiler_runner") + + +def test_no_production_module_imports_an_unleased_compiler_surface() -> None: + assert _production_import_violations( + REPOSITORY_ROOT, + production_roots=PRODUCTION_ROOTS, + ignored_modules=_IGNORED_MODULES, + ) == () + + +def test_direct_production_import_gate_rejects_the_unleased_entry_point( + tmp_path: Path, +) -> None: + (tmp_path / "applications").mkdir() + (tmp_path / "applications/entry.py").write_text( + "from applications.compiler_runner import " + "compile_in_local_compiler_runner\n", + encoding="utf-8", + ) + + assert _production_import_violations( + tmp_path, + production_roots=("applications",), + ignored_modules=frozenset(), + ) == ( + ( + "applications/entry.py", + "applications.compiler_runner.compile_in_local_compiler_runner", + ), + ) + + +def test_production_import_gate_rejects_private_capability_imports( + tmp_path: Path, +) -> None: + (tmp_path / "applications").mkdir() + (tmp_path / "applications/entry.py").write_text( + "from eval import _compiler_acceptance\n", + encoding="utf-8", + ) + + assert _production_import_violations( + tmp_path, + production_roots=("applications",), + ignored_modules=frozenset(), + ) == (("applications/entry.py", "eval._compiler_acceptance"),) + + +def test_production_import_gate_rejects_module_and_submodule_spellings( + tmp_path: Path, +) -> None: + (tmp_path / "applications").mkdir() + (tmp_path / "applications/entry.py").write_text( + "import applications.compiler_runner\n" + "import adapters.parsers.ragflow_markdown.helpers\n", + encoding="utf-8", + ) + + assert _production_import_violations( + tmp_path, + production_roots=("applications",), + ignored_modules=frozenset(), + ) == ( + ("applications/entry.py", "adapters.parsers.ragflow_markdown.helpers"), + ("applications/entry.py", "applications.compiler_runner"), + ) + + +def test_production_import_gate_rejects_live_relative_call_from_package_init( + tmp_path: Path, +) -> None: + package = tmp_path / "adapters/parsers" + package.mkdir(parents=True) + (tmp_path / "adapters/__init__.py").write_text("", encoding="utf-8") + (package / "ragflow_markdown.py").write_text( + "def compile_rich_markdown(source: bytes):\n" + " return source + b' compiled'\n", + encoding="utf-8", + ) + (package / "__init__.py").write_text( + "from .ragflow_markdown import compile_rich_markdown\n" + "\n" + "def production_rich_compile(source: bytes):\n" + " return compile_rich_markdown(source)\n", + encoding="utf-8", + ) + + completed = subprocess.run( + [ + sys.executable, + "-c", + "from adapters.parsers import production_rich_compile; " + "assert production_rich_compile(b'source') == b'source compiled'", + ], + cwd=tmp_path, + check=False, + capture_output=True, + timeout=10, + ) + + assert completed.returncode == 0 + assert _production_import_violations( + tmp_path, + production_roots=("adapters",), + ignored_modules=frozenset(), + ) == ( + ( + "adapters/parsers/__init__.py", + "adapters.parsers.ragflow_markdown.compile_rich_markdown", + ), + ) + + +def test_production_import_gate_resolves_every_relative_package_level( + tmp_path: Path, +) -> None: + parsers = tmp_path / "adapters/parsers" + nested = parsers / "nested" + deeper = nested / "deeper" + deeper.mkdir(parents=True) + (parsers / "__init__.py").write_text( + "from . import ragflow_markdown\n", + encoding="utf-8", + ) + (nested / "__init__.py").write_text( + "from ..ragflow_markdown import compile_rich_markdown\n", + encoding="utf-8", + ) + (deeper / "__init__.py").write_text( + "from ... import ragflow_markdown\n", + encoding="utf-8", + ) + + assert _production_import_violations( + tmp_path, + production_roots=("adapters",), + ignored_modules=frozenset(), + ) == ( + ( + "adapters/parsers/__init__.py", + "adapters.parsers.ragflow_markdown", + ), + ( + "adapters/parsers/nested/__init__.py", + "adapters.parsers.ragflow_markdown.compile_rich_markdown", + ), + ( + "adapters/parsers/nested/deeper/__init__.py", + "adapters.parsers.ragflow_markdown", + ), + ) diff --git a/tests/unit/test_compiler_runner_refusal.py b/tests/unit/test_compiler_runner_refusal.py new file mode 100644 index 00000000..77671b12 --- /dev/null +++ b/tests/unit/test_compiler_runner_refusal.py @@ -0,0 +1,390 @@ +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any, cast + +import pytest + +import applications.compiler_runner as compiler_runner +from adapters.parsers.ragflow_markdown import compile_rich_markdown +from applications.compiler_runner import compile_in_local_compiler_runner +from engine.supply import ( + CompilationFailure, + CompilationFailureCode, + CompiledFragment, + MarkdownCompilerConfig, + ParsedDocument, + UnsupportedConstruct, +) +from eval._compiler_acceptance import _AcceptanceContext, acceptance_context + +CONFIG = MarkdownCompilerConfig(version="markdown-config-v3") + + +def test_unleased_runner_without_acceptance_context_is_a_typed_refusal() -> None: + unchecked_runner = cast(Any, compile_in_local_compiler_runner) + outcome = unchecked_runner(b"# T\n", CONFIG) + + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE + assert outcome.position is None + + +def test_explicit_acceptance_context_allows_the_local_runner() -> None: + outcome = compile_in_local_compiler_runner( + b"# T\n", + CONFIG, + acceptance_context=acceptance_context(), + ) + + assert type(outcome) is ParsedDocument + + +def test_acceptance_context_cannot_be_forged() -> None: + with pytest.raises(TypeError, match="cannot be constructed"): + _AcceptanceContext(object()) + + outcome = compile_in_local_compiler_runner( + b"# T\n", + CONFIG, + acceptance_context=cast(Any, object()), + ) + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE + + +@pytest.mark.parametrize( + ("source", "expected_code"), + [ + ( + b"# Heading\n\n```python\nprint('open')\n", + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + ), + (b"# Heading\n\n\xed\xa0\x80\n", CompilationFailureCode.INVALID_UTF8), + ( + b"# Heading\n\ncontains\x00nul\n", + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + ), + ( + b"```text\ncontains\x07bell\n```\n", + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + ), + ( + b"```text\ncontains\x1bescape\n```\n", + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + ), + ( + b"```text\ncontains\x1fseparator\n```\n", + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + ), + (b"# Heading\n\ntruncated \xe4\xb8", CompilationFailureCode.INVALID_UTF8), + ( + b"# Heading\n\n\n", + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + ), + ], +) +def test_malformed_source_returns_typed_failure_across_runner_boundary( + source: bytes, + expected_code: CompilationFailureCode, +) -> None: + outcome = compile_in_local_compiler_runner( + source, + CONFIG, + acceptance_context=acceptance_context(), + ) + + assert type(outcome) is CompilationFailure + assert outcome.code is expected_code + assert outcome.position is not None + + +@pytest.mark.parametrize( + ("source", "construct"), + ( + (b"# T\n\n&\n", UnsupportedConstruct.ENTITY), + (b"# T\n\nescaped\\*text\n", UnsupportedConstruct.ESCAPE), + (b"# T\n\n indented code\n", UnsupportedConstruct.CODE_BLOCK), + (b"# T\n\n> &\n", UnsupportedConstruct.ENTITY), + (b"# T\n\n- item\n &\n", UnsupportedConstruct.ENTITY), + ), +) +def test_unlisted_inline_constructs_are_typed_refusals( + source: bytes, + construct: UnsupportedConstruct, +) -> None: + outcome = compile_in_local_compiler_runner( + source, + CONFIG, + acceptance_context=acceptance_context(), + ) + + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_CONSTRUCT + assert outcome.construct is construct + + +def test_unlisted_construct_inside_atomic_ragged_table_is_typed_refusal() -> None: + source = ( + b"| A | B |\n" + b"| --- | --- |\n" + b"| x | & |\n" + b"| ragged |\n" + ) + + outcome = compile_rich_markdown(source, CONFIG) + + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_CONSTRUCT + assert outcome.construct is UnsupportedConstruct.ENTITY + + +def test_runner_boundary_converts_unexpected_compiler_exception_to_typed_failure( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + def raise_unexpected( + source: bytes, + config: MarkdownCompilerConfig, + ) -> object: + raise ValueError("internal parser defect") + + monkeypatch.setattr(compiler_runner, "compile_rich_markdown", raise_unexpected) + + compiler_runner._emit( + b"# T\n", + CONFIG, + acceptance_context=acceptance_context(), + ) + + outcome = compiler_runner._failure_from_document( + json.loads(capsys.readouterr().out)["failure"] + ) + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE + assert outcome.position is None + + +def test_runner_api_converts_child_process_failure_to_typed_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + subprocess, + "run", + lambda *args, **kwargs: subprocess.CompletedProcess( + args=args, + returncode=1, + stdout=b"", + stderr=b"internal parser defect", + ), + ) + + outcome = compile_in_local_compiler_runner( + b"# T\n", + CONFIG, + acceptance_context=acceptance_context(), + ) + + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE + assert outcome.position is None + + +def test_runner_api_passes_a_bound_and_converts_timeout_to_typed_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + observed: list[float] = [] + + def wedge(*args: object, timeout: float, **kwargs: object) -> object: + observed.append(timeout) + raise subprocess.TimeoutExpired(cmd="wedged compiler", timeout=timeout) + + monkeypatch.setattr(subprocess, "run", wedge) + + outcome = compile_in_local_compiler_runner( + b"# T\n", + CONFIG, + acceptance_context=acceptance_context(), + ) + + assert observed and observed[0] > 0 + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE + assert outcome.position is None + + +def test_runner_api_terminates_a_deliberately_wedged_child( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + executable = tmp_path / "wedged-compiler" + executable.write_text( + "#!/usr/bin/env python3\nimport time\ntime.sleep(60)\n", + encoding="utf-8", + ) + executable.chmod(0o700) + monkeypatch.setattr( + compiler_runner, + "sys", + type("Sys", (), {"executable": str(executable)}), + ) + monkeypatch.setattr(compiler_runner, "COMPILER_RUNNER_TIMEOUT_SECONDS", 0.05) + + outcome = compile_in_local_compiler_runner( + b"# T\n", + CONFIG, + acceptance_context=acceptance_context(), + ) + + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE + assert outcome.position is None + + +def test_direct_compiler_converts_domain_constructor_rejection_to_typed_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def reject_constructor(*args: object, **kwargs: object) -> ParsedDocument: + raise ValueError("domain constructor rejected parser metadata") + + monkeypatch.setattr( + ParsedDocument, + "rich_v3", + classmethod(reject_constructor), + ) + + outcome = compile_rich_markdown(b"# T\n", CONFIG) + + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE + assert outcome.position is not None + + +def test_direct_compiler_converts_section_constructor_rejection_to_typed_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def reject_section(*args: object, **kwargs: object) -> object: + raise ValueError("section constructor rejected parser metadata") + + monkeypatch.setattr( + "adapters.parsers.ragflow_markdown._section", + reject_section, + ) + + outcome = compile_rich_markdown(b"# T\n", CONFIG) + + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE + assert outcome.position is not None + + +def test_direct_compiler_converts_unexpected_section_exception_to_typed_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def reject_section(*args: object, **kwargs: object) -> object: + raise RuntimeError("unexpected section defect") + + monkeypatch.setattr( + "adapters.parsers.ragflow_markdown._section", + reject_section, + ) + + outcome = compile_rich_markdown(b"# T\n", CONFIG) + + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE + assert outcome.position is not None + + +def test_direct_compiler_converts_unexpected_domain_exception_to_typed_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def reject_constructor(*args: object, **kwargs: object) -> ParsedDocument: + raise RuntimeError("unexpected domain defect") + + monkeypatch.setattr( + ParsedDocument, + "rich_v3", + classmethod(reject_constructor), + ) + + outcome = compile_rich_markdown(b"# T\n", CONFIG) + + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE + assert outcome.position is not None + + +def test_direct_compiler_converts_unexpected_fragment_exception_to_typed_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def reject_fragment(*args: object, **kwargs: object) -> CompiledFragment: + raise RuntimeError("unexpected Fragment defect") + + monkeypatch.setattr( + "adapters.parsers.ragflow_markdown.CompiledFragment", + reject_fragment, + ) + + outcome = compile_rich_markdown(b"# T\n", CONFIG) + + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE + assert outcome.position is not None + + +def test_runner_api_converts_unexpected_deserializer_exception_to_typed_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + subprocess, + "run", + lambda *args, **kwargs: subprocess.CompletedProcess( + args=args, + returncode=0, + stdout=json.dumps( + {"outcome": "parsed", "document": "e30="} + ).encode(), + stderr=b"", + ), + ) + + def reject_document(payload: bytes) -> ParsedDocument: + raise RuntimeError("unexpected deserializer defect") + + monkeypatch.setattr( + compiler_runner, + "deserialize_parsed_document", + reject_document, + ) + + outcome = compile_in_local_compiler_runner( + b"# T\n", + CONFIG, + acceptance_context=acceptance_context(), + ) + + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE + assert outcome.position is None + + +def test_direct_compiler_converts_parser_helper_rejection_to_typed_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def reject_parser_helper(*args: object, **kwargs: object) -> object: + raise ValueError("vendored parser helper rejected source") + + monkeypatch.setattr( + "third_party.ragflow.deepdoc.parser.markdown_parser." + "MarkdownElementExtractor._get_fence_marker", + reject_parser_helper, + ) + + outcome = compile_rich_markdown(b"Body\n", CONFIG) + + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE + assert outcome.position is not None diff --git a/tests/unit/test_compiler_runner_tables.py b/tests/unit/test_compiler_runner_tables.py new file mode 100644 index 00000000..ac05346b --- /dev/null +++ b/tests/unit/test_compiler_runner_tables.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +import pytest + +from adapters.parsers.ragflow_markdown import compile_rich_markdown +from engine.supply import ( + CompilationFailure, + CompilationFailureCode, + MarkdownCompilerConfig, + ParsedDocument, + SectionKind, +) + +CONFIG = MarkdownCompilerConfig(version="markdown-config-v3") + + +def test_every_table_fragment_carries_a_round_tripping_source_span() -> None: + tables = "\n\n".join( + ( + f"| Key | Value |\n| --- | --- |\n" + f"| item-{index} | value-{index} |" + ) + for index in range(40) + ) + source = f"# Tables\n\n{tables}\n".encode() + + outcome = compile_rich_markdown(source, CONFIG) + + assert type(outcome) is ParsedDocument + table_fragments = tuple( + fragment + for fragment in outcome.fragments + if fragment.kind is SectionKind.TABLE + ) + assert len(table_fragments) == 40 + canonical = outcome.canonical_text.encode("utf-8") + for fragment in table_fragments: + span = fragment.position + assert canonical[span.start.byte_offset : span.end.byte_offset].decode( + "utf-8" + ) == fragment.source_text + + +def test_table_spans_round_trip_against_original_crlf_bytes() -> None: + source = ( + b"# Tables\r\n\r\n" + b"| Key | Value |\r\n" + b"| --- | --- |\r\n" + b"| alpha | ready |\r\n" + ) + + outcome = compile_rich_markdown(source, CONFIG) + + assert type(outcome) is ParsedDocument + table = next( + fragment + for fragment in outcome.fragments + if fragment.kind is SectionKind.TABLE + ) + span = table.position + assert source[span.start.byte_offset : span.end.byte_offset] == ( + table.source_text.encode("utf-8") + ) + assert b"\r\n" in table.source_text.encode("utf-8") + + +def test_table_with_trailing_whitespace_compiles_and_retains_exact_span() -> None: + source = ( + b"# Tables\n\n" + b"| Key | Value |\n" + b"| --- | --- |\n" + b"| alpha | ready | \n" + ) + + outcome = compile_rich_markdown(source, CONFIG) + + assert type(outcome) is ParsedDocument + table = next( + fragment + for fragment in outcome.fragments + if fragment.kind is SectionKind.TABLE + ) + span = table.position + assert source[span.start.byte_offset : span.end.byte_offset] == ( + table.source_text.encode("utf-8") + ) + assert table.source_text.endswith(" ") + + +@pytest.mark.parametrize( + "source", + ( + b"| A | B |\n| --- | --- |\n| x | y |\n| ragged |\n", + b"| A | B |\n| --- | --- |\n| x | y |\n| | |\n", + b"| A | B |\n| --- | --- | --- |\n| x | y |\n", + ), +) +def test_ragged_or_empty_row_keeps_table_as_one_exact_atomic_block( + source: bytes, +) -> None: + + outcome = compile_rich_markdown(source, CONFIG) + + assert type(outcome) is ParsedDocument + assert len(outcome.fragments) == 1 + fragment = outcome.fragments[0] + assert fragment.kind is SectionKind.PARAGRAPH + span = fragment.position + assert source[span.start.byte_offset : span.end.byte_offset] == source.rstrip(b"\n") + assert fragment.source_text.encode("utf-8") == source.rstrip(b"\n") + + +def test_oversize_ragged_table_refuses_instead_of_splitting_atomic_source() -> None: + source = ( + b"| A | B |\n" + b"| --- | --- |\n" + b"| one two three | four five six |\n" + b"| ragged seven eight |\n" + ) + bounded_config = MarkdownCompilerConfig( + version="markdown-config-v3", + token_ceiling=8, + ) + + outcome = compile_rich_markdown(source, bounded_config) + + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE + assert outcome.construct is None diff --git a/tests/unit/test_embedding_benchmark_report_privacy.py b/tests/unit/test_embedding_benchmark_report_privacy.py index 6def5f95..0d5290db 100644 --- a/tests/unit/test_embedding_benchmark_report_privacy.py +++ b/tests/unit/test_embedding_benchmark_report_privacy.py @@ -6,7 +6,9 @@ REPORT_PATH = Path("docs/evaluation/2026-07-29-embedding-benchmark.json") FORBIDDEN_KEYS = frozenset({"excerpt", "path", "query", "text", "title"}) -PERSONAL_PATH = re.compile(r"(?:/Users/|[A-Za-z]:\\|\.md(?:\b|$))") +PERSONAL_PATH = re.compile( + r"(?:/" + "Users" + r"/|[A-Za-z]:\\|\.md(?:\b|$))" +) def _keys(value: object) -> set[str]: diff --git a/tests/unit/test_security_gate_cli_privacy.py b/tests/unit/test_security_gate_cli_privacy.py new file mode 100644 index 00000000..cba41c73 --- /dev/null +++ b/tests/unit/test_security_gate_cli_privacy.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import logging +import os +import re +import sys +from pathlib import Path + +import pytest + +from scripts.run_m0_security_gate import main as security_gate_main +from scripts.security_gate.runner import _execute_pytest + +_ABSOLUTE_PATHS = ( + re.compile(r"(? None: + assert _SYNTHETIC_PRIVATE_ROOT not in output + assert all(pattern.search(output) is None for pattern in _ABSOLUTE_PATHS) + + +def test_gate_cli_scrubs_complete_output_across_all_channels( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + absolute_root = f"/private-machine/{_SYNTHETIC_PRIVATE_ROOT}" + + def noisy_gate(_paths: object) -> dict[str, str]: + print(f"stdout evidence at {absolute_root}/stdout.json") + print(f"stderr evidence at {absolute_root}/stderr.json", file=sys.stderr) + logger = logging.getLogger("synthetic-security-gate-output") + logger.addHandler(logging.StreamHandler(sys.stderr)) + logger.warning("log evidence at %s/log.json", absolute_root) + return { + "m0SecurityDecision": "pass", + "releaseDecision": "not-evaluated", + } + + monkeypatch.setattr("scripts.run_m0_security_gate.run_gate", noisy_gate) + + assert security_gate_main(["--output-dir", absolute_root]) == 0 + captured = capsys.readouterr() + _assert_private_safe_output(captured.out + captured.err) + + +def test_gate_child_process_output_is_scrubbed_before_forwarding( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + absolute_root = f"/private-machine/{_SYNTHETIC_PRIVATE_ROOT}" + program = ( + "import sys; " + f"print({str(absolute_root + '/stdout.json')!r}); " + f"print({str(absolute_root + '/stderr.json')!r}, file=sys.stderr)" + ) + + exit_code = _execute_pytest( + (sys.executable, "-c", program), + cwd=tmp_path, + env=os.environ, + ) + + assert exit_code == 0 + captured = capsys.readouterr() + _assert_private_safe_output(captured.out + captured.err) diff --git a/tests/unit/test_third_party_ragflow_registration.py b/tests/unit/test_third_party_ragflow_registration.py new file mode 100644 index 00000000..06b78b5f --- /dev/null +++ b/tests/unit/test_third_party_ragflow_registration.py @@ -0,0 +1,217 @@ +from __future__ import annotations + +import ast +import hashlib +import json +import re +import tomllib +from collections import Counter +from pathlib import Path + +import pytest + +from adapters.parsers.ragflow_markdown import compile_rich_markdown +from engine.supply import MarkdownCompilerConfig, ParsedDocument +from third_party.ragflow.deepdoc.parser.markdown_parser import ( + MarkdownElementExtractor, +) + +REPOSITORY_ROOT = Path(__file__).parents[2] +REGISTRATION_ROOT = REPOSITORY_ROOT / "third_party/ragflow" +REGISTRATION_PATH = REGISTRATION_ROOT / "UPSTREAM.toml" +REQUIRED_EXCLUSIONS = { + "deepdoc/parser/__init__.py", + "rag/app/naive.py", + "rag/nlp", +} +ALLOWED_IMPORT_ROOTS = { + "__future__", + "argparse", + "collections", + "dataclasses", + "enum", + "hashlib", + "html", + "json", + "logging", + "markdown", + "pathlib", + "re", + "sys", + "typing", + "unicodedata", +} + + +def _registration() -> dict[str, object]: + return tomllib.loads(REGISTRATION_PATH.read_text(encoding="utf-8")) + + +def test_vendored_bytes_match_complete_pinned_registration() -> None: + registration = _registration() + + assert registration["repository"] == "https://github.com/infiniflow/ragflow.git" + commit = registration["commit"] + assert isinstance(commit, str) + assert re.fullmatch(r"[0-9a-f]{40}", commit) + assert commit == "4391e03886b996201f3b8818f671b19eb24d0f7b" + assert registration["reuse_mode"] == "copy-patch" + assert registration["approval"] == ( + "https://github.com/stone16/context-engine/issues/124" + ) + assert registration["source_paths"] == ["deepdoc/parser/markdown_parser.py"] + assert registration["nested_dependencies"] == [ + { + "name": "Python-Markdown", + "version": "3.6", + "license": "BSD-3-Clause", + "license_path": "third_party/ragflow/LICENSE.python-markdown", + } + ] + excluded_paths = registration["excluded_paths"] + assert isinstance(excluded_paths, list) + assert set(excluded_paths) >= REQUIRED_EXCLUSIONS + + files = registration["files"] + assert isinstance(files, list) + assert files + registered_paths: set[Path] = set() + for entry in files: + assert isinstance(entry, dict) + assert set(entry) == {"upstream_path", "vendored_path", "sha256"} + upstream_path = entry["upstream_path"] + vendored_path = entry["vendored_path"] + expected_hash = entry["sha256"] + assert isinstance(upstream_path, str) and upstream_path + assert isinstance(vendored_path, str) and vendored_path + assert isinstance(expected_hash, str) + assert re.fullmatch(r"[0-9a-f]{64}", expected_hash) + path = REPOSITORY_ROOT / vendored_path + path.relative_to(REGISTRATION_ROOT) + assert path.is_file() + assert hashlib.sha256(path.read_bytes()).hexdigest() == expected_hash + registered_paths.add(path) + + vendored_files = { + path + for path in REGISTRATION_ROOT.rglob("*") + if path.is_file() + and "__pycache__" not in path.relative_to(REGISTRATION_ROOT).parts + and path.name + not in { + "LICENSE.python-markdown", + "LICENSE.upstream", + "MODIFICATIONS.md", + "UPSTREAM.toml", + "sbom.cyclonedx.json", + } + and "patches" not in path.relative_to(REGISTRATION_ROOT).parts + } + assert registered_paths == vendored_files + assert (REGISTRATION_ROOT / "LICENSE.upstream").is_file() + assert hashlib.sha256( + (REGISTRATION_ROOT / "LICENSE.python-markdown").read_bytes() + ).hexdigest() == "7ba4eb6d10b32b2d11dce13821340351cdbbb30ba8ccc67841db2ffd86e79aca" + assert (REGISTRATION_ROOT / "MODIFICATIONS.md").is_file() + assert (REGISTRATION_ROOT / "patches").is_dir() + assert (REPOSITORY_ROOT / "THIRD_PARTY_NOTICES.md").is_file() + sbom = json.loads( + (REGISTRATION_ROOT / "sbom.cyclonedx.json").read_text(encoding="utf-8") + ) + assert sbom["bomFormat"] == "CycloneDX" + assert sbom["metadata"]["component"]["bom-ref"] == ( + "context-engine:third-party:ragflow" + ) + assert { + (property_value["name"], property_value["value"]) + for property_value in sbom["metadata"]["properties"] + } >= { + ("context-engine:sbom:scope", "third_party/ragflow"), + ("context-engine:sbom:artifact-wide", "false"), + } + assert {component["name"] for component in sbom["components"]} == { + "Python-Markdown", + "RAGFlow Markdown parser", + } + + +def test_vendored_subtree_imports_only_approved_dependencies() -> None: + registration = _registration() + files = registration["files"] + assert isinstance(files, list) + + imports: set[str] = set() + for entry in files: + assert isinstance(entry, dict) + vendored_path = entry["vendored_path"] + assert isinstance(vendored_path, str) + path = REPOSITORY_ROOT / vendored_path + if path.suffix != ".py": + continue + tree = ast.parse(path.read_bytes(), filename=str(path)) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imports.update(alias.name.partition(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.level == 0: + assert node.module is not None + imports.add(node.module.partition(".")[0]) + + assert imports <= ALLOWED_IMPORT_ROOTS + modifications = (REGISTRATION_ROOT / "MODIFICATIONS.md").read_text( + encoding="utf-8" + ) + assert "Python-Markdown" in modifications + assert "BSD 3-Clause" in modifications + + +def test_registered_parser_region_is_executed_by_the_ce_adapter( + monkeypatch: pytest.MonkeyPatch, +) -> None: + adapter = REPOSITORY_ROOT / "adapters/parsers/ragflow_markdown.py" + tree = ast.parse(adapter.read_bytes(), filename=str(adapter)) + + imports = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) + and node.module == "third_party.ragflow.deepdoc.parser.markdown_parser" + ] + assert len(imports) == 1 + assert [alias.name for alias in imports[0].names] == [ + "MarkdownElementExtractor" + ] + helper_names = ( + "_get_fence_marker", + "_is_closing_fence", + "_is_table_row", + "_is_table_separator_row", + "_table_cells", + ) + calls: Counter[str] = Counter() + for helper_name in helper_names: + original = getattr(MarkdownElementExtractor, helper_name) + + def recording_helper( + self: MarkdownElementExtractor, + *args: object, + _helper_name: str = helper_name, + _original: object = original, + ) -> object: + calls[_helper_name] += 1 + assert callable(_original) + return _original(self, *args) + + monkeypatch.setattr(MarkdownElementExtractor, helper_name, recording_helper) + + source = ( + b"# Executed\n\n" + b"```python\nprint('registered')\n```\n\n" + b"| Key | Value |\n| --- | --- |\n| parser | called |\n" + ) + outcome = compile_rich_markdown( + source, + MarkdownCompilerConfig("markdown-config-v3"), + ) + + assert type(outcome) is ParsedDocument + assert all(calls[name] > 0 for name in helper_names) diff --git a/third_party/ragflow/LICENSE.python-markdown b/third_party/ragflow/LICENSE.python-markdown new file mode 100644 index 00000000..6249d60c --- /dev/null +++ b/third_party/ragflow/LICENSE.python-markdown @@ -0,0 +1,30 @@ +BSD 3-Clause License + +Copyright 2007, 2008 The Python Markdown Project (v. 1.7 and later) +Copyright 2004, 2005, 2006 Yuri Takhteyev (v. 0.2-1.6b) +Copyright 2004 Manfred Stienstra (the original version) + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/third_party/ragflow/LICENSE.upstream b/third_party/ragflow/LICENSE.upstream new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/third_party/ragflow/LICENSE.upstream @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/third_party/ragflow/MODIFICATIONS.md b/third_party/ragflow/MODIFICATIONS.md new file mode 100644 index 00000000..c70cb868 --- /dev/null +++ b/third_party/ragflow/MODIFICATIONS.md @@ -0,0 +1,39 @@ +# RAGFlow Markdown parser registration + +## Pinned source + +The registered region is `deepdoc/parser/markdown_parser.py` from RAGFlow at +commit `4391e03886b996201f3b8818f671b19eb24d0f7b`. The exact path carries an +Apache-2.0 header and is covered by the repository-root Apache-2.0 license, +reproduced verbatim as `LICENSE.upstream`. The pinned upstream tree contains no +root `NOTICE` file. + +## Nested notice scan + +The copied file imports only Python standard-library modules and +Python-Markdown. Python-Markdown is licensed under the BSD 3-Clause License. +Its verbatim license for the pinned 3.6 dependency is retained as +`LICENSE.python-markdown`. No other nested third-party dependency is imported +by the copied region. + +## Executed reuse and ContextEngine-owned behavior + +The exact upstream file is copied and executed: the ContextEngine adapter +constructs its `MarkdownElementExtractor` and calls the upstream fence-marker, +closing-fence, table-row, table-separator, and table-cell recognition methods. +No RAGFlow package initializer or `rag/nlp` dependency carrier is copied or +imported. + +The broader compilation pipeline is ContextEngine-owned. It implements the +closed rich grammar, raw-input UTF-8 spans, heading ancestry, hard bounds, +versioned deterministic output, and typed refusal because the upstream parser +does not return the existing ContextEngine contracts and lacks exact byte-span +and hard-bound semantics. This is intentionally narrow reuse of one verified +Apache-2.0 file, not a claim that the entire upstream parser pipeline executes. + +`sbom.cyclonedx.json` is the deterministic component inventory for the copied +parser and its sole nested dependency. Both wheel and source distribution +artifacts include it alongside the applicable license texts. + +The empty `patches/` directory records that no textual source patch applies: +the current integration wraps and directly executes selected copied helpers. diff --git a/third_party/ragflow/UPSTREAM.toml b/third_party/ragflow/UPSTREAM.toml new file mode 100644 index 00000000..7cf5ab0f --- /dev/null +++ b/third_party/ragflow/UPSTREAM.toml @@ -0,0 +1,18 @@ +repository = "https://github.com/infiniflow/ragflow.git" +commit = "4391e03886b996201f3b8818f671b19eb24d0f7b" +source_paths = ["deepdoc/parser/markdown_parser.py"] +excluded_paths = [ + "deepdoc/parser/__init__.py", + "rag/app/naive.py", + "rag/nlp", +] +reuse_mode = "copy-patch" +approval = "https://github.com/stone16/context-engine/issues/124" +nested_dependencies = [ + { name = "Python-Markdown", version = "3.6", license = "BSD-3-Clause", license_path = "third_party/ragflow/LICENSE.python-markdown" }, +] + +[[files]] +upstream_path = "deepdoc/parser/markdown_parser.py" +vendored_path = "third_party/ragflow/deepdoc/parser/markdown_parser.py" +sha256 = "94c8e2515d05e141fcf65e10336ceca7f9116e54b31a668637ba3f901943cb66" diff --git a/third_party/ragflow/deepdoc/parser/markdown_parser.py b/third_party/ragflow/deepdoc/parser/markdown_parser.py new file mode 100644 index 00000000..583e4ffd --- /dev/null +++ b/third_party/ragflow/deepdoc/parser/markdown_parser.py @@ -0,0 +1,527 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2025 The InfiniFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import re + +from markdown import markdown + + +class RAGFlowMarkdownParser: + def __init__(self, chunk_token_num=128): + self.chunk_token_num = int(chunk_token_num) + + def extract_tables_and_remainder(self, markdown_text, separate_tables=True): + tables = [] + working_text = markdown_text + + def replace_tables_with_rendered_html(pattern, table_list, render=True): + new_text = "" + last_end = 0 + for match in pattern.finditer(working_text): + raw_table = match.group() + table_list.append(raw_table) + if separate_tables: + # Skip this match (i.e., remove it) + new_text += working_text[last_end : match.start()] + "\n\n" + else: + # Replace with rendered HTML + html_table = markdown(raw_table, extensions=["markdown.extensions.tables"]) if render else raw_table + new_text += working_text[last_end : match.start()] + html_table + "\n\n" + last_end = match.end() + new_text += working_text[last_end:] + return new_text + + if "|" in markdown_text: # for optimize performance + # Standard Markdown table + border_table_pattern = re.compile( + r""" + (?:\n|^) + (?:\|.*?\|.*?\|.*?\n) + (?:\|(?:\s*[:-]+[-| :]*\s*)\|.*?\n) + (?:\|.*?\|.*?\|.*?\n)+ + """, + re.VERBOSE, + ) + working_text = replace_tables_with_rendered_html(border_table_pattern, tables, render=separate_tables) + + # Borderless Markdown table + no_border_table_pattern = re.compile( + r""" + (?:\n|^) + (?:\S.*?\|.*?\n) + (?:(?:\s*[:-]+[-| :]*\s*).*?\n) + (?:\S.*?\|.*?\n)+ + """, + re.VERBOSE, + ) + working_text = replace_tables_with_rendered_html(no_border_table_pattern, tables, render=separate_tables) + + # Replace any TAGS e.g. to
+ TAGS = ["table", "td", "tr", "th", "tbody", "thead", "div"] + table_with_attributes_pattern = re.compile(rf"<(?:{'|'.join(TAGS)})[^>]*>", re.IGNORECASE) + + def replace_tag(m): + tag_name = re.match(r"<(\w+)", m.group()).group(1) + return "<{}>".format(tag_name) + + working_text = re.sub(table_with_attributes_pattern, replace_tag, working_text) + + if "
" in working_text.lower(): # for optimize performance + # HTML table extraction - handle possible html/body wrapper tags + html_table_pattern = re.compile( + r""" + (?:\n|^) + \s* + (?: + # case1:
...
+ (?:]*>\s*]*>\s*]*>.*?\s*\s*) + | + # case2: ...
+ (?:]*>\s*]*>.*?\s*) + | + # case3: only...
+ (?:]*>.*?) + ) + \s* + (?=\n|$) + """, + re.VERBOSE | re.DOTALL | re.IGNORECASE, + ) + + def replace_html_tables(): + nonlocal working_text + new_text = "" + last_end = 0 + for match in html_table_pattern.finditer(working_text): + raw_table = match.group() + tables.append(raw_table) + if separate_tables: + new_text += working_text[last_end : match.start()] + "\n\n" + else: + new_text += working_text[last_end : match.start()] + raw_table + "\n\n" + last_end = match.end() + new_text += working_text[last_end:] + working_text = new_text + + replace_html_tables() + + return working_text, tables + + +class MarkdownElementExtractor: + def __init__(self, markdown_content): + self.markdown_content = markdown_content + self.lines = markdown_content.split("\n") + + def get_delimiters(self, delimiters): + toks = re.findall(r"`([^`]+)`", delimiters) + toks = sorted(set(toks), key=lambda x: -len(x)) + return "|".join(re.escape(t) for t in toks if t) + + def _get_fence_marker(self, line): + match = re.match(r"^[ \t]{0,3}(?P`{3,}|~{3,})(?:.*)$", line) + if not match: + return None + fence = match.group("fence") + return fence[0], len(fence) + + def _is_closing_fence(self, line, fence_char, fence_len): + pattern = r"^[ \t]{0,3}" + re.escape(fence_char) + r"{" + str(fence_len) + r",}\s*$" + return re.match(pattern, line) is not None + + def _line_start_offsets(self, text): + offsets = [] + offset = 0 + for line in self.lines: + offsets.append(offset) + offset += len(line) + 1 + return offsets + + def _fenced_code_ranges(self, text): + ranges = [] + line_offsets = self._line_start_offsets(text) + + i = 0 + while i < len(self.lines): + marker = self._get_fence_marker(self.lines[i]) + if not marker: + i += 1 + continue + + fence_char, fence_len = marker + start_pos = line_offsets[i] + end_line = len(self.lines) - 1 + for j in range(i + 1, len(self.lines)): + if self._is_closing_fence(self.lines[j], fence_char, fence_len): + end_line = j + break + + end_pos = min(len(text), line_offsets[end_line] + len(self.lines[end_line])) + ranges.append((start_pos, end_pos)) + i = end_line + 1 + + return ranges + + def _table_cells(self, line): + stripped = line.strip() + if "|" not in stripped: + return [] + if stripped.startswith("|"): + stripped = stripped[1:] + if stripped.endswith("|"): + stripped = stripped[:-1] + return [cell.strip() for cell in stripped.split("|")] + + def _is_table_row(self, line): + cells = self._table_cells(line) + return len(cells) >= 2 and any(cell for cell in cells) + + def _is_table_separator_row(self, line): + cells = self._table_cells(line) + return len(cells) >= 2 and all(re.match(r"^:?-+:?$", cell.replace(" ", "")) for cell in cells) + + def _markdown_table_ranges(self, text): + ranges = [] + line_offsets = self._line_start_offsets(text) + + i = 0 + while i < len(self.lines) - 1: + if not self._is_table_row(self.lines[i]) or not self._is_table_separator_row(self.lines[i + 1]): + i += 1 + continue + + end_line = i + 1 + j = i + 2 + while j < len(self.lines) and self._is_table_row(self.lines[j]): + end_line = j + j += 1 + + end_pos = min(len(text), line_offsets[end_line] + len(self.lines[end_line])) + ranges.append((line_offsets[i], end_pos)) + i = end_line + 1 + + return ranges + + def _html_table_ranges(self, text): + table_pattern = re.compile( + r""" + (?: + (?:]*>\s*]*>\s*]*>.*?\s*\s*) + | + (?:]*>\s*]*>.*?\s*) + | + (?:]*>.*?) + ) + """, + re.VERBOSE | re.DOTALL | re.IGNORECASE, + ) + return [(match.start(), match.end()) for match in table_pattern.finditer(text)] + + def _merge_ranges(self, ranges): + if not ranges: + return [] + + merged = [] + for start, end in sorted(ranges): + if not merged or start > merged[-1][1]: + merged.append((start, end)) + else: + merged[-1] = (merged[-1][0], max(merged[-1][1], end)) + return merged + + def _protected_ranges(self, text): + return self._merge_ranges(self._fenced_code_ranges(text) + self._markdown_table_ranges(text) + self._html_table_ranges(text)) + + def _append_delimited_section(self, sections, text, start, end, include_meta): + part = text[start:end] + if not part or not part.strip(): + return + if include_meta: + sections.append( + { + "content": part.strip(), + "start_line": text.count("\n", 0, start), + "end_line": text.count("\n", 0, end), + } + ) + else: + sections.append(part.strip()) + + def _extract_delimited_elements(self, text, delimiters, include_meta=False): + sections = [] + pattern = re.compile(delimiters) + protected_ranges = self._protected_ranges(text) + if protected_ranges: + logging.debug("markdown_parser: detected %d protected ranges for delimiter extraction", len(protected_ranges)) + protected_idx = 0 + last_end = 0 + + for match in pattern.finditer(text): + while protected_idx < len(protected_ranges) and protected_ranges[protected_idx][1] <= match.start(): + protected_idx += 1 + + if protected_idx < len(protected_ranges): + start, end = protected_ranges[protected_idx] + if start <= match.start() < end: + logging.debug( + "markdown_parser: skipped delimiter match at pos=%d delimiter=%r inside fenced range %s", + match.start(), + match.group(), + (start, end), + ) + continue + + self._append_delimited_section(sections, text, last_end, match.start(), include_meta) + last_end = match.end() + + self._append_delimited_section(sections, text, last_end, len(text), include_meta) + return sections + + def extract_elements(self, delimiter=None, include_meta=False): + """Extract individual elements (headers, code blocks, lists, etc.)""" + sections = [] + + i = 0 + dels = "" + if delimiter: + dels = self.get_delimiters(delimiter) + if len(dels) > 0: + text = "\n".join(self.lines) + sections = self._extract_delimited_elements(text, dels, include_meta) + + # Attach lone header lines to the section that follows them so that + # "## Title\n" never becomes an isolated chunk when the delimiter + # splits at every newline. A header is "lone" when it occupies a + # single line (no embedded newline after stripping). + def _is_lone_header(section_content): + stripped = section_content.strip() + return bool(re.match(r"^#{1,6}\s+\S", stripped)) and "\n" not in stripped + + def _is_attachable_body(section_content): + """True when the following chunk is prose body, not code/table/list/etc.""" + stripped = section_content.strip() + if not stripped: + return False + first_line = stripped.split("\n", 1)[0] + if self._get_fence_marker(first_line): + return False + if first_line.lstrip().startswith("|"): + return False + if re.match(r"^\S+\s*\|", first_line): + return False + if first_line.lstrip().startswith("<"): + return False + if re.match(r"^\s*[-*+]\s+", first_line) or re.match(r"^\s*\d+\.\s+", first_line): + return False + if first_line.lstrip().startswith(">"): + return False + return True + + merged = [] + merged_header_count = 0 + i = 0 + while i < len(sections): + content = sections[i]["content"] if include_meta else sections[i] + if _is_lone_header(content): + header_parts = [content.strip()] + j = i + 1 + while j < len(sections): + next_content = sections[j]["content"] if include_meta else sections[j] + if not _is_lone_header(next_content): + break + header_parts.append(next_content.strip()) + j += 1 + if j < len(sections): + body_content = sections[j]["content"] if include_meta else sections[j] + if _is_attachable_body(body_content): + combined = "\n".join(header_parts) + "\n" + body_content + if include_meta: + merged.append( + { + **sections[i], + "content": combined, + "end_line": sections[j]["end_line"], + } + ) + else: + merged.append(combined) + merged_header_count += len(header_parts) + i = j + 1 + continue + for k in range(i, j): + merged.append(sections[k]) + i = j + continue + merged.append(sections[i]) + i += 1 + if merged_header_count: + logging.debug( + "markdown_parser: merged %d lone header line(s) into following sections", + merged_header_count, + ) + return merged + while i < len(self.lines): + line = self.lines[i] + + if re.match(r"^#{1,6}\s+.*$", line): + # header + element = self._extract_header(i) + sections.append(element if include_meta else element["content"]) + i = element["end_line"] + 1 + elif self._get_fence_marker(line): + # code block + element = self._extract_code_block(i) + sections.append(element if include_meta else element["content"]) + i = element["end_line"] + 1 + elif re.match(r"^\s*[-*+]\s+.*$", line) or re.match(r"^\s*\d+\.\s+.*$", line): + # list block + element = self._extract_list_block(i) + sections.append(element if include_meta else element["content"]) + i = element["end_line"] + 1 + elif line.strip().startswith(">"): + # blockquote + element = self._extract_blockquote(i) + sections.append(element if include_meta else element["content"]) + i = element["end_line"] + 1 + elif line.strip(): + # text block (paragraphs and inline elements until next block element) + element = self._extract_text_block(i) + sections.append(element if include_meta else element["content"]) + i = element["end_line"] + 1 + else: + i += 1 + + if include_meta: + sections = [section for section in sections if section["content"].strip()] + else: + sections = [section for section in sections if section.strip()] + return sections + + def _extract_header(self, start_pos): + return { + "type": "header", + "content": self.lines[start_pos], + "start_line": start_pos, + "end_line": start_pos, + } + + def _extract_code_block(self, start_pos): + end_pos = start_pos + content_lines = [self.lines[start_pos]] + fence_char, fence_len = self._get_fence_marker(self.lines[start_pos]) + + # Find the end of the code block + for i in range(start_pos + 1, len(self.lines)): + content_lines.append(self.lines[i]) + end_pos = i + if self._is_closing_fence(self.lines[i], fence_char, fence_len): + break + + return { + "type": "code_block", + "content": "\n".join(content_lines), + "start_line": start_pos, + "end_line": end_pos, + } + + def _extract_list_block(self, start_pos): + end_pos = start_pos + content_lines = [] + + i = start_pos + while i < len(self.lines): + line = self.lines[i] + # check if this line is a list item or continuation of a list + if ( + re.match(r"^\s*[-*+]\s+.*$", line) + or re.match(r"^\s*\d+\.\s+.*$", line) + or (i > start_pos and not line.strip()) + or (i > start_pos and re.match(r"^\s{2,}[-*+]\s+.*$", line)) + or (i > start_pos and re.match(r"^\s{2,}\d+\.\s+.*$", line)) + or (i > start_pos and re.match(r"^\s+\w+.*$", line)) + ): + content_lines.append(line) + end_pos = i + i += 1 + else: + break + + return { + "type": "list_block", + "content": "\n".join(content_lines), + "start_line": start_pos, + "end_line": end_pos, + } + + def _extract_blockquote(self, start_pos): + end_pos = start_pos + content_lines = [] + + i = start_pos + while i < len(self.lines): + line = self.lines[i] + if line.strip().startswith(">") or (i > start_pos and not line.strip()): + content_lines.append(line) + end_pos = i + i += 1 + else: + break + + return { + "type": "blockquote", + "content": "\n".join(content_lines), + "start_line": start_pos, + "end_line": end_pos, + } + + def _extract_text_block(self, start_pos): + """Extract a text block (paragraphs, inline elements) until next block element""" + end_pos = start_pos + content_lines = [self.lines[start_pos]] + + i = start_pos + 1 + while i < len(self.lines): + line = self.lines[i] + # stop if we encounter a block element + if re.match(r"^#{1,6}\s+.*$", line) or self._get_fence_marker(line) or re.match(r"^\s*[-*+]\s+.*$", line) or re.match(r"^\s*\d+\.\s+.*$", line) or line.strip().startswith(">"): + break + elif not line.strip(): + # check if the next line is a block element + if i + 1 < len(self.lines) and ( + re.match(r"^#{1,6}\s+.*$", self.lines[i + 1]) + or self._get_fence_marker(self.lines[i + 1]) + or re.match(r"^\s*[-*+]\s+.*$", self.lines[i + 1]) + or re.match(r"^\s*\d+\.\s+.*$", self.lines[i + 1]) + or self.lines[i + 1].strip().startswith(">") + ): + break + else: + content_lines.append(line) + end_pos = i + i += 1 + else: + content_lines.append(line) + end_pos = i + i += 1 + + return { + "type": "text_block", + "content": "\n".join(content_lines), + "start_line": start_pos, + "end_line": end_pos, + } diff --git a/third_party/ragflow/patches/.gitkeep b/third_party/ragflow/patches/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/third_party/ragflow/sbom.cyclonedx.json b/third_party/ragflow/sbom.cyclonedx.json new file mode 100644 index 00000000..011d5c3c --- /dev/null +++ b/third_party/ragflow/sbom.cyclonedx.json @@ -0,0 +1,50 @@ +{ + "bomFormat": "CycloneDX", + "metadata": { + "component": { + "bom-ref": "context-engine:third-party:ragflow", + "name": "ContextEngine registered RAGFlow subtree", + "type": "library" + }, + "properties": [ + { + "name": "context-engine:sbom:scope", + "value": "third_party/ragflow" + }, + { + "name": "context-engine:sbom:artifact-wide", + "value": "false" + } + ] + }, + "components": [ + { + "licenses": [ + { + "license": { + "id": "Apache-2.0" + } + } + ], + "name": "RAGFlow Markdown parser", + "purl": "pkg:github/infiniflow/ragflow@4391e03886b996201f3b8818f671b19eb24d0f7b#deepdoc/parser/markdown_parser.py", + "type": "library", + "version": "4391e03886b996201f3b8818f671b19eb24d0f7b" + }, + { + "licenses": [ + { + "license": { + "id": "BSD-3-Clause" + } + } + ], + "name": "Python-Markdown", + "purl": "pkg:pypi/markdown@3.6", + "type": "library", + "version": "3.6" + } + ], + "specVersion": "1.6", + "version": 1 +} diff --git a/uv.lock b/uv.lock index b0aade86..15b6150b 100644 --- a/uv.lock +++ b/uv.lock @@ -119,6 +119,7 @@ dependencies = [ { name = "cryptography" }, { name = "fastapi" }, { name = "jsonschema" }, + { name = "markdown" }, { name = "psycopg", extra = ["binary"] }, { name = "pydantic" }, { name = "rfc8785" }, @@ -148,6 +149,7 @@ requires-dist = [ { name = "cryptography", specifier = ">=49,<50" }, { name = "fastapi", specifier = ">=0.116,<0.117" }, { name = "jsonschema", specifier = ">=4.25,<5" }, + { name = "markdown", specifier = ">=3.6,<3.7" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.2,<3.3" }, { name = "pydantic", specifier = ">=2.13,<2.14" }, { name = "rfc8785", specifier = ">=0.1.4,<0.2" }, @@ -502,6 +504,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bc/b1/a0ec7a5a9db730a08daef1fdfb8090435b82465abbf758a596f0ea88727e/mako-1.3.12-py3-none-any.whl", hash = "sha256:8f61569480282dbf557145ce441e4ba888be453c30989f879f0d652e39f53ea9", size = 78521, upload-time = "2026-04-28T19:01:10.393Z" }, ] +[[package]] +name = "markdown" +version = "3.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/02/4785861427848cc11e452cc62bb541006a1087cf04a1de83aedd5530b948/Markdown-3.6.tar.gz", hash = "sha256:ed4f41f6daecbeeb96e576ce414c41d2d876daa9a16cb35fa8ed8c2ddfad0224", size = 354715, upload-time = "2024-03-14T15:37:59.775Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/b3/0c0c994fe49cd661084f8d5dc06562af53818cc0abefaca35bdc894577c3/Markdown-3.6-py3-none-any.whl", hash = "sha256:48f276f4d8cfb8ce6527c8f79e2ee29708508bf4d40aa410fbc3b4ee832c850f", size = 105381, upload-time = "2024-03-14T15:37:57.457Z" }, +] + [[package]] name = "markdown-it-py" version = "4.2.0"