diff --git a/STATUS.md b/STATUS.md index 38e36140..60c636e5 100644 --- a/STATUS.md +++ b/STATUS.md @@ -104,7 +104,7 @@ content. This activates no pi consumer, MCP Adapter, `Continue`, dogfood | ADR | Activates | |---|---| -| [0090](./docs/decisions/0090-admit-a-co-resident-local-evidence-console.md), [0093](./docs/decisions/0093-activate-leased-rich-markdown-and-revision-link-graph.md) | Explicitly authenticated server-rendered loopback UI, private File citation reopening, and separately Control-authorized source/import/Article jobs through schema-hidden typed HTTP carriers while OpenAPI v0 remains frozen; link-bearing imports receive a content-free handoff to the activated File scan and exact leased worker path | +| [0090](./docs/decisions/0090-admit-a-co-resident-local-evidence-console.md), [0093](./docs/decisions/0093-activate-leased-rich-markdown-and-revision-link-graph.md) | Explicitly authenticated server-rendered loopback UI, private File citation reopening, and separately Control-authorized source/import/Article jobs through schema-hidden typed HTTP carriers while OpenAPI v0 remains frozen; imports satisfying ADR-0093's closed v1-refusal and whole-document rich-syntax checks receive a content-free handoff to the activated File scan and exact leased worker path | Feedback persists through the current Runtime identity and exact ContextRun binding, with no Control or release-publication authority. Numeric Hit Test scores diff --git a/adapters/http/ui_api.py b/adapters/http/ui_api.py index c48d563c..4eee2f41 100644 --- a/adapters/http/ui_api.py +++ b/adapters/http/ui_api.py @@ -47,7 +47,7 @@ MarkdownCompilerConfig, ParsedDocument, UnsupportedConstruct, - contains_rich_markdown_link, + contains_only_accepted_rich_markdown_inline, ) _PREVIEW_TTL: Final = timedelta(minutes=10) @@ -65,15 +65,24 @@ _MAX_SIGNED_BIGINT: Final = (1 << 63) - 1 -def _contains_rich_markdown_link(source: bytes) -> bool: +def _decode_rich_markdown(source: bytes) -> str | None: try: - decoded = source.removeprefix(b"\xef\xbb\xbf").decode( + return source.removeprefix(b"\xef\xbb\xbf").decode( "utf-8", errors="strict", ) except UnicodeDecodeError: + return None + + +def _contains_only_accepted_rich_markdown_inline( + source: bytes, + construct: UnsupportedConstruct, +) -> bool: + decoded = _decode_rich_markdown(source) + if decoded is None: return False - return contains_rich_markdown_link(decoded) + return contains_only_accepted_rich_markdown_inline(decoded, construct) class UiApiUnavailable(RuntimeError): @@ -806,14 +815,22 @@ def preview_import( ) except (LookupError, RuntimeError, TypeError, ValueError): raise UiApiUnavailable from None - requires_scan_handoff = ( - ( - type(outcome) is CompilationFailure - and outcome.code is CompilationFailureCode.UNSUPPORTED_CONSTRUCT - and outcome.construct is UnsupportedConstruct.LINK_OR_IMAGE - ) - or _contains_rich_markdown_link(raw) + requires_scan_handoff = _contains_only_accepted_rich_markdown_inline( + raw, + UnsupportedConstruct.LINK_OR_IMAGE, ) + if ( + type(outcome) is CompilationFailure + and outcome.code is CompilationFailureCode.UNSUPPORTED_CONSTRUCT + and outcome.construct is not None + ): + requires_scan_handoff = ( + requires_scan_handoff + or _contains_only_accepted_rich_markdown_inline( + raw, + outcome.construct, + ) + ) if requires_scan_handoff: source_arguments = ( "--organization-id " diff --git a/adapters/parsers/ragflow_markdown.py b/adapters/parsers/ragflow_markdown.py index 15b358fc..aeba21f5 100644 --- a/adapters/parsers/ragflow_markdown.py +++ b/adapters/parsers/ragflow_markdown.py @@ -25,6 +25,7 @@ StructuralPath, UnsupportedConstruct, is_markdown_control_character, + rich_markdown_token_count, unsupported_rich_markdown_inline, ) from third_party.ragflow.deepdoc.parser.markdown_parser import MarkdownElementExtractor @@ -83,9 +84,7 @@ def _is_table_separator_row(self, line: str) -> bool: ... def rich_token_count(value: str) -> int: """Count deterministic representation tokens for the v3 hard bound.""" - if type(value) is not str: - raise TypeError("rich Markdown token counting requires exact text") - return sum(1 for _ in _TOKEN.finditer(value)) + return rich_markdown_token_count(value) def _failure( diff --git a/docs/decisions/0093-activate-leased-rich-markdown-and-revision-link-graph.md b/docs/decisions/0093-activate-leased-rich-markdown-and-revision-link-graph.md index 839d527f..c1b9247a 100644 --- a/docs/decisions/0093-activate-leased-rich-markdown-and-revision-link-graph.md +++ b/docs/decisions/0093-activate-leased-rich-markdown-and-revision-link-graph.md @@ -1,6 +1,6 @@ --- name: adr-0093-activate-leased-rich-markdown-and-revision-link-graph -version: "1.0.1" +version: "1.0.3" description: > Activate rich Markdown v3 behind the exact File-import WorkerLease, persist immutable content-free Revision link edges, and admit one authorized graph @@ -43,13 +43,15 @@ ADR-0075 requires. are not reinterpreted or backfilled. The co-resident local evidence console's exact preview flow remains pinned to v1 because it has no durable import job or WorkerLease redemption authority with which to select the v3 child. Issue - #203 ships the compatibility resolution: when v1's closed refusal is exactly - `LINK_OR_IMAGE`, or the exact source contains any accepted rich link syntax - not classified as such by v1, the console returns a content-free actionable - handoff to the existing source `scan` plus independent worker-dispatch path; - every other compilation refusal remains generically unavailable. The worker - honors a redeemed, exact v1 preview binding only for the console's successful - v1 preview/confirm flow; all scan-scheduled imports use active v3. + #203 and #207 ship the compatibility resolution: when v1's closed refusal is + exactly `LINK_OR_IMAGE`, `EMPHASIS`, `INLINE_CODE`, or `STRIKETHROUGH` and the + exact source matches that accepted v3 inline syntax, or when the source + contains accepted rich link syntax not classified as such by v1, the console + returns a content-free actionable handoff to the existing source `scan` plus + independent worker-dispatch path. Malformed syntax and every other compilation + refusal remain generically unavailable. The worker honors a redeemed, exact + v1 preview binding only for the console's successful v1 preview/confirm flow; + all scan-scheduled imports use active v3. 2. A File import redeems and durably verifies its exact WorkerLease before selecting the rich compiler subprocess. The child is a pure transform that receives source bytes, the closed configuration version, and token ceiling @@ -139,8 +141,9 @@ an operating-system sandbox. ## Consequences -- Rich link-bearing File notes can now publish without changing v1/v2 bytes or - historical Revision meaning. +- File notes whose frozen-v1 preview outcome satisfies the closed accepted + rich-link, emphasis, inline-code, or strikethrough handoff checks can now + publish without changing v1/v2 bytes or historical Revision meaning. - Outgoing links and backlinks are reproducible from immutable v3 Revision lineage, but the graph itself grants no access and exposes no content. - A denied neighbour is indistinguishable from an absent or irrelevant diff --git a/engine/supply/__init__.py b/engine/supply/__init__.py index ab154601..cf68a8c5 100644 --- a/engine/supply/__init__.py +++ b/engine/supply/__init__.py @@ -91,6 +91,8 @@ StructuralPath, UnsupportedConstruct, canonicalize_parsed_document, + contains_accepted_rich_markdown_construct, + contains_only_accepted_rich_markdown_inline, contains_rich_markdown_link, deserialize_parsed_document, ) @@ -175,6 +177,8 @@ "WorkerLeaseToken", "generate_worker_lease_nonce", "canonicalize_parsed_document", + "contains_accepted_rich_markdown_construct", + "contains_only_accepted_rich_markdown_inline", "contains_rich_markdown_link", "deserialize_parsed_document", "worker_lease_digest", diff --git a/engine/supply/markdown.py b/engine/supply/markdown.py index fa4aef94..fbc7cda6 100644 --- a/engine/supply/markdown.py +++ b/engine/supply/markdown.py @@ -28,6 +28,7 @@ _COMPILATION_DIGEST_DOMAIN: Final = b"context-engine.markdown-compilation.v2\x00" _COMPILATION_DIGEST_V3_DOMAIN: Final = b"context-engine.markdown-compilation.v3\x00" _MAX_VERSION_LENGTH: Final = 128 +_RICH_TOKEN_PATTERN: Final = re.compile(r"\S+") def is_markdown_control_character(character: str) -> bool: @@ -41,6 +42,14 @@ def is_markdown_control_character(character: str) -> bool: ) +def rich_markdown_token_count(value: str) -> int: + """Count deterministic representation tokens for the v3 hard bound.""" + + if type(value) is not str: + raise TypeError("rich Markdown token counting requires exact text") + return sum(1 for _ in _RICH_TOKEN_PATTERN.finditer(value)) + + def _require_version(value: object) -> str: if ( type(value) is not str @@ -585,6 +594,33 @@ class UnsupportedConstruct(StrEnum): r"th|td|p|ul|ol|li)\b[^>]*>", re.IGNORECASE, ) +_ACCEPTED_RICH_MARKDOWN_CONSTRUCT_PATTERNS: Final = { + UnsupportedConstruct.EMPHASIS: (_EMPHASIS_PATTERN,), + UnsupportedConstruct.INLINE_CODE: (_RICH_INLINE_CODE_PATTERN,), + UnsupportedConstruct.LINK_OR_IMAGE: ( + _RICH_WIKILINK_PATTERN, + _RICH_AUTOLINK_PATTERN, + _RICH_INLINE_LINK_PATTERN, + _RICH_REFERENCE_LINK_PATTERN, + ), + UnsupportedConstruct.STRIKETHROUGH: (_RICH_STRIKETHROUGH_PATTERN,), +} + + +def contains_accepted_rich_markdown_construct( + source: str, + construct: UnsupportedConstruct, +) -> bool: + """Return whether exact text contains the named accepted rich syntax.""" + + if type(source) is not str: + raise TypeError("rich Markdown construct detection requires exact text") + if type(construct) is not UnsupportedConstruct: + raise TypeError("rich Markdown construct detection requires a closed construct") + return any( + pattern.search(source) is not None + for pattern in _ACCEPTED_RICH_MARKDOWN_CONSTRUCT_PATTERNS.get(construct, ()) + ) def contains_rich_markdown_link(source: str) -> bool: @@ -592,14 +628,9 @@ def contains_rich_markdown_link(source: str) -> bool: if type(source) is not str: raise TypeError("rich Markdown link detection requires exact text") - return any( - pattern.search(source) is not None - for pattern in ( - _RICH_WIKILINK_PATTERN, - _RICH_AUTOLINK_PATTERN, - _RICH_INLINE_LINK_PATTERN, - _RICH_REFERENCE_LINK_PATTERN, - ) + return contains_accepted_rich_markdown_construct( + source, + UnsupportedConstruct.LINK_OR_IMAGE, ) @@ -678,6 +709,143 @@ def unsupported_rich_markdown_inline(line: str) -> UnsupportedConstruct | None: return None if construct is UnsupportedConstruct.LIST else construct +def contains_only_accepted_rich_markdown_inline( + source: str, + construct: UnsupportedConstruct, +) -> bool: + """Return whether one accepted construct has no malformed inline peer.""" + + if type(source) is not str: + raise TypeError("rich Markdown inline validation requires exact text") + if not contains_accepted_rich_markdown_construct(source, construct): + return False + if any(is_markdown_control_character(character) for character in source): + return False + fence: str | None = None + fence_body_has_content = False + list_open = False + previous_line: str | None = None + lines = source.splitlines() + html_block_end = -1 + for index, line in enumerate(lines): + if index <= html_block_end: + previous_line = line + continue + marker = _RICH_FENCE_PATTERN.match(line) + if ( + fence is None + and list_open + and line.startswith((" ", "\t")) + and marker is not None + ): + return False + if marker is not None: + candidate = marker.group("fence") + if fence is None: + fence = candidate + language = line.lstrip()[len(candidate) :].strip() + if len(language) > MARKDOWN_CODE_LANGUAGE_MAX_LENGTH or any( + character.isspace() for character in language + ): + return False + fence_body_has_content = False + elif candidate[0] == fence[0] and len(candidate) >= len(fence): + if not fence_body_has_content: + return False + fence = None + previous_line = line + continue + if fence is not None: + fence_body_has_content = fence_body_has_content or bool(line.strip()) + previous_line = line + continue + if not line.strip(): + list_open = False + previous_line = line + continue + if _RICH_SETEXT_PATTERN.fullmatch(line) is not None: + if ( + previous_line is not None + and previous_line.strip() + and unsupported_rich_markdown_inline(previous_line.strip()) is not None + ): + return False + list_open = False + previous_line = line + continue + if _THEMATIC_BREAK_PATTERN.fullmatch(line) is not None: + list_open = False + previous_line = line + continue + list_item = _RICH_LIST_ITEM_PATTERN.fullmatch(line) + if list_open and line.startswith((" ", "\t")) and list_item is None: + if unsupported_rich_markdown_inline(line.lstrip()) is not None: + return False + previous_line = line + continue + list_open = list_item is not None + stripped = line.strip() + if stripped.startswith("<"): + if _RICH_ANGLE_LITERAL_PATTERN.fullmatch(stripped) is not None: + continue + html_block_end = index + while ( + html_block_end + 1 < len(lines) + and lines[html_block_end + 1].strip() + ): + html_block_end += 1 + html_block_lines = lines[index : html_block_end + 1] + if not _has_closed_rich_html_block( + line, + "\n".join(html_block_lines), + ): + return False + if any( + unsupported_rich_markdown_inline(candidate) is not None + for candidate in html_block_lines[1:] + ): + return False + html_block_source = "\n".join(html_block_lines) + table_ranges = _rich_table_source_ranges(html_block_source) + if table_ranges and table_ranges != ((0, len(html_block_source)),): + return False + previous_line = line + continue + inspected = _rich_markdown_inline_payload(line) + if unsupported_rich_markdown_inline(inspected) is not None: + return False + previous_line = line + if fence is not None: + return False + try: + _expected_rich_fragment_layout(source, MARKDOWN_RICH_TOKEN_CEILING) + except ValueError: + return False + return True + + +def _rich_markdown_inline_payload(line: str) -> str: + inspected = line + if (heading := _RICH_ATX_HEADING_PATTERN.fullmatch(line)) is not None: + inspected = heading.group(2).strip() + elif (item := _RICH_LIST_ITEM_PATTERN.fullmatch(line)) is not None: + inspected = item.group(2) + elif line.lstrip().startswith(">"): + inspected = line.lstrip()[1:].lstrip() + if inspected.startswith("[!"): + inspected = _RICH_FOOTNOTE_PATTERN.sub("x", inspected, count=1) + return inspected + + +def _has_closed_rich_html_block(line: str, source: str) -> bool: + html_open = _RICH_HTML_OPEN_PATTERN.match(line) + return html_open is not None and re.search( + rf"", + source, + re.IGNORECASE, + ) is not None + + @dataclass(frozen=True, slots=True) class CompilationFailure: """Typed all-or-nothing failure; it never carries partial ParsedDocument data.""" @@ -1243,7 +1411,7 @@ def _expected_rich_fragment_layout( ancestry = "\n\n".join( f"{'#' * level} {text}" for level, text in headings ) - capacity = token_ceiling - len(re.findall(r"\S+", ancestry)) + capacity = token_ceiling - rich_markdown_token_count(ancestry) indivisible = block.indivisible or block.kind in { SectionKind.HEADING, SectionKind.LIST, @@ -1252,7 +1420,7 @@ def _expected_rich_fragment_layout( } ranges: tuple[tuple[int, int], ...] if indivisible: - if len(re.findall(r"\S+", source)) > capacity: + if rich_markdown_token_count(source) > capacity: raise ValueError("rich indivisible source exceeds its ceiling") ranges = ((0, len(source)),) else: @@ -1356,15 +1524,7 @@ def _validate_rich_closed_grammar(section: ParsedSection, source: str) -> None: return if len(lines) >= 2 and lines[0] == "---" and lines[-1] == "---": return - html_open = _RICH_HTML_OPEN_PATTERN.match(lines[0]) if lines else None - if ( - html_open is not None - and re.search( - rf"", - source, - re.IGNORECASE, - ) - ): + if lines and _has_closed_rich_html_block(lines[0], source): return if ( section.kind is SectionKind.PARAGRAPH diff --git a/tests/integration/test_import_preview_requires_confirm.py b/tests/integration/test_import_preview_requires_confirm.py index 5e8b8200..61da3006 100644 --- a/tests/integration/test_import_preview_requires_confirm.py +++ b/tests/integration/test_import_preview_requires_confirm.py @@ -23,7 +23,11 @@ PostgreSQLWorkerLeaseIssuer, create_database_engine, ) -from engine.supply import MarkdownCompilerConfig, ParsedDocument +from engine.supply import ( + MARKDOWN_RICH_TOKEN_CEILING, + MarkdownCompilerConfig, + ParsedDocument, +) from tests.support.file_imports import ( NOW, FileImportScenario, @@ -76,7 +80,7 @@ def _ui_import_scenario( migration_engine = create_database_engine(migration_configuration) roots = FileRootRegistry( {scenario.root_ref: scenario.root}, - limits=FileReadLimits(max_file_bytes=4096), + limits=FileReadLimits(max_file_bytes=8192), ) try: with migration_engine.connect() as connection: @@ -134,9 +138,16 @@ def _ui_import_scenario( b"# Handbook\n\nEmbed ![[private-runbook]].\n", b"# Handbook\n\nRead .\n", b"# Handbook\n\nFirst paragraph.\n\nRead [[private-runbook]].\n", + b"# Handbook\n\n- Read [[private-runbook]].\n", + b"# Handbook\n\nOnly *emphasis*.\n", + b"# Handbook\n\nOnly `inline code`.\n", + b"# Handbook\n\nOnly ~~strikethrough~~.\n", + b"# Handbook\n\n*Accepted*\n\n
\n", + b"# Handbook\n\nOnly *emphasis*.\n\n---\n", + b"*Emphasized heading*\n---\n", ], ) -def test_link_bearing_import_preview_hands_off_to_the_leased_scan_path( +def test_v3_only_import_preview_hands_off_to_the_leased_scan_path( tmp_path: Path, migration_configuration: DatabaseConfiguration, guarded_control_engine: Engine, @@ -213,18 +224,43 @@ def test_link_bearing_import_preview_hands_off_to_the_leased_scan_path( assert published_count == 0 +@pytest.mark.parametrize( + "payload", + [ + b"# Handbook\n\n[[private-runbook]]\xffprivate malformed body\n", + b"# Handbook\n\nOnly `private malformed body.\n", + b"# Handbook\n\nOnly ~~private malformed body.\n", + b"# Handbook\n\n*Accepted* plus ~~private malformed body.\n", + b"# Handbook\n\n`Accepted` plus ~~private malformed body.\n", + b"# Handbook\n\n~~Accepted~~ plus `private malformed body.\n", + b"# Handbook\n\n[Accepted](note.md) plus ~~private malformed body.\n", + b"# Handbook\n\n[[Accepted]] plus `private malformed body.\n", + b"# Handbook\n\n- [Accepted](note.md) plus `private malformed body.\n", + b"# Handbook\n\n> [[Accepted]] plus `private malformed body.\n", + b"# Handbook\n\n## [Accepted](note.md) plus `private malformed body.\n", + ( + b"# Handbook\n\n[Accepted](note.md)\n\n```" + + b"x" * 65 + + b"\nbody\n```\n" + ), + b"# Handbook\n\n[Accepted](note.md)\n\n```text\n\n```\n", + b"# Handbook\n\n*Accepted*\n\nunclosed\n", + ], +) def test_malformed_import_refusal_stays_content_free_without_scan_handoff( tmp_path: Path, migration_configuration: DatabaseConfiguration, guarded_control_engine: Engine, guarded_runtime_engine: Engine, + payload: bytes, ) -> None: with _ui_import_scenario( tmp_path=tmp_path, migration_configuration=migration_configuration, guarded_control_engine=guarded_control_engine, guarded_runtime_engine=guarded_runtime_engine, - payload=b"# Handbook\n\n[[private-runbook]]\xffprivate malformed body\n", + payload=payload, ) as (scenario, client, _migration_engine): response = client.post( "/ui/import/preview", @@ -244,6 +280,69 @@ def test_malformed_import_refusal_stays_content_free_without_scan_handoff( assert CONTROL_TOKEN not in response.text +@pytest.mark.parametrize( + "payload", + [ + b"# Handbook\n\n*Accepted*\n\n- item one\n > quoted\n", + b"# Handbook\n\n*Accepted*\n\n- item one\n # heading\n", + b"# Handbook\n\n*Accepted*\n\n- item one\n ```text\n body\n ```\n", + b"# Handbook\n\n*Accepted*\n\n> Quoted\n---\n", + ( + b"# Handbook\n\n[Accepted](note.md)\n\n
body
\n" + b"| A | B |\n| --- | --- |\n| x | y |\n" + ), + b"# Handbook\n\n*Accepted*\n\n
body
\n---\n##\n", + b"# Handbook\n\n[Accepted](note.md)\n\n\n
unclosed\n", + ( + b"# Handbook\n\n[Accepted](note.md)\n\n
unclosed\n\n" + b"```html\n
\n```\n" + ), + b'# Handbook\n\n[Accepted](note.md)\n\n
body
\n', + b"# Handbook\n\n[Accepted](note.md)\n\n\n", + ( + b"# Handbook\n\n[Accepted](note.md)\n\n- " + + b"x " * (MARKDOWN_RICH_TOKEN_CEILING + 1) + + b"\n" + ), + ( + b"# Handbook\n\n[Accepted](note.md)\n\n| A | B |\n| --- | --- |\n| " + + b"x " * (MARKDOWN_RICH_TOKEN_CEILING + 1) + + b"| y |\n" + ), + ], +) +def test_v3_refused_block_construct_stays_content_free_without_scan_handoff( + tmp_path: Path, + migration_configuration: DatabaseConfiguration, + guarded_control_engine: Engine, + guarded_runtime_engine: Engine, + payload: bytes, +) -> None: + with _ui_import_scenario( + tmp_path=tmp_path, + migration_configuration=migration_configuration, + guarded_control_engine=guarded_control_engine, + guarded_runtime_engine=guarded_runtime_engine, + payload=payload, + ) as (scenario, client, _migration_engine): + response = client.post( + "/ui/import/preview", + content=( + f"sourceRef={scenario.source_ref.value}&path=handbook.md&" + f"controlCredential={CONTROL_TOKEN}" + ), + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + + assert response.status_code == 503 + assert "Request refused" in response.text + assert "provider_unavailable" in response.text + assert "Rich Markdown requires the leased scan path" not in response.text + assert "context-engine-control scan" not in response.text + assert "previewToken" not in response.text + assert CONTROL_TOKEN not in response.text + + def test_import_preview_requires_confirm( tmp_path: Path, migration_configuration: DatabaseConfiguration, diff --git a/tests/unit/test_compiler_runner_acceptance.py b/tests/unit/test_compiler_runner_acceptance.py index 3797efe6..993e866a 100644 --- a/tests/unit/test_compiler_runner_acceptance.py +++ b/tests/unit/test_compiler_runner_acceptance.py @@ -10,6 +10,7 @@ MARKDOWN_COMPILER_V3_VERSION, MARKDOWN_RICH_CANONICALIZATION_PROFILE, MARKDOWN_RICH_COMPILATION_DIGEST_PROFILE, + MARKDOWN_RICH_TOKEN_CEILING, CompilationFailure, CompilationProvenance, CompiledFragment, @@ -20,6 +21,8 @@ SourcePoint, SourceSpan, StructuralPath, + UnsupportedConstruct, + contains_only_accepted_rich_markdown_inline, ) FIXTURES = Path(__file__).parents[1] / "fixtures/markdown" @@ -50,6 +53,125 @@ def test_tracked_rich_construct_corpus_compiles_all_or_nothing(fixture: str) -> assert outcome.fragments +@pytest.mark.parametrize( + ("source", "construct"), + ( + ( + b"# Handbook\n\nRead [the private note](private-runbook.md).\n", + UnsupportedConstruct.LINK_OR_IMAGE, + ), + ( + b"# Handbook\n\nRead [[private-runbook]].\n", + UnsupportedConstruct.LINK_OR_IMAGE, + ), + ( + b"# Handbook\n\nEmbed ![[private-runbook]].\n", + UnsupportedConstruct.LINK_OR_IMAGE, + ), + ( + b"# Handbook\n\nRead .\n", + UnsupportedConstruct.LINK_OR_IMAGE, + ), + ( + b"# Handbook\n\nFirst paragraph.\n\nRead [[private-runbook]].\n", + UnsupportedConstruct.LINK_OR_IMAGE, + ), + ( + b"# Handbook\n\n- Read [[private-runbook]].\n", + UnsupportedConstruct.LINK_OR_IMAGE, + ), + ( + b"# Handbook\n\nOnly *emphasis*.\n", + UnsupportedConstruct.EMPHASIS, + ), + ( + b"# Handbook\n\nOnly `inline code`.\n", + UnsupportedConstruct.INLINE_CODE, + ), + ( + b"# Handbook\n\nOnly ~~strikethrough~~.\n", + UnsupportedConstruct.STRIKETHROUGH, + ), + ( + b"# Handbook\n\n*Accepted*\n\n
\n", + UnsupportedConstruct.EMPHASIS, + ), + ( + b"# Handbook\n\nOnly *emphasis*.\n\n---\n", + UnsupportedConstruct.EMPHASIS, + ), + ( + b"*Emphasized heading*\n---\n", + UnsupportedConstruct.EMPHASIS, + ), + ), +) +def test_preview_handoff_predicate_implies_rich_compiler_acceptance( + source: bytes, + construct: UnsupportedConstruct, +) -> None: + assert contains_only_accepted_rich_markdown_inline( + source.decode("utf-8"), + construct, + ) + assert type(compile_rich_markdown(source, CONFIG)) is ParsedDocument + + +@pytest.mark.parametrize( + ("source", "construct"), + ( + ( + b"# Handbook\n\n[Accepted](note.md)\n\n
body
\n" + b"| A | B |\n| --- | --- |\n| x | y |\n", + UnsupportedConstruct.LINK_OR_IMAGE, + ), + ( + b"# Handbook\n\n*Accepted*\n\n
body
\n---\n##\n", + UnsupportedConstruct.EMPHASIS, + ), + ( + b"# Handbook\n\n[Accepted](note.md)\n\n
\n
unclosed\n", + UnsupportedConstruct.LINK_OR_IMAGE, + ), + ( + b"# Handbook\n\n[Accepted](note.md)\n\n
unclosed\n\n" + b"```html\n
\n```\n", + UnsupportedConstruct.LINK_OR_IMAGE, + ), + ( + b'# Handbook\n\n[Accepted](note.md)\n\n
body
\n', + UnsupportedConstruct.LINK_OR_IMAGE, + ), + ( + b"# Handbook\n\n[Accepted](note.md)\n\n\n", + UnsupportedConstruct.LINK_OR_IMAGE, + ), + ( + b"# Handbook\n\n[Accepted](note.md)\n\n- " + + b"x " * (MARKDOWN_RICH_TOKEN_CEILING + 1) + + b"\n", + UnsupportedConstruct.LINK_OR_IMAGE, + ), + ( + b"# Handbook\n\n[Accepted](note.md)\n\n| A | B |\n| --- | --- |\n| " + + b"x " * (MARKDOWN_RICH_TOKEN_CEILING + 1) + + b"| y |\n", + UnsupportedConstruct.LINK_OR_IMAGE, + ), + ), +) +def test_preview_handoff_predicate_rejects_v3_refused_document_shape( + source: bytes, + construct: UnsupportedConstruct, +) -> None: + + assert not contains_only_accepted_rich_markdown_inline( + source.decode("utf-8"), + construct, + ) + assert type(compile_rich_markdown(source, CONFIG)) is CompilationFailure + + @pytest.mark.parametrize("fixture", RICH_FIXTURES) def test_every_fragment_span_round_trips_to_exact_original_utf8( fixture: str, @@ -786,6 +908,50 @@ def test_rich_constructor_rejects_unlisted_construct_in_nested_context( ) +@pytest.mark.parametrize( + ("original", "forged_source", "section_text"), + ( + (b"- item one\n ordinary\n", "- item one\n > quoted", None), + (b"- item one\n continued\n", "- item one\n # heading", None), + (b"Plainxxx\n---\n", "> Quoted\n---", "> Quoted"), + ), +) +def test_rich_constructor_rejects_block_construct_hidden_by_section_kind( + original: bytes, + forged_source: str, + section_text: str | None, +) -> None: + compiled = compile_rich_markdown(original, CONFIG) + assert type(compiled) is ParsedDocument + section = compiled.sections[0] + fragment = compiled.fragments[0] + assert len(forged_source.encode()) == fragment.position.end.byte_offset + + with pytest.raises(ValueError, match="closed grammar"): + ParsedDocument.rich_v3( + canonical_text=f"{forged_source}\n", + sections=( + replace( + section, + text=forged_source if section_text is None else section_text, + ), + ), + fragments=( + replace( + fragment, + source_text=forged_source, + contextual_text=forged_source, + search_phrases=( + (forged_source,) + if section_text is None + else (forged_source, section_text) + ), + ), + ), + provenance=compiled.provenance, + ) + + def test_rich_constructor_rederives_list_metadata() -> None: compiled = compile_rich_markdown(b"- first\n 1. child\n", CONFIG) assert type(compiled) is ParsedDocument diff --git a/tests/unit/test_markdown_compiler.py b/tests/unit/test_markdown_compiler.py index 27c582ea..cc163398 100644 --- a/tests/unit/test_markdown_compiler.py +++ b/tests/unit/test_markdown_compiler.py @@ -31,6 +31,8 @@ StructuralPath, UnsupportedConstruct, canonicalize_parsed_document, + contains_accepted_rich_markdown_construct, + contains_only_accepted_rich_markdown_inline, contains_rich_markdown_link, ) @@ -53,8 +55,56 @@ def test_rich_link_detection_uses_the_accepted_inline_grammar(source: str) -> No assert contains_rich_markdown_link(source) -def test_rich_link_detection_ignores_plain_text() -> None: - assert not contains_rich_markdown_link("Plain text only.") +@pytest.mark.parametrize( + "source", + ( + "Plain text only.", + "- [ ] todo", + "- [x] done", + "see [1] for details", + "array[0]", + "[not a link", + ), +) +def test_rich_link_detection_rejects_bracketed_non_links(source: str) -> None: + assert not contains_rich_markdown_link(source) + + +@pytest.mark.parametrize( + ("source", "construct"), + ( + ("Only *emphasis*.", UnsupportedConstruct.EMPHASIS), + ("Only `inline code`.", UnsupportedConstruct.INLINE_CODE), + ("Only ~~strikethrough~~.", UnsupportedConstruct.STRIKETHROUGH), + ), +) +def test_rich_construct_detection_recognizes_accepted_v3_inline_syntax( + source: str, + construct: UnsupportedConstruct, +) -> None: + assert contains_accepted_rich_markdown_construct(source, construct) + + +@pytest.mark.parametrize( + ("source", "construct"), + ( + ("Only `unclosed.", UnsupportedConstruct.INLINE_CODE), + ("Only ~~unclosed.", UnsupportedConstruct.STRIKETHROUGH), + ), +) +def test_rich_construct_detection_rejects_malformed_inline_syntax( + source: str, + construct: UnsupportedConstruct, +) -> None: + assert not contains_accepted_rich_markdown_construct(source, construct) + + +def test_rich_inline_validation_rejects_an_accepted_construct_with_malformed_peer( +) -> None: + assert not contains_only_accepted_rich_markdown_inline( + "*Accepted* and `accepted` but ~~malformed.", + UnsupportedConstruct.EMPHASIS, + ) def _hex_fixture(name: str) -> bytes: