From f0a3fa89e746e8ce0c387475f3448e0aca9261e5 Mon Sep 17 00:00:00 2001 From: stone Date: Wed, 29 Jul 2026 21:16:22 +0800 Subject: [PATCH 01/16] docs: decide rich Markdown compiler runner (#124) --- ...ompile-rich-markdown-in-an-owned-runner.md | 93 +++++++++++++++++++ docs/decisions/README.md | 2 + 2 files changed, 95 insertions(+) create mode 100644 docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md diff --git a/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md b/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md new file mode 100644 index 00000000..bbd7a11e --- /dev/null +++ b/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md @@ -0,0 +1,93 @@ +--- +name: adr-0079-compile-rich-markdown-in-an-owned-runner +version: "1.0.0" +description: > + Add an explicit rich-Markdown v3 representation behind a pure owned + compiler-runner while keeping frozen v1/v2 publication inactive. +--- + +# 0079. Compile rich Markdown in an owned runner + +- Status: accepted +- Date: 2026-07-29 +- Refines: ADR-0038, ADR-0074, ADR-0075 + +## Context + +The frozen Markdown v1 and v2 representations deliberately reject syntax used +by the measured File corpus. ADR-0038 requires a new decision before accepting +another construct or adding size-based splitting. It also requires any +replacement to retain version-explicit compilation, exact source provenance, +all-or-nothing failure, atomic immutable publication, budget-visible context, +and the sealed Runtime authorization order. + +ADR-0074 permits a pinned, registered copy of RAGFlow's Apache-2.0 Markdown +parser region after path-level license and nested-dependency verification. +ADR-0075 permits Supply work in a ContextEngine-owned runner subprocess only +when the runner is a pure transform: it owns no persistence, cache, corpus, or +index and executes under the exact parent WorkerLease binding. + +## Decision + +1. **Explicit v3 representation.** `markdown-config-v3` selects + `context-engine-markdown-v3` with its own canonicalization and compilation + digest profiles. V1 and v2 remain frozen and byte-reproducible; v3 never + silently reinterprets either version. +2. **Closed rich grammar.** V3 accepts UTF-8 Markdown containing YAML + frontmatter, ATX and setext headings, nested ordered or unordered lists, + backtick or tilde fenced code blocks whose content may contain shorter fence + runs, pipe tables, wikilinks, embeds, footnotes, HTML blocks, Obsidian + callouts, inline math, and ordinary paragraphs. CRLF and lone CR input are + normalized to LF before positions and digests are derived. CommonMark or + unrestricted HTML compatibility is not claimed. +3. **Existing output contracts.** The runner emits and deserializes only the + existing `ParsedDocument`, `CompiledFragment`, `SourceSpan`, + `StructuralPath`, `CompilationProvenance`, or typed `CompilationFailure` + contracts. Rich constructs are projected into those existing section kinds; + no parallel document or Fragment model is introduced. +4. **Exact provenance.** Every Fragment carries an end-exclusive UTF-8 byte, + line, and column span. Its `source_text` is exactly the canonical input byte + slice at that span. Non-whitespace source bytes cannot be omitted. Table + Fragments obey the same round-trip rule. Heading ancestry is derived during + compilation and copied into the same budget-visible Fragment. +5. **Hard splitting gate.** V3 uses a representation-bound token ceiling. The + deterministic counter treats each non-whitespace run as one token. A block + whose contextual text would exceed the ceiling is split at source token + boundaries into ordered Fragments; every part retains the same preceding + heading ancestry, exact source span, and structural lineage. If heading + ancestry alone leaves no capacity, or an indivisible construct cannot be + represented within the ceiling, compilation refuses all-or-nothing. +6. **Owned pure process.** The compiler-runner receives exact source bytes and + configuration, invokes the registered vendored parser region plus the + ContextEngine hierarchy/span/bounds kernel, and emits deterministic bytes. + It performs no network or database I/O and retains no independent state, + cache, index, or checkpoint. +7. **Activation remains deferred.** This decision proves the v3 pure transform + and local acceptance reporting only. The active File import configuration, + database publication functions, immutable Revision schemas, embeddings, + and existing v1 Revision migration remain unchanged. Activating v3 for + production publication requires a separate decision and complete atomic + publication and re-embedding evidence. +8. **Runtime remains sealed.** Compilation changes no Runtime composition. + After any future v3 activation, each published Fragment is still only a + candidate until it crosses the exact + `CandidateRef -> AuthorizationKernel -> AuthorizedProjection` path. + +## Consequences + +- Rich-note and hard-bound behavior can be verified without creating durable + content or adding another persistence truth. +- Deterministic source spans and heading context remain part of the same + Fragment and therefore visible to existing provenance and Package budgets. +- The first controlled RAGFlow reuse pays its own pinned registration and + dependency-audit cost; artifact-wide aggregation remains owned by its + separately accepted work. +- Production File imports continue to use their current frozen representation + until activation and migration are decided explicitly. + +## Revisit trigger + +Revisit before accepting a construct outside the closed v3 grammar, changing +the token counter or ceiling, splitting an indivisible construct differently, +activating v3 publication, migrating an existing Revision, adding runner-local +state, or changing the same-Fragment ancestry rule. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 4da67764..5b6e46b1 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -42,6 +42,7 @@ kernel, capability separation, and publication visibility model. | First File source registration | [0035 — Trusted File source registration](0035-register-file-sources-through-context-control.md) | One operation-bound trusted Control call atomically creates an Organization-owned source plus immutable active first version; all acquisition carriers remain unavailable | Caller-authored Organization/mode, host paths, registration-time File I/O, future capability claims, or cross-tenant idempotency | | First Markdown compiler | [0036 — Deterministic narrow Markdown compilation](0036-compile-narrow-markdown-deterministically.md) | Exact bytes compile purely into one canonical heading-plus-paragraph ParsedDocument with normalized coordinates and versioned content/compilation identities | Path-coupled parsing, silent unsupported syntax, partial documents, unversioned derived identities, or parser-side I/O | | Structural Markdown compiler | [0038 — Structural Markdown units](0038-compile-and-publish-structural-markdown.md) | Explicit v2 compilation publishes one coherent Fragment per heading, paragraph, list, fenced code block, or table with exact provenance and same-Fragment heading ancestry | Silent v1 reinterpretation, item/cell fragmentation, post-authorization parent expansion, or unbudgeted context | +| Rich Markdown compiler-runner | [0079 — Rich Markdown in an owned runner](0079-compile-rich-markdown-in-an-owned-runner.md) | Explicit v3 pure compilation accepts the closed rich grammar with exact source spans, same-Fragment ancestry, and a hard token ceiling while production activation stays deferred | Silent v1/v2 reinterpretation, runner-local state, unbounded Fragments, or activating publication without a separate decision | | Unchanged File acquisition | [0039 — File acquisition no-op](0039-deduplicate-unchanged-file-acquisitions.md) | Tenant/source/resource-scoped canonical identity and one PostgreSQL guard lock classify a complete active artifact before publication; each observation retains an immutable digest-only outcome | Cross-tenant/global deduplication, process-local locking, partial-artifact reuse, silent version reuse, or no-op by bypassing publication validation | | File Resource deletion | [0042 — Tombstone before cleanup](0042-tombstone-file-resources-before-cleanup.md) | One trusted Control transaction tombstones the active File Resource, advances its Organization Policy Epoch, and records immutable pending cleanup lineage before any physical deletion | Cleanup-defined visibility, caller-authored tenant/epoch/cleanup identity, index deletion as authorization, restore, or native watcher claims | | File Source progress | [0043 — Separate acquisition and publication progress](0043-separate-file-acquisition-progress-from-publication-progress.md), [0072 — Report File source status with closed refusals](0072-report-file-source-status-with-closed-refusals.md) | Append accepted changes separately from contiguous Runtime-visibility completion and expose content-free operational status through an Organization/Source-scoped Control read | One ambiguous checkpoint, skipped publication gaps, retained source/compiler diagnostics, Runtime authorization from status or watermarks, or false standard ProviderPort capability claims | @@ -182,5 +183,6 @@ touched: - [0076 — Rejoin rank evidence after authorization](0076-rejoin-rank-evidence-after-authorization.md) - [0077 — Fix the Article as the content authorization atom](0077-fix-the-article-as-the-content-authorization-atom.md) - [0078 — Narrow the contract-kit gate to per-connector twins](0078-narrow-the-contract-kit-gate-to-per-connector-twins.md) +- [0079 — Compile rich Markdown in an owned runner](0079-compile-rich-markdown-in-an-owned-runner.md) - [0080 — Refuse authoritative evaluation without an executor](0080-refuse-authoritative-evaluation-without-an-executor.md) - [0081 — Seal candidate discovery behind a data-only session](0081-seal-candidate-discovery-behind-a-data-only-session.md) From 6258f52d4df2df7961ae351c4cc34dc4ea9337c7 Mon Sep 17 00:00:00 2001 From: stone Date: Wed, 29 Jul 2026 21:16:22 +0800 Subject: [PATCH 02/16] chore: register pinned RAGFlow parser (#124) --- THIRD_PARTY_NOTICES.md | 13 + pyproject.toml | 20 +- .../test_third_party_ragflow_registration.py | 119 ++++ third_party/ragflow/LICENSE.upstream | 201 +++++++ third_party/ragflow/MODIFICATIONS.md | 28 + third_party/ragflow/UPSTREAM.toml | 15 + .../ragflow/deepdoc/parser/markdown_parser.py | 527 ++++++++++++++++++ third_party/ragflow/patches/.gitkeep | 0 uv.lock | 11 + 9 files changed, 932 insertions(+), 2 deletions(-) create mode 100644 THIRD_PARTY_NOTICES.md create mode 100644 tests/unit/test_third_party_ragflow_registration.py create mode 100644 third_party/ragflow/LICENSE.upstream create mode 100644 third_party/ragflow/MODIFICATIONS.md create mode 100644 third_party/ragflow/UPSTREAM.toml create mode 100644 third_party/ragflow/deepdoc/parser/markdown_parser.py create mode 100644 third_party/ragflow/patches/.gitkeep diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md new file mode 100644 index 00000000..9bc46f66 --- /dev/null +++ b/THIRD_PARTY_NOTICES.md @@ -0,0 +1,13 @@ +# Third-party notices + +## RAGFlow Markdown parser + +- Project: RAGFlow +- Upstream repository: +- Pinned commit: `4391e03886b996201f3b8818f671b19eb24d0f7b` +- Registered source: `deepdoc/parser/markdown_parser.py` +- License: Apache License 2.0 + +The verbatim upstream license and the complete path-level registration ship in +`third_party/ragflow/`. The registered parser imports Python-Markdown, which is +distributed under the BSD 3-Clause License. diff --git a/pyproject.toml b/pyproject.toml index 92953fa3..967560e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,13 +8,19 @@ version = "0.1.0" description = "Permission-aware context delivery engine" readme = "README.md" license = "Apache-2.0" -license-files = ["LICENSE", "NOTICE"] +license-files = [ + "LICENSE", + "NOTICE", + "THIRD_PARTY_NOTICES.md", + "third_party/ragflow/LICENSE.upstream", +] requires-python = ">=3.13,<3.14" dependencies = [ "alembic>=1.16,<1.17", "cryptography>=49,<50", "fastapi>=0.116,<0.117", "jsonschema>=4.25,<5", + "markdown>=3.6,<3.7", "pydantic>=2.13,<2.14", "psycopg[binary]>=3.2,<3.3", "rfc8785>=0.1.4,<0.2", @@ -48,7 +54,14 @@ benchmark = [ ] [tool.hatch.build.targets.wheel] -packages = ["engine", "adapters", "applications", "eval", "migrations"] +packages = [ + "engine", + "adapters", + "applications", + "eval", + "migrations", + "third_party", +] [tool.hatch.build.targets.wheel.force-include] "eval/golden/v1/schema.json" = "eval/golden/v1/schema.json" @@ -82,3 +95,6 @@ line-length = 88 [tool.ruff.lint] select = ["E", "F", "I", "UP", "B", "SIM"] + +[tool.ruff.lint.per-file-ignores] +"third_party/**/*.py" = ["E501", "SIM103", "UP009", "UP032"] diff --git a/tests/unit/test_third_party_ragflow_registration.py b/tests/unit/test_third_party_ragflow_registration.py new file mode 100644 index 00000000..f24e5e7d --- /dev/null +++ b/tests/unit/test_third_party_ragflow_registration.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +import ast +import hashlib +import re +import tomllib +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).parents[2] +REGISTRATION_ROOT = REPOSITORY_ROOT / "third_party/ragflow" +REGISTRATION_PATH = REGISTRATION_ROOT / "UPSTREAM.toml" +REQUIRED_EXCLUSIONS = { + "deepdoc/parser/__init__.py", + "rag/app/naive.py", + "rag/nlp", +} +ALLOWED_IMPORT_ROOTS = { + "__future__", + "argparse", + "collections", + "dataclasses", + "enum", + "hashlib", + "html", + "json", + "logging", + "markdown", + "pathlib", + "re", + "sys", + "typing", + "unicodedata", +} + + +def _registration() -> dict[str, object]: + return tomllib.loads(REGISTRATION_PATH.read_text(encoding="utf-8")) + + +def test_vendored_bytes_match_complete_pinned_registration() -> None: + registration = _registration() + + assert registration["repository"] == "https://github.com/infiniflow/ragflow.git" + commit = registration["commit"] + assert isinstance(commit, str) + assert re.fullmatch(r"[0-9a-f]{40}", commit) + assert commit == "4391e03886b996201f3b8818f671b19eb24d0f7b" + assert registration["reuse_mode"] == "copy-patch" + assert registration["approval"] == ( + "https://github.com/stone16/context-engine/issues/124" + ) + assert registration["source_paths"] == ["deepdoc/parser/markdown_parser.py"] + excluded_paths = registration["excluded_paths"] + assert isinstance(excluded_paths, list) + assert set(excluded_paths) >= REQUIRED_EXCLUSIONS + + files = registration["files"] + assert isinstance(files, list) + assert files + registered_paths: set[Path] = set() + for entry in files: + assert isinstance(entry, dict) + assert set(entry) == {"upstream_path", "vendored_path", "sha256"} + upstream_path = entry["upstream_path"] + vendored_path = entry["vendored_path"] + expected_hash = entry["sha256"] + assert isinstance(upstream_path, str) and upstream_path + assert isinstance(vendored_path, str) and vendored_path + assert isinstance(expected_hash, str) + assert re.fullmatch(r"[0-9a-f]{64}", expected_hash) + path = REPOSITORY_ROOT / vendored_path + path.relative_to(REGISTRATION_ROOT) + assert path.is_file() + assert hashlib.sha256(path.read_bytes()).hexdigest() == expected_hash + registered_paths.add(path) + + vendored_files = { + path + for path in REGISTRATION_ROOT.rglob("*") + if path.is_file() + and "__pycache__" not in path.relative_to(REGISTRATION_ROOT).parts + and path.name + not in {"LICENSE.upstream", "MODIFICATIONS.md", "UPSTREAM.toml"} + and "patches" not in path.relative_to(REGISTRATION_ROOT).parts + } + assert registered_paths == vendored_files + assert (REGISTRATION_ROOT / "LICENSE.upstream").is_file() + assert (REGISTRATION_ROOT / "MODIFICATIONS.md").is_file() + assert (REGISTRATION_ROOT / "patches").is_dir() + assert (REPOSITORY_ROOT / "THIRD_PARTY_NOTICES.md").is_file() + + +def test_vendored_subtree_imports_only_approved_dependencies() -> None: + registration = _registration() + files = registration["files"] + assert isinstance(files, list) + + imports: set[str] = set() + for entry in files: + assert isinstance(entry, dict) + vendored_path = entry["vendored_path"] + assert isinstance(vendored_path, str) + path = REPOSITORY_ROOT / vendored_path + if path.suffix != ".py": + continue + tree = ast.parse(path.read_bytes(), filename=str(path)) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imports.update(alias.name.partition(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.level == 0: + assert node.module is not None + imports.add(node.module.partition(".")[0]) + + assert imports <= ALLOWED_IMPORT_ROOTS + modifications = (REGISTRATION_ROOT / "MODIFICATIONS.md").read_text( + encoding="utf-8" + ) + assert "Python-Markdown" in modifications + assert "BSD 3-Clause" in modifications diff --git a/third_party/ragflow/LICENSE.upstream b/third_party/ragflow/LICENSE.upstream new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/third_party/ragflow/LICENSE.upstream @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/third_party/ragflow/MODIFICATIONS.md b/third_party/ragflow/MODIFICATIONS.md new file mode 100644 index 00000000..921c96c5 --- /dev/null +++ b/third_party/ragflow/MODIFICATIONS.md @@ -0,0 +1,28 @@ +# RAGFlow Markdown parser registration + +## Pinned source + +The registered region is `deepdoc/parser/markdown_parser.py` from RAGFlow at +commit `4391e03886b996201f3b8818f671b19eb24d0f7b`. The exact path carries an +Apache-2.0 header and is covered by the repository-root Apache-2.0 license, +reproduced verbatim as `LICENSE.upstream`. The pinned upstream tree contains no +root `NOTICE` file. + +## Nested notice scan + +The copied file imports only Python standard-library modules and +Python-Markdown. Python-Markdown is licensed under the BSD 3-Clause License. +No other nested third-party dependency is imported by the copied region. + +## ContextEngine modifications + +The upstream bytes are retained under `deepdoc/parser/markdown_parser.py` for +auditable provenance. ContextEngine adapters patch behavior through the owned +compiler-runner and do not import RAGFlow package initializers or its `rag/nlp` +dependency carriers. Local changes add exact UTF-8 spans, heading ancestry, +hard bounds, versioned deterministic output, and typed refusal at the +ContextEngine seam. + +The empty `patches/` directory is reserved for future upgrades that change the +registered upstream file itself. The current integration wraps the copied file, +so no source patch applies. diff --git a/third_party/ragflow/UPSTREAM.toml b/third_party/ragflow/UPSTREAM.toml new file mode 100644 index 00000000..569bca79 --- /dev/null +++ b/third_party/ragflow/UPSTREAM.toml @@ -0,0 +1,15 @@ +repository = "https://github.com/infiniflow/ragflow.git" +commit = "4391e03886b996201f3b8818f671b19eb24d0f7b" +source_paths = ["deepdoc/parser/markdown_parser.py"] +excluded_paths = [ + "deepdoc/parser/__init__.py", + "rag/app/naive.py", + "rag/nlp", +] +reuse_mode = "copy-patch" +approval = "https://github.com/stone16/context-engine/issues/124" + +[[files]] +upstream_path = "deepdoc/parser/markdown_parser.py" +vendored_path = "third_party/ragflow/deepdoc/parser/markdown_parser.py" +sha256 = "94c8e2515d05e141fcf65e10336ceca7f9116e54b31a668637ba3f901943cb66" diff --git a/third_party/ragflow/deepdoc/parser/markdown_parser.py b/third_party/ragflow/deepdoc/parser/markdown_parser.py new file mode 100644 index 00000000..583e4ffd --- /dev/null +++ b/third_party/ragflow/deepdoc/parser/markdown_parser.py @@ -0,0 +1,527 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2025 The InfiniFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import re + +from markdown import markdown + + +class RAGFlowMarkdownParser: + def __init__(self, chunk_token_num=128): + self.chunk_token_num = int(chunk_token_num) + + def extract_tables_and_remainder(self, markdown_text, separate_tables=True): + tables = [] + working_text = markdown_text + + def replace_tables_with_rendered_html(pattern, table_list, render=True): + new_text = "" + last_end = 0 + for match in pattern.finditer(working_text): + raw_table = match.group() + table_list.append(raw_table) + if separate_tables: + # Skip this match (i.e., remove it) + new_text += working_text[last_end : match.start()] + "\n\n" + else: + # Replace with rendered HTML + html_table = markdown(raw_table, extensions=["markdown.extensions.tables"]) if render else raw_table + new_text += working_text[last_end : match.start()] + html_table + "\n\n" + last_end = match.end() + new_text += working_text[last_end:] + return new_text + + if "|" in markdown_text: # for optimize performance + # Standard Markdown table + border_table_pattern = re.compile( + r""" + (?:\n|^) + (?:\|.*?\|.*?\|.*?\n) + (?:\|(?:\s*[:-]+[-| :]*\s*)\|.*?\n) + (?:\|.*?\|.*?\|.*?\n)+ + """, + re.VERBOSE, + ) + working_text = replace_tables_with_rendered_html(border_table_pattern, tables, render=separate_tables) + + # Borderless Markdown table + no_border_table_pattern = re.compile( + r""" + (?:\n|^) + (?:\S.*?\|.*?\n) + (?:(?:\s*[:-]+[-| :]*\s*).*?\n) + (?:\S.*?\|.*?\n)+ + """, + re.VERBOSE, + ) + working_text = replace_tables_with_rendered_html(no_border_table_pattern, tables, render=separate_tables) + + # Replace any TAGS e.g. to
+ TAGS = ["table", "td", "tr", "th", "tbody", "thead", "div"] + table_with_attributes_pattern = re.compile(rf"<(?:{'|'.join(TAGS)})[^>]*>", re.IGNORECASE) + + def replace_tag(m): + tag_name = re.match(r"<(\w+)", m.group()).group(1) + return "<{}>".format(tag_name) + + working_text = re.sub(table_with_attributes_pattern, replace_tag, working_text) + + if "
" in working_text.lower(): # for optimize performance + # HTML table extraction - handle possible html/body wrapper tags + html_table_pattern = re.compile( + r""" + (?:\n|^) + \s* + (?: + # case1:
...
+ (?:]*>\s*]*>\s*]*>.*?\s*\s*) + | + # case2: ...
+ (?:]*>\s*]*>.*?\s*) + | + # case3: only...
+ (?:]*>.*?) + ) + \s* + (?=\n|$) + """, + re.VERBOSE | re.DOTALL | re.IGNORECASE, + ) + + def replace_html_tables(): + nonlocal working_text + new_text = "" + last_end = 0 + for match in html_table_pattern.finditer(working_text): + raw_table = match.group() + tables.append(raw_table) + if separate_tables: + new_text += working_text[last_end : match.start()] + "\n\n" + else: + new_text += working_text[last_end : match.start()] + raw_table + "\n\n" + last_end = match.end() + new_text += working_text[last_end:] + working_text = new_text + + replace_html_tables() + + return working_text, tables + + +class MarkdownElementExtractor: + def __init__(self, markdown_content): + self.markdown_content = markdown_content + self.lines = markdown_content.split("\n") + + def get_delimiters(self, delimiters): + toks = re.findall(r"`([^`]+)`", delimiters) + toks = sorted(set(toks), key=lambda x: -len(x)) + return "|".join(re.escape(t) for t in toks if t) + + def _get_fence_marker(self, line): + match = re.match(r"^[ \t]{0,3}(?P`{3,}|~{3,})(?:.*)$", line) + if not match: + return None + fence = match.group("fence") + return fence[0], len(fence) + + def _is_closing_fence(self, line, fence_char, fence_len): + pattern = r"^[ \t]{0,3}" + re.escape(fence_char) + r"{" + str(fence_len) + r",}\s*$" + return re.match(pattern, line) is not None + + def _line_start_offsets(self, text): + offsets = [] + offset = 0 + for line in self.lines: + offsets.append(offset) + offset += len(line) + 1 + return offsets + + def _fenced_code_ranges(self, text): + ranges = [] + line_offsets = self._line_start_offsets(text) + + i = 0 + while i < len(self.lines): + marker = self._get_fence_marker(self.lines[i]) + if not marker: + i += 1 + continue + + fence_char, fence_len = marker + start_pos = line_offsets[i] + end_line = len(self.lines) - 1 + for j in range(i + 1, len(self.lines)): + if self._is_closing_fence(self.lines[j], fence_char, fence_len): + end_line = j + break + + end_pos = min(len(text), line_offsets[end_line] + len(self.lines[end_line])) + ranges.append((start_pos, end_pos)) + i = end_line + 1 + + return ranges + + def _table_cells(self, line): + stripped = line.strip() + if "|" not in stripped: + return [] + if stripped.startswith("|"): + stripped = stripped[1:] + if stripped.endswith("|"): + stripped = stripped[:-1] + return [cell.strip() for cell in stripped.split("|")] + + def _is_table_row(self, line): + cells = self._table_cells(line) + return len(cells) >= 2 and any(cell for cell in cells) + + def _is_table_separator_row(self, line): + cells = self._table_cells(line) + return len(cells) >= 2 and all(re.match(r"^:?-+:?$", cell.replace(" ", "")) for cell in cells) + + def _markdown_table_ranges(self, text): + ranges = [] + line_offsets = self._line_start_offsets(text) + + i = 0 + while i < len(self.lines) - 1: + if not self._is_table_row(self.lines[i]) or not self._is_table_separator_row(self.lines[i + 1]): + i += 1 + continue + + end_line = i + 1 + j = i + 2 + while j < len(self.lines) and self._is_table_row(self.lines[j]): + end_line = j + j += 1 + + end_pos = min(len(text), line_offsets[end_line] + len(self.lines[end_line])) + ranges.append((line_offsets[i], end_pos)) + i = end_line + 1 + + return ranges + + def _html_table_ranges(self, text): + table_pattern = re.compile( + r""" + (?: + (?:]*>\s*]*>\s*]*>.*?\s*\s*) + | + (?:]*>\s*]*>.*?\s*) + | + (?:]*>.*?) + ) + """, + re.VERBOSE | re.DOTALL | re.IGNORECASE, + ) + return [(match.start(), match.end()) for match in table_pattern.finditer(text)] + + def _merge_ranges(self, ranges): + if not ranges: + return [] + + merged = [] + for start, end in sorted(ranges): + if not merged or start > merged[-1][1]: + merged.append((start, end)) + else: + merged[-1] = (merged[-1][0], max(merged[-1][1], end)) + return merged + + def _protected_ranges(self, text): + return self._merge_ranges(self._fenced_code_ranges(text) + self._markdown_table_ranges(text) + self._html_table_ranges(text)) + + def _append_delimited_section(self, sections, text, start, end, include_meta): + part = text[start:end] + if not part or not part.strip(): + return + if include_meta: + sections.append( + { + "content": part.strip(), + "start_line": text.count("\n", 0, start), + "end_line": text.count("\n", 0, end), + } + ) + else: + sections.append(part.strip()) + + def _extract_delimited_elements(self, text, delimiters, include_meta=False): + sections = [] + pattern = re.compile(delimiters) + protected_ranges = self._protected_ranges(text) + if protected_ranges: + logging.debug("markdown_parser: detected %d protected ranges for delimiter extraction", len(protected_ranges)) + protected_idx = 0 + last_end = 0 + + for match in pattern.finditer(text): + while protected_idx < len(protected_ranges) and protected_ranges[protected_idx][1] <= match.start(): + protected_idx += 1 + + if protected_idx < len(protected_ranges): + start, end = protected_ranges[protected_idx] + if start <= match.start() < end: + logging.debug( + "markdown_parser: skipped delimiter match at pos=%d delimiter=%r inside fenced range %s", + match.start(), + match.group(), + (start, end), + ) + continue + + self._append_delimited_section(sections, text, last_end, match.start(), include_meta) + last_end = match.end() + + self._append_delimited_section(sections, text, last_end, len(text), include_meta) + return sections + + def extract_elements(self, delimiter=None, include_meta=False): + """Extract individual elements (headers, code blocks, lists, etc.)""" + sections = [] + + i = 0 + dels = "" + if delimiter: + dels = self.get_delimiters(delimiter) + if len(dels) > 0: + text = "\n".join(self.lines) + sections = self._extract_delimited_elements(text, dels, include_meta) + + # Attach lone header lines to the section that follows them so that + # "## Title\n" never becomes an isolated chunk when the delimiter + # splits at every newline. A header is "lone" when it occupies a + # single line (no embedded newline after stripping). + def _is_lone_header(section_content): + stripped = section_content.strip() + return bool(re.match(r"^#{1,6}\s+\S", stripped)) and "\n" not in stripped + + def _is_attachable_body(section_content): + """True when the following chunk is prose body, not code/table/list/etc.""" + stripped = section_content.strip() + if not stripped: + return False + first_line = stripped.split("\n", 1)[0] + if self._get_fence_marker(first_line): + return False + if first_line.lstrip().startswith("|"): + return False + if re.match(r"^\S+\s*\|", first_line): + return False + if first_line.lstrip().startswith("<"): + return False + if re.match(r"^\s*[-*+]\s+", first_line) or re.match(r"^\s*\d+\.\s+", first_line): + return False + if first_line.lstrip().startswith(">"): + return False + return True + + merged = [] + merged_header_count = 0 + i = 0 + while i < len(sections): + content = sections[i]["content"] if include_meta else sections[i] + if _is_lone_header(content): + header_parts = [content.strip()] + j = i + 1 + while j < len(sections): + next_content = sections[j]["content"] if include_meta else sections[j] + if not _is_lone_header(next_content): + break + header_parts.append(next_content.strip()) + j += 1 + if j < len(sections): + body_content = sections[j]["content"] if include_meta else sections[j] + if _is_attachable_body(body_content): + combined = "\n".join(header_parts) + "\n" + body_content + if include_meta: + merged.append( + { + **sections[i], + "content": combined, + "end_line": sections[j]["end_line"], + } + ) + else: + merged.append(combined) + merged_header_count += len(header_parts) + i = j + 1 + continue + for k in range(i, j): + merged.append(sections[k]) + i = j + continue + merged.append(sections[i]) + i += 1 + if merged_header_count: + logging.debug( + "markdown_parser: merged %d lone header line(s) into following sections", + merged_header_count, + ) + return merged + while i < len(self.lines): + line = self.lines[i] + + if re.match(r"^#{1,6}\s+.*$", line): + # header + element = self._extract_header(i) + sections.append(element if include_meta else element["content"]) + i = element["end_line"] + 1 + elif self._get_fence_marker(line): + # code block + element = self._extract_code_block(i) + sections.append(element if include_meta else element["content"]) + i = element["end_line"] + 1 + elif re.match(r"^\s*[-*+]\s+.*$", line) or re.match(r"^\s*\d+\.\s+.*$", line): + # list block + element = self._extract_list_block(i) + sections.append(element if include_meta else element["content"]) + i = element["end_line"] + 1 + elif line.strip().startswith(">"): + # blockquote + element = self._extract_blockquote(i) + sections.append(element if include_meta else element["content"]) + i = element["end_line"] + 1 + elif line.strip(): + # text block (paragraphs and inline elements until next block element) + element = self._extract_text_block(i) + sections.append(element if include_meta else element["content"]) + i = element["end_line"] + 1 + else: + i += 1 + + if include_meta: + sections = [section for section in sections if section["content"].strip()] + else: + sections = [section for section in sections if section.strip()] + return sections + + def _extract_header(self, start_pos): + return { + "type": "header", + "content": self.lines[start_pos], + "start_line": start_pos, + "end_line": start_pos, + } + + def _extract_code_block(self, start_pos): + end_pos = start_pos + content_lines = [self.lines[start_pos]] + fence_char, fence_len = self._get_fence_marker(self.lines[start_pos]) + + # Find the end of the code block + for i in range(start_pos + 1, len(self.lines)): + content_lines.append(self.lines[i]) + end_pos = i + if self._is_closing_fence(self.lines[i], fence_char, fence_len): + break + + return { + "type": "code_block", + "content": "\n".join(content_lines), + "start_line": start_pos, + "end_line": end_pos, + } + + def _extract_list_block(self, start_pos): + end_pos = start_pos + content_lines = [] + + i = start_pos + while i < len(self.lines): + line = self.lines[i] + # check if this line is a list item or continuation of a list + if ( + re.match(r"^\s*[-*+]\s+.*$", line) + or re.match(r"^\s*\d+\.\s+.*$", line) + or (i > start_pos and not line.strip()) + or (i > start_pos and re.match(r"^\s{2,}[-*+]\s+.*$", line)) + or (i > start_pos and re.match(r"^\s{2,}\d+\.\s+.*$", line)) + or (i > start_pos and re.match(r"^\s+\w+.*$", line)) + ): + content_lines.append(line) + end_pos = i + i += 1 + else: + break + + return { + "type": "list_block", + "content": "\n".join(content_lines), + "start_line": start_pos, + "end_line": end_pos, + } + + def _extract_blockquote(self, start_pos): + end_pos = start_pos + content_lines = [] + + i = start_pos + while i < len(self.lines): + line = self.lines[i] + if line.strip().startswith(">") or (i > start_pos and not line.strip()): + content_lines.append(line) + end_pos = i + i += 1 + else: + break + + return { + "type": "blockquote", + "content": "\n".join(content_lines), + "start_line": start_pos, + "end_line": end_pos, + } + + def _extract_text_block(self, start_pos): + """Extract a text block (paragraphs, inline elements) until next block element""" + end_pos = start_pos + content_lines = [self.lines[start_pos]] + + i = start_pos + 1 + while i < len(self.lines): + line = self.lines[i] + # stop if we encounter a block element + if re.match(r"^#{1,6}\s+.*$", line) or self._get_fence_marker(line) or re.match(r"^\s*[-*+]\s+.*$", line) or re.match(r"^\s*\d+\.\s+.*$", line) or line.strip().startswith(">"): + break + elif not line.strip(): + # check if the next line is a block element + if i + 1 < len(self.lines) and ( + re.match(r"^#{1,6}\s+.*$", self.lines[i + 1]) + or self._get_fence_marker(self.lines[i + 1]) + or re.match(r"^\s*[-*+]\s+.*$", self.lines[i + 1]) + or re.match(r"^\s*\d+\.\s+.*$", self.lines[i + 1]) + or self.lines[i + 1].strip().startswith(">") + ): + break + else: + content_lines.append(line) + end_pos = i + i += 1 + else: + content_lines.append(line) + end_pos = i + i += 1 + + return { + "type": "text_block", + "content": "\n".join(content_lines), + "start_line": start_pos, + "end_line": end_pos, + } diff --git a/third_party/ragflow/patches/.gitkeep b/third_party/ragflow/patches/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/uv.lock b/uv.lock index b0aade86..15b6150b 100644 --- a/uv.lock +++ b/uv.lock @@ -119,6 +119,7 @@ dependencies = [ { name = "cryptography" }, { name = "fastapi" }, { name = "jsonschema" }, + { name = "markdown" }, { name = "psycopg", extra = ["binary"] }, { name = "pydantic" }, { name = "rfc8785" }, @@ -148,6 +149,7 @@ requires-dist = [ { name = "cryptography", specifier = ">=49,<50" }, { name = "fastapi", specifier = ">=0.116,<0.117" }, { name = "jsonschema", specifier = ">=4.25,<5" }, + { name = "markdown", specifier = ">=3.6,<3.7" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.2,<3.3" }, { name = "pydantic", specifier = ">=2.13,<2.14" }, { name = "rfc8785", specifier = ">=0.1.4,<0.2" }, @@ -502,6 +504,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bc/b1/a0ec7a5a9db730a08daef1fdfb8090435b82465abbf758a596f0ea88727e/mako-1.3.12-py3-none-any.whl", hash = "sha256:8f61569480282dbf557145ce441e4ba888be453c30989f879f0d652e39f53ea9", size = 78521, upload-time = "2026-04-28T19:01:10.393Z" }, ] +[[package]] +name = "markdown" +version = "3.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/02/4785861427848cc11e452cc62bb541006a1087cf04a1de83aedd5530b948/Markdown-3.6.tar.gz", hash = "sha256:ed4f41f6daecbeeb96e576ce414c41d2d876daa9a16cb35fa8ed8c2ddfad0224", size = 354715, upload-time = "2024-03-14T15:37:59.775Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/b3/0c0c994fe49cd661084f8d5dc06562af53818cc0abefaca35bdc894577c3/Markdown-3.6-py3-none-any.whl", hash = "sha256:48f276f4d8cfb8ce6527c8f79e2ee29708508bf4d40aa410fbc3b4ee832c850f", size = 105381, upload-time = "2024-03-14T15:37:57.457Z" }, +] + [[package]] name = "markdown-it-py" version = "4.2.0" From b5c52641a5c20fd89c0c62fb0a52c4ca181ddd48 Mon Sep 17 00:00:00 2001 From: stone Date: Wed, 29 Jul 2026 21:16:22 +0800 Subject: [PATCH 03/16] feat: add rich Markdown compiler runner (#124) --- adapters/parsers/ragflow_markdown.py | 537 ++++++++++++++++++ applications/compiler_runner.py | 257 +++++++++ .../compiler-runner-acceptance-v1.schema.json | 70 +++ engine/supply/__init__.py | 10 + engine/supply/markdown.py | 237 +++++++- tests/fixtures/markdown/rich-code-tables.md | 12 + tests/fixtures/markdown/rich-frontmatter.md | 8 + .../fixtures/markdown/rich-headings-lists.md | 12 + tests/fixtures/markdown/rich-inline-html.md | 12 + .../fixtures/markdown/rich-mixed-newlines.hex | 1 + tests/unit/test_compiler_runner_acceptance.py | 81 +++ .../test_compiler_runner_acceptance_report.py | 44 ++ tests/unit/test_compiler_runner_bounds.py | 35 ++ .../unit/test_compiler_runner_determinism.py | 25 + tests/unit/test_compiler_runner_refusal.py | 42 ++ tests/unit/test_compiler_runner_tables.py | 33 ++ 16 files changed, 1409 insertions(+), 7 deletions(-) create mode 100644 adapters/parsers/ragflow_markdown.py create mode 100644 applications/compiler_runner.py create mode 100644 docs/contracts/compiler-runner-acceptance-v1.schema.json create mode 100644 tests/fixtures/markdown/rich-code-tables.md create mode 100644 tests/fixtures/markdown/rich-frontmatter.md create mode 100644 tests/fixtures/markdown/rich-headings-lists.md create mode 100644 tests/fixtures/markdown/rich-inline-html.md create mode 100644 tests/fixtures/markdown/rich-mixed-newlines.hex create mode 100644 tests/unit/test_compiler_runner_acceptance.py create mode 100644 tests/unit/test_compiler_runner_acceptance_report.py create mode 100644 tests/unit/test_compiler_runner_bounds.py create mode 100644 tests/unit/test_compiler_runner_determinism.py create mode 100644 tests/unit/test_compiler_runner_refusal.py create mode 100644 tests/unit/test_compiler_runner_tables.py diff --git a/adapters/parsers/ragflow_markdown.py b/adapters/parsers/ragflow_markdown.py new file mode 100644 index 00000000..f42b0d80 --- /dev/null +++ b/adapters/parsers/ragflow_markdown.py @@ -0,0 +1,537 @@ +"""Rich Markdown compiler built around the registered RAGFlow parser region.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any, Final, Protocol, cast + +from engine.supply.markdown import ( + MARKDOWN_CODE_LANGUAGE_MAX_LENGTH, + MARKDOWN_COMPILER_V3_VERSION, + MARKDOWN_RICH_CANONICALIZATION_PROFILE, + MARKDOWN_RICH_COMPILATION_DIGEST_PROFILE, + MARKDOWN_RICH_TOKEN_CEILING, + CompilationFailure, + CompilationFailureCode, + CompilationOutcome, + CompilationProvenance, + CompiledFragment, + MarkdownCompilerConfig, + ParsedDocument, + ParsedSection, + SectionKind, + SourcePoint, + SourceSpan, + StructuralPath, + UnsupportedConstruct, +) +from third_party.ragflow.deepdoc.parser.markdown_parser import MarkdownElementExtractor + +_UTF8_BOM: Final = b"\xef\xbb\xbf" +_ATX_HEADING: Final = re.compile(r"^ {0,3}(#{1,6})[ \t]+(.+?)[ \t]*#*[ \t]*$") +_SETEXT: Final = re.compile(r"^ {0,3}(=+|-+)[ \t]*$") +_LIST_ITEM: Final = re.compile(r"^([ \t]*)(?:[-+*]|[0-9]+[.)])[ \t]+(.+)$") +_FRONTMATTER_FIELD: Final = re.compile(r"^[A-Za-z0-9_.-]+[ \t]*:[ \t]*(?:.*)$") +_TOKEN: Final = re.compile(r"\S+") + + +@dataclass(frozen=True, slots=True) +class _Block: + kind: SectionKind + start: int + end: int + level: int | None = None + list_ordered: bool | None = None + list_items: tuple[str, ...] = () + code_language: str | None = None + code_body: str | None = None + table_header: tuple[str, ...] = () + table_rows: tuple[tuple[str, ...], ...] = () + + +class _ElementExtractor(Protocol): + def _get_fence_marker(self, line: str) -> tuple[str, int] | None: ... + + def _is_closing_fence( + self, + line: str, + fence_char: str, + fence_len: int, + ) -> bool: ... + + def _table_cells(self, line: str) -> list[str]: ... + + def _is_table_row(self, line: str) -> bool: ... + + def _is_table_separator_row(self, line: str) -> bool: ... + + +def rich_token_count(value: str) -> int: + """Count deterministic representation tokens for the v3 hard bound.""" + + if type(value) is not str: + raise TypeError("rich Markdown token counting requires exact text") + return sum(1 for _ in _TOKEN.finditer(value)) + + +def _failure( + code: CompilationFailureCode, + text: str, + offset: int, + construct: UnsupportedConstruct | None = None, +) -> CompilationFailure: + return CompilationFailure( + code=code, + position=_point(text, offset), + construct=construct, + ) + + +def _point(text: str, offset: int) -> SourcePoint: + prefix = text[:offset] + last_newline = prefix.rfind("\n") + return SourcePoint( + line=prefix.count("\n") + 1, + column=len(prefix[last_newline + 1 :]) + 1, + byte_offset=len(prefix.encode("utf-8")), + ) + + +def _span(text: str, start: int, end: int) -> SourceSpan: + return SourceSpan(start=_point(text, start), end=_point(text, end)) + + +def _normalize(source: bytes) -> str | CompilationFailure: + raw = source.removeprefix(_UTF8_BOM) + try: + decoded = raw.decode("utf-8", errors="strict") + except UnicodeDecodeError as error: + safe_prefix = raw[: error.start].decode("utf-8", errors="strict") + normalized_prefix = safe_prefix.replace("\r\n", "\n").replace("\r", "\n") + return _failure( + CompilationFailureCode.INVALID_UTF8, + normalized_prefix, + len(normalized_prefix), + ) + normalized = decoded.replace("\r\n", "\n").replace("\r", "\n") + if any( + character == "\x00" or 0x7F <= ord(character) <= 0x9F + for character in normalized + ): + offset = next( + index + for index, character in enumerate(normalized) + if character == "\x00" or 0x7F <= ord(character) <= 0x9F + ) + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + normalized, + offset, + UnsupportedConstruct.CONTROL_CHARACTER, + ) + return normalized.rstrip("\n") + "\n" + + +def _line_layout(text: str) -> tuple[list[str], list[int]]: + lines = text.removesuffix("\n").split("\n") + starts: list[int] = [] + offset = 0 + for line in lines: + starts.append(offset) + offset += len(line) + 1 + return lines, starts + + +def _line_end(lines: list[str], starts: list[int], index: int) -> int: + return starts[index] + len(lines[index]) + + +def _table_cells(extractor: _ElementExtractor, line: str) -> tuple[str, ...]: + return tuple(extractor._table_cells(line)) + + +def _frontmatter_end(lines: list[str]) -> int | None: + if not lines or lines[0].strip() != "---": + return None + for index in range(1, len(lines)): + if lines[index].strip() == "---": + if index == 1: + return -1 + for line in lines[1:index]: + if not line.strip(): + continue + if line.startswith((" ", "\t")): + if re.fullmatch(r"[ \t]+-[ \t]+.+", line) is None: + return -1 + elif _FRONTMATTER_FIELD.fullmatch(line) is None: + return -1 + return index + return -1 + + +def _blocks(text: str) -> tuple[_Block, ...] | CompilationFailure: + lines, starts = _line_layout(text) + extractor = cast( + _ElementExtractor, + cast(Any, MarkdownElementExtractor)(text.removesuffix("\n")), + ) + blocks: list[_Block] = [] + index = 0 + frontmatter_end = _frontmatter_end(lines) + if frontmatter_end == -1: + return _failure( + CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE, + text, + 0, + ) + if frontmatter_end is not None: + blocks.append( + _Block( + kind=SectionKind.PARAGRAPH, + start=starts[0], + end=_line_end(lines, starts, frontmatter_end), + ) + ) + index = frontmatter_end + 1 + + while index < len(lines): + line = lines[index] + if not line.strip(): + index += 1 + continue + + atx = _ATX_HEADING.fullmatch(line) + if atx is not None: + blocks.append( + _Block( + kind=SectionKind.HEADING, + start=starts[index], + end=_line_end(lines, starts, index), + level=len(atx.group(1)), + ) + ) + index += 1 + continue + if index + 1 < len(lines) and _SETEXT.fullmatch(lines[index + 1]): + blocks.append( + _Block( + kind=SectionKind.HEADING, + start=starts[index], + end=_line_end(lines, starts, index + 1), + level=1 if lines[index + 1].lstrip().startswith("=") else 2, + ) + ) + index += 2 + continue + + fence = extractor._get_fence_marker(line) + if fence is not None: + fence_character, fence_length = fence + closing = index + 1 + while closing < len(lines) and not extractor._is_closing_fence( + lines[closing], fence_character, fence_length + ): + closing += 1 + if closing >= len(lines): + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + text, + starts[index], + UnsupportedConstruct.CODE_BLOCK, + ) + language = line.lstrip()[fence_length:].strip() or None + if language is not None and ( + len(language) > MARKDOWN_CODE_LANGUAGE_MAX_LENGTH + or any(character.isspace() for character in language) + ): + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + text, + starts[index], + UnsupportedConstruct.CODE_BLOCK, + ) + body = "\n".join(lines[index + 1 : closing]) + if not body or body.isspace(): + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + text, + starts[index], + UnsupportedConstruct.CODE_BLOCK, + ) + blocks.append( + _Block( + kind=SectionKind.FENCED_CODE, + start=starts[index], + end=_line_end(lines, starts, closing), + code_language=language, + code_body=body, + ) + ) + index = closing + 1 + continue + + if ( + index + 1 < len(lines) + and extractor._is_table_row(line) + and extractor._is_table_separator_row(lines[index + 1]) + ): + header = _table_cells(extractor, line) + width = len(header) + rows: list[tuple[str, ...]] = [] + end = index + 2 + while end < len(lines) and extractor._is_table_row(lines[end]): + row = _table_cells(extractor, lines[end]) + if len(row) != width: + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + text, + starts[end], + UnsupportedConstruct.TABLE, + ) + rows.append(row) + end += 1 + if not rows or any(not cell for row in (header, *rows) for cell in row): + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + text, + starts[index], + UnsupportedConstruct.TABLE, + ) + blocks.append( + _Block( + kind=SectionKind.TABLE, + start=starts[index], + end=_line_end(lines, starts, end - 1), + table_header=header, + table_rows=tuple(rows), + ) + ) + index = end + continue + + list_match = _LIST_ITEM.fullmatch(line) + if list_match is not None: + items: list[str] = [] + end = index + ordered = re.match(r"[0-9]+[.)]", line.lstrip()) is not None + while end < len(lines): + candidate = lines[end] + match = _LIST_ITEM.fullmatch(candidate) + if match is not None: + items.append(match.group(2)) + end += 1 + continue + if candidate.strip() and candidate.startswith((" ", "\t")): + end += 1 + continue + break + blocks.append( + _Block( + kind=SectionKind.LIST, + start=starts[index], + end=_line_end(lines, starts, end - 1), + list_ordered=ordered, + list_items=tuple(items), + ) + ) + index = end + continue + + if line.lstrip().startswith("<"): + end = index + 1 + while end < len(lines) and lines[end].strip(): + end += 1 + blocks.append( + _Block( + kind=SectionKind.PARAGRAPH, + start=starts[index], + end=_line_end(lines, starts, end - 1), + ) + ) + index = end + continue + + end = index + 1 + while end < len(lines) and lines[end].strip(): + if ( + _ATX_HEADING.fullmatch(lines[end]) + or extractor._get_fence_marker(lines[end]) is not None + or _LIST_ITEM.fullmatch(lines[end]) + or ( + end + 1 < len(lines) + and extractor._is_table_row(lines[end]) + and extractor._is_table_separator_row(lines[end + 1]) + ) + ): + break + end += 1 + blocks.append( + _Block( + kind=SectionKind.PARAGRAPH, + start=starts[index], + end=_line_end(lines, starts, end - 1), + ) + ) + index = end + return tuple(blocks) + + +def _heading_text(source: str, level: int) -> str: + lines = source.split("\n") + if len(lines) == 2 and _SETEXT.fullmatch(lines[1]): + return lines[0].strip() + match = _ATX_HEADING.fullmatch(source) + if match is None: + raise ValueError("rich heading source must match its declared syntax") + assert len(match.group(1)) == level + return match.group(2).strip() + + +def _split_ranges(source: str, capacity: int) -> tuple[tuple[int, int], ...]: + tokens = tuple(_TOKEN.finditer(source)) + if not tokens or capacity < 1: + return () + return tuple( + (tokens[index].start(), tokens[min(index + capacity, len(tokens)) - 1].end()) + for index in range(0, len(tokens), capacity) + ) + + +def _section( + block: _Block, + source: str, + path: StructuralPath, + position: SourceSpan, +) -> ParsedSection: + return ParsedSection( + kind=block.kind, + text=( + _heading_text(source, block.level) + if block.kind is SectionKind.HEADING and block.level is not None + else source + ), + path=path, + position=position, + level=block.level, + list_ordered=block.list_ordered, + list_items=block.list_items, + code_language=block.code_language, + code_body=block.code_body, + table_header=block.table_header, + table_rows=block.table_rows, + ) + + +def compile_rich_markdown( + source: bytes, + config: MarkdownCompilerConfig, + *, + token_ceiling: int = MARKDOWN_RICH_TOKEN_CEILING, +) -> CompilationOutcome: + """Compile exact bytes through the explicit rich v3 representation.""" + + if type(source) is not bytes: + raise TypeError("rich Markdown compiler source must be exact bytes") + if type(config) is not MarkdownCompilerConfig: + raise TypeError("rich Markdown compiler config must be exact") + if config.version != "markdown-config-v3": + raise ValueError("rich Markdown compiler requires markdown-config-v3") + if type(token_ceiling) is not int or token_ceiling < 1: + raise ValueError("rich Markdown token ceiling must be positive") + normalized = _normalize(source) + if isinstance(normalized, CompilationFailure): + return normalized + blocks = _blocks(normalized) + if isinstance(blocks, CompilationFailure): + return blocks + if not blocks: + return _failure( + CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE, + normalized, + 0, + ) + + headings: list[ParsedSection] = [] + counters: dict[tuple[tuple[str, ...], SectionKind], int] = {} + ordinals: dict[SectionKind, int] = {} + sections: list[ParsedSection] = [] + fragments: list[CompiledFragment] = [] + for block in blocks: + full_source = normalized[block.start : block.end] + if block.kind is SectionKind.HEADING: + assert block.level is not None + headings = [ + heading + for heading in headings + if heading.level is not None and heading.level < block.level + ] + parents = tuple(headings) + parent_path = parents[-1].path.segments if parents else ("document",) + ancestry = "\n\n".join( + f"{'#' * heading.level} {heading.text}" + for heading in parents + if heading.level is not None + ) + capacity = token_ceiling - rich_token_count(ancestry) + if block.kind is SectionKind.HEADING: + ranges: tuple[tuple[int, int], ...] = ((0, len(full_source)),) + if rich_token_count(full_source) > capacity: + return _failure( + CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE, + normalized, + block.start, + ) + else: + ranges = _split_ranges(full_source, capacity) + if not ranges: + return _failure( + CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE, + normalized, + block.start, + ) + for relative_start, relative_end in ranges: + source_text = full_source[relative_start:relative_end] + key = (parent_path, block.kind) + counters[key] = counters.get(key, 0) + 1 + path = StructuralPath( + parent_path + (f"{block.kind.value}[{counters[key]}]",) + ) + position = _span( + normalized, + block.start + relative_start, + block.start + relative_end, + ) + section = _section(block, source_text, path, position) + ordinals[block.kind] = ordinals.get(block.kind, 0) + 1 + contextual = f"{ancestry}\n\n{source_text}" if ancestry else source_text + phrases = tuple(dict.fromkeys((source_text, section.text))) + fragments.append( + CompiledFragment( + fragment_ref=( + f"fragment:{block.kind.value}:{ordinals[block.kind]}" + ), + kind=block.kind, + path=path, + position=position, + source_text=source_text, + contextual_text=contextual, + parent_headings=parents, + search_phrases=phrases, + ) + ) + sections.append(section) + if block.kind is SectionKind.HEADING: + headings.append(section) + provenance = CompilationProvenance( + compiler_version=MARKDOWN_COMPILER_V3_VERSION, + config_version=config.version, + canonicalization_profile=MARKDOWN_RICH_CANONICALIZATION_PROFILE, + compilation_digest_profile=MARKDOWN_RICH_COMPILATION_DIGEST_PROFILE, + ) + return ParsedDocument.rich_v3( + canonical_text=normalized, + sections=tuple(sections), + fragments=tuple(fragments), + provenance=provenance, + ) diff --git a/applications/compiler_runner.py b/applications/compiler_runner.py new file mode 100644 index 00000000..ac908dd9 --- /dev/null +++ b/applications/compiler_runner.py @@ -0,0 +1,257 @@ +"""Pure subprocess boundary for the registered rich Markdown compiler.""" + +from __future__ import annotations + +import argparse +import base64 +import hashlib +import json +import re +import subprocess +import sys +from pathlib import Path, PurePath +from typing import Final, cast + +from adapters.parsers.ragflow_markdown import compile_rich_markdown, rich_token_count +from engine.supply import ( + MARKDOWN_RICH_TOKEN_CEILING, + CompilationFailure, + CompilationFailureCode, + CompilationOutcome, + MarkdownCompilerConfig, + ParsedDocument, + SourcePoint, + UnsupportedConstruct, + canonicalize_parsed_document, + deserialize_parsed_document, +) + +_RUNNER_MODULE: Final = "applications.compiler_runner" + + +def _failure_document(failure: CompilationFailure) -> dict[str, object]: + return { + "code": failure.code.value, + "construct": failure.construct.value if failure.construct is not None else None, + "position": ( + { + "line": failure.position.line, + "column": failure.position.column, + "byteOffset": failure.position.byte_offset, + } + if failure.position is not None + else None + ), + } + + +def _failure_from_document(value: object) -> CompilationFailure: + if type(value) is not dict: + raise ValueError("runner failure must be an object") + document = cast(dict[str, object], value) + position_value = document["position"] + position = None + if type(position_value) is dict: + point = cast(dict[str, object], position_value) + position = SourcePoint( + line=cast(int, point["line"]), + column=cast(int, point["column"]), + byte_offset=cast(int, point["byteOffset"]), + ) + construct_value = document["construct"] + return CompilationFailure( + code=CompilationFailureCode(cast(str, document["code"])), + position=position, + construct=( + UnsupportedConstruct(cast(str, construct_value)) + if construct_value is not None + else None + ), + ) + + +def compile_in_compiler_runner( + source: bytes, + config: MarkdownCompilerConfig, + *, + token_ceiling: int = MARKDOWN_RICH_TOKEN_CEILING, +) -> CompilationOutcome: + """Compile once in a fresh, state-free ContextEngine-owned process.""" + + if type(source) is not bytes: + raise TypeError("compiler-runner source must be exact bytes") + if type(config) is not MarkdownCompilerConfig: + raise TypeError("compiler-runner config must be exact") + completed = subprocess.run( + [ + sys.executable, + "-m", + _RUNNER_MODULE, + "--compile", + "--config", + config.version, + "--token-ceiling", + str(token_ceiling), + ], + input=source, + capture_output=True, + check=False, + ) + if completed.returncode != 0: + raise RuntimeError("compiler-runner process failed") + envelope = json.loads(completed.stdout) + if type(envelope) is not dict: + raise RuntimeError("compiler-runner emitted an invalid envelope") + document = cast(dict[str, object], envelope) + if document.get("outcome") == "parsed": + encoded = document.get("document") + if type(encoded) is not str: + raise RuntimeError("compiler-runner parsed payload is invalid") + return deserialize_parsed_document(base64.b64decode(encoded, validate=True)) + if document.get("outcome") == "failure": + return _failure_from_document(document.get("failure")) + raise RuntimeError("compiler-runner outcome is invalid") + + +def _emit(source: bytes, config: MarkdownCompilerConfig, token_ceiling: int) -> None: + outcome = compile_rich_markdown(source, config, token_ceiling=token_ceiling) + if type(outcome) is ParsedDocument: + envelope: dict[str, object] = { + "outcome": "parsed", + "document": base64.b64encode( + canonicalize_parsed_document(outcome) + ).decode("ascii"), + } + else: + assert type(outcome) is CompilationFailure + envelope = {"outcome": "failure", "failure": _failure_document(outcome)} + sys.stdout.write(json.dumps(envelope, sort_keys=True, separators=(",", ":"))) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--compile", action="store_true") + parser.add_argument("--config", default="markdown-config-v3") + parser.add_argument( + "--token-ceiling", + type=int, + default=MARKDOWN_RICH_TOKEN_CEILING, + ) + parser.add_argument("--acceptance-report", action="store_true") + parser.add_argument("--root", type=Path) + parser.add_argument( + "--output", + type=Path, + default=Path(".context-engine/compiler-runner-acceptance.json"), + ) + return parser + + +_CONSTRUCT_PATTERNS: Final[dict[str, re.Pattern[str]]] = { + "atxHeadings": re.compile(r"(?m)^ {0,3}#{1,6}[ \t]+"), + "setextHeadings": re.compile(r"(?m)^.+\n {0,3}(?:=+|-+)[ \t]*$"), + "lists": re.compile(r"(?m)^[ \t]*(?:[-+*]|[0-9]+[.)])[ \t]+"), + "fencedCode": re.compile(r"(?m)^ {0,3}(?:`{3,}|~{3,})"), + "tables": re.compile(r"(?m)^.*\|.*\n[ \t]*\|?[ :|-]+\|"), + "wikilinks": re.compile(r"(?[ \t]*\[![A-Za-z0-9_-]+]"), + "inlineMath": re.compile(r"(? tuple[Path, ...]: + if not root.is_dir(): + raise ValueError("acceptance root must be a directory") + return tuple( + sorted( + (path for path in root.rglob("*.md") if path.is_file()), + key=lambda path: PurePath(*path.relative_to(root).parts).as_posix(), + ) + ) + + +def _acceptance_report(root: Path, token_ceiling: int) -> dict[str, object]: + accepted = 0 + refused = 0 + digests: list[str] = [] + maximum = 0 + histogram = {name: 0 for name in _CONSTRUCT_PATTERNS} + for path in _safe_markdown_files(root): + source = path.read_bytes() + try: + inspected = source.removeprefix(b"\xef\xbb\xbf").decode("utf-8") + except UnicodeDecodeError: + inspected = "" + normalized = inspected.replace("\r\n", "\n").replace("\r", "\n") + for name, pattern in _CONSTRUCT_PATTERNS.items(): + histogram[name] += len(pattern.findall(normalized)) + outcome = compile_rich_markdown( + source, + MarkdownCompilerConfig("markdown-config-v3"), + token_ceiling=token_ceiling, + ) + if type(outcome) is ParsedDocument: + accepted += 1 + digests.append(outcome.compilation_digest) + maximum = max( + maximum, + *( + rich_token_count(fragment.contextual_text) + for fragment in outcome.fragments + ), + ) + else: + refused += 1 + aggregate = hashlib.sha256() + for digest in sorted(digests): + aggregate.update(bytes.fromhex(digest)) + total = accepted + refused + return { + "schemaVersion": "compiler-runner-acceptance-v1", + "compilerVersion": "context-engine-markdown-v3", + "configVersion": "markdown-config-v3", + "documents": {"accepted": accepted, "refused": refused, "total": total}, + "constructHistogram": histogram, + "tokenCeiling": token_ceiling, + "maxFragmentTokenCount": maximum, + "aggregateCompilationDigest": aggregate.hexdigest(), + } + + +def _write_acceptance_report(root: Path, output: Path, token_ceiling: int) -> None: + if ".context-engine" not in output.parts: + raise ValueError("acceptance reports must be written under .context-engine") + report = _acceptance_report(root, token_ceiling) + serialized = json.dumps(report, sort_keys=True, separators=(",", ":")) + "\n" + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(serialized, encoding="utf-8") + sys.stdout.write(serialized) + + +def main() -> None: + args = _parser().parse_args() + if args.compile: + _emit( + sys.stdin.buffer.read(), + MarkdownCompilerConfig(args.config), + cast(int, args.token_ceiling), + ) + return + if args.acceptance_report: + if args.root is None: + raise SystemExit("--acceptance-report requires --root") + _write_acceptance_report( + cast(Path, args.root), + cast(Path, args.output), + cast(int, args.token_ceiling), + ) + return + raise SystemExit("one runner operation is required") + + +if __name__ == "__main__": + main() diff --git a/docs/contracts/compiler-runner-acceptance-v1.schema.json b/docs/contracts/compiler-runner-acceptance-v1.schema.json new file mode 100644 index 00000000..b3e87a34 --- /dev/null +++ b/docs/contracts/compiler-runner-acceptance-v1.schema.json @@ -0,0 +1,70 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://context-engine.invalid/schemas/compiler-runner-acceptance-v1.json", + "title": "CompilerRunnerAcceptanceReportV1", + "type": "object", + "additionalProperties": false, + "required": [ + "aggregateCompilationDigest", + "compilerVersion", + "configVersion", + "constructHistogram", + "documents", + "maxFragmentTokenCount", + "schemaVersion", + "tokenCeiling" + ], + "properties": { + "aggregateCompilationDigest": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "compilerVersion": {"const": "context-engine-markdown-v3"}, + "configVersion": {"const": "markdown-config-v3"}, + "constructHistogram": { + "type": "object", + "additionalProperties": false, + "required": [ + "atxHeadings", + "callouts", + "embeds", + "fencedCode", + "footnotes", + "frontmatter", + "htmlBlocks", + "inlineMath", + "lists", + "setextHeadings", + "tables", + "wikilinks" + ], + "properties": { + "atxHeadings": {"type": "integer", "minimum": 0}, + "callouts": {"type": "integer", "minimum": 0}, + "embeds": {"type": "integer", "minimum": 0}, + "fencedCode": {"type": "integer", "minimum": 0}, + "footnotes": {"type": "integer", "minimum": 0}, + "frontmatter": {"type": "integer", "minimum": 0}, + "htmlBlocks": {"type": "integer", "minimum": 0}, + "inlineMath": {"type": "integer", "minimum": 0}, + "lists": {"type": "integer", "minimum": 0}, + "setextHeadings": {"type": "integer", "minimum": 0}, + "tables": {"type": "integer", "minimum": 0}, + "wikilinks": {"type": "integer", "minimum": 0} + } + }, + "documents": { + "type": "object", + "additionalProperties": false, + "required": ["accepted", "refused", "total"], + "properties": { + "accepted": {"type": "integer", "minimum": 0}, + "refused": {"type": "integer", "minimum": 0}, + "total": {"type": "integer", "minimum": 0} + } + }, + "maxFragmentTokenCount": {"type": "integer", "minimum": 0}, + "schemaVersion": {"const": "compiler-runner-acceptance-v1"}, + "tokenCeiling": {"type": "integer", "minimum": 1} + } +} diff --git a/engine/supply/__init__.py b/engine/supply/__init__.py index b46af33e..b8f234ab 100644 --- a/engine/supply/__init__.py +++ b/engine/supply/__init__.py @@ -57,8 +57,12 @@ MARKDOWN_COMPILATION_DIGEST_PROFILE, MARKDOWN_COMPILATION_DIGEST_V1_PROFILE, MARKDOWN_COMPILER_V1_VERSION, + MARKDOWN_COMPILER_V3_VERSION, MARKDOWN_COMPILER_VERSION, MARKDOWN_CONTENT_HASH_PROFILE, + MARKDOWN_RICH_CANONICALIZATION_PROFILE, + MARKDOWN_RICH_COMPILATION_DIGEST_PROFILE, + MARKDOWN_RICH_TOKEN_CEILING, CompilationFailure, CompilationFailureCode, CompilationOutcome, @@ -75,6 +79,7 @@ StructuralPath, UnsupportedConstruct, canonicalize_parsed_document, + deserialize_parsed_document, ) __all__ = [ @@ -86,8 +91,12 @@ "MARKDOWN_COMPILATION_DIGEST_PROFILE", "MARKDOWN_COMPILATION_DIGEST_V1_PROFILE", "MARKDOWN_COMPILER_VERSION", + "MARKDOWN_COMPILER_V3_VERSION", "MARKDOWN_COMPILER_V1_VERSION", "MARKDOWN_CONTENT_HASH_PROFILE", + "MARKDOWN_RICH_CANONICALIZATION_PROFILE", + "MARKDOWN_RICH_COMPILATION_DIGEST_PROFILE", + "MARKDOWN_RICH_TOKEN_CEILING", "WORKER_LEASE_ACTOR_KIND", "WORKER_LEASE_OPERATION", "SUPPLY_CONNECTOR_WORKER_LEASE_OPERATION", @@ -141,6 +150,7 @@ "WorkerLeaseToken", "generate_worker_lease_nonce", "canonicalize_parsed_document", + "deserialize_parsed_document", "worker_lease_digest", "worker_lease_nonce_digest", "validate_embedding_batch", diff --git a/engine/supply/markdown.py b/engine/supply/markdown.py index 588b3a62..18ced969 100644 --- a/engine/supply/markdown.py +++ b/engine/supply/markdown.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import re from dataclasses import dataclass from enum import StrEnum @@ -12,15 +13,20 @@ MARKDOWN_COMPILER_V1_VERSION: Final = "context-engine-markdown-v1" MARKDOWN_COMPILER_VERSION: Final = "context-engine-markdown-v2" +MARKDOWN_COMPILER_V3_VERSION: Final = "context-engine-markdown-v3" ACTIVE_FILE_IMPORT_MARKDOWN_CONFIG_VERSION: Final = "markdown-config-v1" MARKDOWN_CANONICALIZATION_V1_PROFILE: Final = "markdown-heading-paragraph-v1" MARKDOWN_CANONICALIZATION_PROFILE: Final = "markdown-structural-units-v2" MARKDOWN_CONTENT_HASH_PROFILE: Final = "sha256-canonical-utf8-v1" MARKDOWN_COMPILATION_DIGEST_V1_PROFILE: Final = "rfc8785-sha256-v1" MARKDOWN_COMPILATION_DIGEST_PROFILE: Final = "rfc8785-sha256-v2" +MARKDOWN_RICH_CANONICALIZATION_PROFILE: Final = "markdown-rich-structural-v3" +MARKDOWN_RICH_COMPILATION_DIGEST_PROFILE: Final = "rfc8785-sha256-v3" MARKDOWN_CODE_LANGUAGE_MAX_LENGTH: Final = 64 +MARKDOWN_RICH_TOKEN_CEILING: Final = 512 _COMPILATION_DIGEST_V1_DOMAIN: Final = b"context-engine.markdown-compilation.v1\x00" _COMPILATION_DIGEST_DOMAIN: Final = b"context-engine.markdown-compilation.v2\x00" +_COMPILATION_DIGEST_V3_DOMAIN: Final = b"context-engine.markdown-compilation.v3\x00" _MAX_VERSION_LENGTH: Final = 128 @@ -293,6 +299,10 @@ def __post_init__(self) -> None: MARKDOWN_COMPILATION_DIGEST_V1_PROFILE, ), (MARKDOWN_CANONICALIZATION_PROFILE, MARKDOWN_COMPILATION_DIGEST_PROFILE), + ( + MARKDOWN_RICH_CANONICALIZATION_PROFILE, + MARKDOWN_RICH_COMPILATION_DIGEST_PROFILE, + ), }: raise ValueError("Markdown canonicalization and digest profiles must match") if self.content_hash_profile != MARKDOWN_CONTENT_HASH_PROFILE: @@ -302,6 +312,10 @@ def __post_init__(self) -> None: def is_structural_v2(self) -> bool: return self.canonicalization_profile == MARKDOWN_CANONICALIZATION_PROFILE + @property + def is_rich_v3(self) -> bool: + return self.canonicalization_profile == MARKDOWN_RICH_CANONICALIZATION_PROFILE + @dataclass(frozen=True, slots=True) class ParsedDocument: @@ -338,7 +352,9 @@ def __post_init__(self) -> None: raise TypeError("parsed document warnings must be typed immutable values") if self.warnings: raise ValueError("the active Markdown compiler emits no warnings") - if self.provenance.is_structural_v2: + if self.provenance.is_rich_v3: + _validate_rich_content(self.canonical_text, self.sections, self.fragments) + elif self.provenance.is_structural_v2: _validate_structural_content( self.canonical_text, self.sections, @@ -426,6 +442,37 @@ def structural_v2( fragments=fragments, ) + @classmethod + def rich_v3( + cls, + *, + canonical_text: str, + sections: tuple[ParsedSection, ...], + fragments: tuple[CompiledFragment, ...], + provenance: CompilationProvenance, + ) -> ParsedDocument: + """Build one self-validating rich Markdown compilation result.""" + + if cls is not ParsedDocument or not provenance.is_rich_v3: + raise TypeError("rich ParsedDocument construction requires v3 provenance") + content_hash = sha256(canonical_text.encode("utf-8")).hexdigest() + compilation_digest = _compilation_digest( + canonical_text=canonical_text, + sections=sections, + fragments=fragments, + content_hash=content_hash, + provenance=provenance, + warnings=(), + ) + return ParsedDocument( + canonical_text=canonical_text, + sections=sections, + content_hash=content_hash, + compilation_digest=compilation_digest, + provenance=provenance, + fragments=fragments, + ) + class CompilationFailureCode(StrEnum): INVALID_UTF8 = "invalid_utf8" @@ -649,7 +696,7 @@ def _compilation_document( for warning in warnings ], } - if provenance.is_structural_v2: + if provenance.is_structural_v2 or provenance.is_rich_v3: document["fragments"] = [ _fragment_document(fragment) for fragment in fragments ] @@ -684,11 +731,12 @@ def _compilation_digest( provenance=provenance, warnings=warnings, ) - domain = ( - _COMPILATION_DIGEST_DOMAIN - if provenance.is_structural_v2 - else _COMPILATION_DIGEST_V1_DOMAIN - ) + if provenance.is_rich_v3: + domain = _COMPILATION_DIGEST_V3_DOMAIN + elif provenance.is_structural_v2: + domain = _COMPILATION_DIGEST_DOMAIN + else: + domain = _COMPILATION_DIGEST_V1_DOMAIN return sha256(domain + rfc8785.dumps(cast(Any, document))).hexdigest() @@ -928,6 +976,57 @@ def _validate_structural_content( raise ValueError("structural sections cannot omit trailing canonical content") +def _validate_rich_content( + canonical_text: str, + sections: tuple[ParsedSection, ...], + fragments: tuple[CompiledFragment, ...], +) -> None: + if ( + "\r" in canonical_text + or canonical_text.startswith("\ufeff") + or canonical_text.endswith("\n\n") + ): + raise ValueError("rich Markdown must use canonical transport text") + if not sections or not fragments or len(sections) != len(fragments): + raise ValueError("rich Markdown requires one Fragment per parsed section") + canonical_bytes = canonical_text.encode("utf-8") + prior_end = 0 + refs: set[str] = set() + for section, fragment in zip(sections, fragments, strict=True): + if ( + fragment.kind is not section.kind + or fragment.path != section.path + or fragment.position != section.position + or fragment.fragment_ref in refs + or section.position.start.byte_offset < prior_end + ): + raise ValueError("rich Fragment lineage must match source order") + if _expected_point(canonical_text, section.position.start.byte_offset) != ( + section.position.start + ) or _expected_point(canonical_text, section.position.end.byte_offset) != ( + section.position.end + ): + raise ValueError("rich source coordinates must match UTF-8 offsets") + source = canonical_bytes[ + section.position.start.byte_offset : section.position.end.byte_offset + ].decode("utf-8") + if source != fragment.source_text: + raise ValueError("rich Fragment source text must match its span") + if section.text != source and section.kind is not SectionKind.HEADING: + raise ValueError("rich section text must match its span") + omitted = canonical_bytes[ + prior_end : section.position.start.byte_offset + ] + if any(byte not in b" \t\n" for byte in omitted): + raise ValueError("rich sections cannot omit non-whitespace source") + if fragment.contextual_text != _expected_contextual_text(fragment): + raise ValueError("rich Fragment context must be exact heading ancestry") + refs.add(fragment.fragment_ref) + prior_end = section.position.end.byte_offset + if any(byte not in b" \t\n" for byte in canonical_bytes[prior_end:]): + raise ValueError("rich sections cannot omit trailing source") + + def _validate_issue_22_content( canonical_text: str, sections: tuple[ParsedSection, ...], @@ -987,3 +1086,127 @@ def canonicalize_parsed_document(document: ParsedDocument) -> bytes: canonical = _document_without_digest(document) canonical["compilationDigest"] = document.compilation_digest return rfc8785.dumps(cast(Any, canonical)) + + +def _source_point_from_document(value: object) -> SourcePoint: + if type(value) is not dict: + raise ValueError("source point document must be an object") + document = cast(dict[str, object], value) + if set(document) != {"line", "column", "byteOffset"}: + raise ValueError("source point document has unexpected fields") + return SourcePoint( + line=cast(int, document["line"]), + column=cast(int, document["column"]), + byte_offset=cast(int, document["byteOffset"]), + ) + + +def _source_span_from_document(value: object) -> SourceSpan: + if type(value) is not dict: + raise ValueError("source span document must be an object") + document = cast(dict[str, object], value) + if set(document) != {"start", "end"}: + raise ValueError("source span document has unexpected fields") + return SourceSpan( + start=_source_point_from_document(document["start"]), + end=_source_point_from_document(document["end"]), + ) + + +def _section_from_document(value: object) -> ParsedSection: + if type(value) is not dict: + raise ValueError("section document must be an object") + document = cast(dict[str, object], value) + kind = SectionKind(cast(str, document["kind"])) + path = StructuralPath(tuple(cast(list[str], document["path"]))) + return ParsedSection( + kind=kind, + text=cast(str, document["text"]), + path=path, + position=_source_span_from_document(document["position"]), + level=cast(int | None, document.get("level")), + list_ordered=cast(bool | None, document.get("ordered")), + list_items=tuple(cast(list[str], document.get("items", []))), + code_language=cast(str | None, document.get("language")), + code_body=cast(str | None, document.get("code")), + table_header=tuple(cast(list[str], document.get("header", []))), + table_rows=tuple( + tuple(row) for row in cast(list[list[str]], document.get("rows", [])) + ), + ) + + +def _fragment_from_document( + value: object, + heading_by_key: dict[tuple[str, ...], ParsedSection], +) -> CompiledFragment: + if type(value) is not dict: + raise ValueError("Fragment document must be an object") + document = cast(dict[str, object], value) + parents: list[ParsedSection] = [] + for parent_value in cast(list[object], document["parentHeadings"]): + if type(parent_value) is not dict: + raise ValueError("parent heading document must be an object") + parent = cast(dict[str, object], parent_value) + key = tuple(cast(list[str], parent["path"])) + heading = heading_by_key.get(key) + if heading is None: + raise ValueError("Fragment parent heading must name a parsed section") + parents.append(heading) + return CompiledFragment( + fragment_ref=cast(str, document["fragmentRef"]), + kind=SectionKind(cast(str, document["kind"])), + path=StructuralPath(tuple(cast(list[str], document["path"]))), + position=_source_span_from_document(document["position"]), + source_text=cast(str, document["sourceText"]), + contextual_text=cast(str, document["contextualText"]), + parent_headings=tuple(parents), + search_phrases=tuple(cast(list[str], document["searchPhrases"])), + ) + + +def deserialize_parsed_document(payload: bytes) -> ParsedDocument: + """Deserialize runner bytes into the existing self-validating contract.""" + + if type(payload) is not bytes: + raise TypeError("parsed document payload must be exact bytes") + raw = json.loads(payload) + if type(raw) is not dict: + raise ValueError("parsed document payload must contain one object") + document = cast(dict[str, object], raw) + provenance_value = document["provenance"] + if type(provenance_value) is not dict: + raise ValueError("parsed document provenance must be an object") + provenance_document = cast(dict[str, object], provenance_value) + provenance = CompilationProvenance( + compiler_version=cast(str, provenance_document["compilerVersion"]), + config_version=cast(str, provenance_document["configVersion"]), + canonicalization_profile=cast( + str, provenance_document["canonicalizationProfile"] + ), + content_hash_profile=cast(str, provenance_document["contentHashProfile"]), + compilation_digest_profile=cast( + str, provenance_document["compilationDigestProfile"] + ), + ) + sections = tuple( + _section_from_document(value) + for value in cast(list[object], document["sections"]) + ) + heading_by_key = { + section.path.segments: section + for section in sections + if section.kind is SectionKind.HEADING + } + fragments = tuple( + _fragment_from_document(value, heading_by_key) + for value in cast(list[object], document.get("fragments", [])) + ) + return ParsedDocument( + canonical_text=cast(str, document["canonicalText"]), + sections=sections, + content_hash=cast(str, document["contentHash"]), + compilation_digest=cast(str, document["compilationDigest"]), + provenance=provenance, + fragments=fragments, + ) diff --git a/tests/fixtures/markdown/rich-code-tables.md b/tests/fixtures/markdown/rich-code-tables.md new file mode 100644 index 00000000..323bd36a --- /dev/null +++ b/tests/fixtures/markdown/rich-code-tables.md @@ -0,0 +1,12 @@ +# Technical notes + +````python +print("outer") +``` +print("still outer") +```` + +| Name | State | +| :--- | ---: | +| alpha | ready | +| beta | held | diff --git a/tests/fixtures/markdown/rich-frontmatter.md b/tests/fixtures/markdown/rich-frontmatter.md new file mode 100644 index 00000000..78f6ab7e --- /dev/null +++ b/tests/fixtures/markdown/rich-frontmatter.md @@ -0,0 +1,8 @@ +--- +category: handbook +aliases: + - field guide +--- +# Handbook + +Opening paragraph with [[guide|the guide]] and ![[diagram]]. diff --git a/tests/fixtures/markdown/rich-headings-lists.md b/tests/fixtures/markdown/rich-headings-lists.md new file mode 100644 index 00000000..fd3ae313 --- /dev/null +++ b/tests/fixtures/markdown/rich-headings-lists.md @@ -0,0 +1,12 @@ +Handbook +======== + +## Repeated + +- first item + - nested item + 1. ordered child + +## Repeated + +Closing paragraph. diff --git a/tests/fixtures/markdown/rich-inline-html.md b/tests/fixtures/markdown/rich-inline-html.md new file mode 100644 index 00000000..3bebfc43 --- /dev/null +++ b/tests/fixtures/markdown/rich-inline-html.md @@ -0,0 +1,12 @@ +# Reference + +> [!NOTE] +> A callout with inline math $x^2 + y^2$. + +Term with a footnote.[^1] + +[^1]: Footnote body. + +
+HTML block body. +
diff --git a/tests/fixtures/markdown/rich-mixed-newlines.hex b/tests/fixtures/markdown/rich-mixed-newlines.hex new file mode 100644 index 00000000..6095309b --- /dev/null +++ b/tests/fixtures/markdown/rich-mixed-newlines.hex @@ -0,0 +1 @@ +23204d697865640d0a0d0a4669727374206c696e652e0a5365636f6e64206c696e652e0d0a diff --git a/tests/unit/test_compiler_runner_acceptance.py b/tests/unit/test_compiler_runner_acceptance.py new file mode 100644 index 00000000..40440159 --- /dev/null +++ b/tests/unit/test_compiler_runner_acceptance.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from adapters.parsers.ragflow_markdown import compile_rich_markdown +from engine.supply import ( + CompilationFailure, + MarkdownCompilerConfig, + ParsedDocument, + SectionKind, +) + +FIXTURES = Path(__file__).parents[1] / "fixtures/markdown" +CONFIG = MarkdownCompilerConfig(version="markdown-config-v3") +RICH_FIXTURES = ( + "rich-frontmatter.md", + "rich-headings-lists.md", + "rich-code-tables.md", + "rich-inline-html.md", + "rich-mixed-newlines.hex", +) + + +def _source(name: str) -> bytes: + path = FIXTURES / name + if path.suffix == ".hex": + return bytes.fromhex(path.read_text(encoding="ascii")) + return path.read_bytes() + + +@pytest.mark.parametrize("fixture", RICH_FIXTURES) +def test_tracked_rich_construct_corpus_compiles_all_or_nothing(fixture: str) -> None: + outcome = compile_rich_markdown(_source(fixture), CONFIG) + + assert not isinstance(outcome, CompilationFailure) + assert type(outcome) is ParsedDocument + assert outcome.fragments + + +@pytest.mark.parametrize("fixture", RICH_FIXTURES) +def test_every_fragment_span_round_trips_to_exact_canonical_utf8( + fixture: str, +) -> None: + outcome = compile_rich_markdown(_source(fixture), CONFIG) + + assert type(outcome) is ParsedDocument + canonical = outcome.canonical_text.encode("utf-8") + for fragment in outcome.fragments: + span = fragment.position + assert canonical[span.start.byte_offset : span.end.byte_offset].decode( + "utf-8" + ) == fragment.source_text + + +def test_nested_setext_and_duplicate_headings_have_stable_ancestry() -> None: + outcome = compile_rich_markdown(_source("rich-headings-lists.md"), CONFIG) + + assert type(outcome) is ParsedDocument + headings = [ + fragment + for fragment in outcome.fragments + if fragment.kind is SectionKind.HEADING + ] + assert [fragment.path.segments for fragment in headings] == [ + ("document", "heading[1]"), + ("document", "heading[1]", "heading[1]"), + ("document", "heading[1]", "heading[2]"), + ] + closing = outcome.fragments[-1] + assert tuple(heading.text for heading in closing.parent_headings) == ( + "Handbook", + "Repeated", + ) + assert closing.path.segments == ( + "document", + "heading[1]", + "heading[2]", + "paragraph[1]", + ) diff --git a/tests/unit/test_compiler_runner_acceptance_report.py b/tests/unit/test_compiler_runner_acceptance_report.py new file mode 100644 index 00000000..be5b81bf --- /dev/null +++ b/tests/unit/test_compiler_runner_acceptance_report.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + + +def test_acceptance_report_is_count_only_deterministic_and_written_under_ignore( + tmp_path: Path, +) -> None: + corpus = tmp_path / "private-corpus" + corpus.mkdir() + (corpus / "one.md").write_text("# One\n\nFirst.\n", encoding="utf-8") + (corpus / "two.md").write_text( + "# Two\n\n| A | B |\n| --- | --- |\n| x | y |\n", + encoding="utf-8", + ) + output = tmp_path / ".context-engine/compiler-runner-acceptance.json" + command = [ + sys.executable, + "-m", + "applications.compiler_runner", + "--acceptance-report", + "--root", + str(corpus), + "--output", + str(output), + ] + + first = subprocess.run(command, check=True, capture_output=True, text=True) + first_bytes = output.read_bytes() + second = subprocess.run(command, check=True, capture_output=True, text=True) + + report = json.loads(first_bytes) + assert report["schemaVersion"] == "compiler-runner-acceptance-v1" + assert report["documents"] == {"accepted": 2, "refused": 0, "total": 2} + assert report["aggregateCompilationDigest"] + assert report["maxFragmentTokenCount"] <= report["tokenCeiling"] + assert report["constructHistogram"]["tables"] == 1 + assert str(corpus) not in first_bytes.decode("utf-8") + assert "one.md" not in first_bytes.decode("utf-8") + assert first_bytes == output.read_bytes() + assert first.stdout == second.stdout diff --git a/tests/unit/test_compiler_runner_bounds.py b/tests/unit/test_compiler_runner_bounds.py new file mode 100644 index 00000000..58cf5118 --- /dev/null +++ b/tests/unit/test_compiler_runner_bounds.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from adapters.parsers.ragflow_markdown import compile_rich_markdown, rich_token_count +from engine.supply import MarkdownCompilerConfig, ParsedDocument + +CONFIG = MarkdownCompilerConfig(version="markdown-config-v3") + + +def test_oversize_blocks_split_under_hard_bound_with_exact_spans_and_ancestry() -> None: + ceiling = 64 + source = ( + "# Root\n\n## Deep\n\n" + + " ".join(f"token{index}" for index in range(17000)) + + "\n" + ).encode() + + outcome = compile_rich_markdown(source, CONFIG, token_ceiling=ceiling) + + assert type(outcome) is ParsedDocument + paragraph_fragments = outcome.fragments[2:] + assert len(paragraph_fragments) > 1 + canonical = outcome.canonical_text.encode("utf-8") + for fragment in paragraph_fragments: + assert rich_token_count(fragment.contextual_text) <= ceiling + assert tuple(heading.text for heading in fragment.parent_headings) == ( + "Root", + "Deep", + ) + span = fragment.position + assert canonical[span.start.byte_offset : span.end.byte_offset].decode( + "utf-8" + ) == fragment.source_text + assert " ".join(fragment.source_text for fragment in paragraph_fragments) == ( + outcome.canonical_text.split("\n\n", 2)[2].rstrip("\n") + ) diff --git a/tests/unit/test_compiler_runner_determinism.py b/tests/unit/test_compiler_runner_determinism.py new file mode 100644 index 00000000..48b16a8d --- /dev/null +++ b/tests/unit/test_compiler_runner_determinism.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from pathlib import Path + +from adapters.parsers.ragflow_markdown import compile_rich_markdown +from applications.compiler_runner import compile_in_compiler_runner +from engine.supply import MarkdownCompilerConfig, ParsedDocument + +FIXTURES = Path(__file__).parents[1] / "fixtures/markdown" +CONFIG = MarkdownCompilerConfig(version="markdown-config-v3") + + +def test_rich_compilation_digest_is_stable_in_process_and_across_runner() -> None: + source = (FIXTURES / "rich-code-tables.md").read_bytes() + + first = compile_rich_markdown(source, CONFIG) + second = compile_rich_markdown(source, CONFIG) + subprocess_result = compile_in_compiler_runner(source, CONFIG) + + assert type(first) is ParsedDocument + assert type(second) is ParsedDocument + assert type(subprocess_result) is ParsedDocument + assert first.compilation_digest == second.compilation_digest + assert first.compilation_digest == subprocess_result.compilation_digest + assert subprocess_result == first diff --git a/tests/unit/test_compiler_runner_refusal.py b/tests/unit/test_compiler_runner_refusal.py new file mode 100644 index 00000000..3be7a19a --- /dev/null +++ b/tests/unit/test_compiler_runner_refusal.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import pytest + +from applications.compiler_runner import compile_in_compiler_runner +from engine.supply import ( + CompilationFailure, + CompilationFailureCode, + MarkdownCompilerConfig, +) + +CONFIG = MarkdownCompilerConfig(version="markdown-config-v3") + + +@pytest.mark.parametrize( + ("source", "expected_code"), + [ + ( + b"# Heading\n\n```python\nprint('open')\n", + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + ), + ( + b"---\nkey: value\n# Missing close\n", + CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE, + ), + (b"# Heading\n\n\xed\xa0\x80\n", CompilationFailureCode.INVALID_UTF8), + ( + b"# Heading\n\ncontains\x00nul\n", + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + ), + (b"# Heading\n\ntruncated \xe4\xb8", CompilationFailureCode.INVALID_UTF8), + ], +) +def test_malformed_source_returns_typed_failure_across_runner_boundary( + source: bytes, + expected_code: CompilationFailureCode, +) -> None: + outcome = compile_in_compiler_runner(source, CONFIG) + + assert type(outcome) is CompilationFailure + assert outcome.code is expected_code + assert outcome.position is not None diff --git a/tests/unit/test_compiler_runner_tables.py b/tests/unit/test_compiler_runner_tables.py new file mode 100644 index 00000000..e0acef4f --- /dev/null +++ b/tests/unit/test_compiler_runner_tables.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from adapters.parsers.ragflow_markdown import compile_rich_markdown +from engine.supply import MarkdownCompilerConfig, ParsedDocument, SectionKind + +CONFIG = MarkdownCompilerConfig(version="markdown-config-v3") + + +def test_every_table_fragment_carries_a_round_tripping_source_span() -> None: + tables = "\n\n".join( + ( + f"| Key | Value |\n| --- | --- |\n" + f"| item-{index} | value-{index} |" + ) + for index in range(40) + ) + source = f"# Tables\n\n{tables}\n".encode() + + outcome = compile_rich_markdown(source, CONFIG) + + assert type(outcome) is ParsedDocument + table_fragments = tuple( + fragment + for fragment in outcome.fragments + if fragment.kind is SectionKind.TABLE + ) + assert len(table_fragments) == 40 + canonical = outcome.canonical_text.encode("utf-8") + for fragment in table_fragments: + span = fragment.position + assert canonical[span.start.byte_offset : span.end.byte_offset].decode( + "utf-8" + ) == fragment.source_text From 0b8b4ae019e63d2230300f3fcc02ff7e54e0c90f Mon Sep 17 00:00:00 2001 From: stone Date: Wed, 29 Jul 2026 22:31:23 +0800 Subject: [PATCH 04/16] fix: enforce compiler runner provenance and bounds (#124) --- THIRD_PARTY_NOTICES.md | 3 +- adapters/parsers/ragflow_markdown.py | 127 +++++++++++++----- applications/compiler_runner.py | 4 +- ...ompile-rich-markdown-in-an-owned-runner.md | 40 ++++-- engine/supply/markdown.py | 39 +++--- pyproject.toml | 1 + tests/unit/test_compiler_runner_acceptance.py | 45 ++++++- tests/unit/test_compiler_runner_bounds.py | 34 ++++- .../unit/test_compiler_runner_determinism.py | 47 ++++++- ...est_compiler_runner_production_boundary.py | 36 +++++ tests/unit/test_compiler_runner_refusal.py | 20 ++- tests/unit/test_compiler_runner_tables.py | 23 ++++ .../test_third_party_ragflow_registration.py | 66 ++++++++- third_party/ragflow/LICENSE.python-markdown | 30 +++++ third_party/ragflow/MODIFICATIONS.md | 33 +++-- third_party/ragflow/UPSTREAM.toml | 3 + third_party/ragflow/sbom.cyclonedx.json | 50 +++++++ 17 files changed, 512 insertions(+), 89 deletions(-) create mode 100644 tests/unit/test_compiler_runner_production_boundary.py create mode 100644 third_party/ragflow/LICENSE.python-markdown create mode 100644 third_party/ragflow/sbom.cyclonedx.json diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 9bc46f66..d181f082 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -10,4 +10,5 @@ The verbatim upstream license and the complete path-level registration ship in `third_party/ragflow/`. The registered parser imports Python-Markdown, which is -distributed under the BSD 3-Clause License. +distributed under the BSD 3-Clause License; its verbatim license ships beside +the registration as `LICENSE.python-markdown`. diff --git a/adapters/parsers/ragflow_markdown.py b/adapters/parsers/ragflow_markdown.py index f42b0d80..2324d3d1 100644 --- a/adapters/parsers/ragflow_markdown.py +++ b/adapters/parsers/ragflow_markdown.py @@ -32,8 +32,13 @@ _ATX_HEADING: Final = re.compile(r"^ {0,3}(#{1,6})[ \t]+(.+?)[ \t]*#*[ \t]*$") _SETEXT: Final = re.compile(r"^ {0,3}(=+|-+)[ \t]*$") _LIST_ITEM: Final = re.compile(r"^([ \t]*)(?:[-+*]|[0-9]+[.)])[ \t]+(.+)$") -_FRONTMATTER_FIELD: Final = re.compile(r"^[A-Za-z0-9_.-]+[ \t]*:[ \t]*(?:.*)$") _TOKEN: Final = re.compile(r"\S+") +_CALLOUT: Final = re.compile(r"^ {0,3}>[ \t]*\[![A-Za-z0-9_-]+][+-]?(?: .*)?$") +_HTML_OPEN: Final = re.compile( + r"^ {0,3}<(?Psection|div|details|summary|table|thead|tbody|tr|" + r"th|td|p|ul|ol|li)\b[^>]*>", + re.IGNORECASE, +) @dataclass(frozen=True, slots=True) @@ -90,10 +95,11 @@ def _failure( def _point(text: str, offset: int) -> SourcePoint: prefix = text[:offset] - last_newline = prefix.rfind("\n") + logical_prefix = prefix.replace("\r\n", "\n").replace("\r", "\n") + last_newline = logical_prefix.rfind("\n") return SourcePoint( - line=prefix.count("\n") + 1, - column=len(prefix[last_newline + 1 :]) + 1, + line=logical_prefix.count("\n") + 1, + column=len(logical_prefix[last_newline + 1 :]) + 1, byte_offset=len(prefix.encode("utf-8")), ) @@ -103,43 +109,46 @@ def _span(text: str, start: int, end: int) -> SourceSpan: def _normalize(source: bytes) -> str | CompilationFailure: - raw = source.removeprefix(_UTF8_BOM) try: - decoded = raw.decode("utf-8", errors="strict") + decoded = source.decode("utf-8", errors="strict") except UnicodeDecodeError as error: - safe_prefix = raw[: error.start].decode("utf-8", errors="strict") - normalized_prefix = safe_prefix.replace("\r\n", "\n").replace("\r", "\n") + safe_prefix = source[: error.start].decode("utf-8", errors="strict") return _failure( CompilationFailureCode.INVALID_UTF8, - normalized_prefix, - len(normalized_prefix), + safe_prefix, + len(safe_prefix), ) - normalized = decoded.replace("\r\n", "\n").replace("\r", "\n") if any( character == "\x00" or 0x7F <= ord(character) <= 0x9F - for character in normalized + for character in decoded ): offset = next( index - for index, character in enumerate(normalized) + for index, character in enumerate(decoded) if character == "\x00" or 0x7F <= ord(character) <= 0x9F ) return _failure( CompilationFailureCode.UNSUPPORTED_CONSTRUCT, - normalized, + decoded, offset, UnsupportedConstruct.CONTROL_CHARACTER, ) - return normalized.rstrip("\n") + "\n" + return decoded def _line_layout(text: str) -> tuple[list[str], list[int]]: - lines = text.removesuffix("\n").split("\n") + raw_lines = text.splitlines(keepends=True) + if not raw_lines: + raw_lines = [""] + lines: list[str] = [] starts: list[int] = [] offset = 0 - for line in lines: - starts.append(offset) - offset += len(line) + 1 + for index, raw_line in enumerate(raw_lines): + content = raw_line.removesuffix("\n").removesuffix("\r") + bom_width = 1 if index == 0 and content.startswith("\ufeff") else 0 + lines.append(content[bom_width:]) + starts.append(offset + bom_width) + offset += len(raw_line) return lines, starts @@ -152,20 +161,12 @@ def _table_cells(extractor: _ElementExtractor, line: str) -> tuple[str, ...]: def _frontmatter_end(lines: list[str]) -> int | None: - if not lines or lines[0].strip() != "---": + if not lines or lines[0] != "---": return None for index in range(1, len(lines)): - if lines[index].strip() == "---": - if index == 1: + if lines[index] == "---": + if index == 1 or not any(line.strip() for line in lines[1:index]): return -1 - for line in lines[1:index]: - if not line.strip(): - continue - if line.startswith((" ", "\t")): - if re.fullmatch(r"[ \t]+-[ \t]+.+", line) is None: - return -1 - elif _FRONTMATTER_FIELD.fullmatch(line) is None: - return -1 return index return -1 @@ -339,9 +340,50 @@ def _blocks(text: str) -> tuple[_Block, ...] | CompilationFailure: continue if line.lstrip().startswith("<"): + html_open = _HTML_OPEN.match(line) + if html_open is None: + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + text, + starts[index], + UnsupportedConstruct.HTML, + ) + tag = html_open.group("tag") end = index + 1 while end < len(lines) and lines[end].strip(): end += 1 + html_source = text[starts[index] : _line_end(lines, starts, end - 1)] + closing_tag = re.search( + rf"", html_source, re.IGNORECASE + ) + if closing_tag is None: + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + text, + starts[index], + UnsupportedConstruct.HTML, + ) + blocks.append( + _Block( + kind=SectionKind.PARAGRAPH, + start=starts[index], + end=_line_end(lines, starts, end - 1), + ) + ) + index = end + continue + + if line.lstrip().startswith(">"): + if _CALLOUT.fullmatch(line) is None: + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + text, + starts[index], + UnsupportedConstruct.BLOCKQUOTE, + ) + end = index + 1 + while end < len(lines) and lines[end].lstrip().startswith(">"): + end += 1 blocks.append( _Block( kind=SectionKind.PARAGRAPH, @@ -352,12 +394,25 @@ def _blocks(text: str) -> tuple[_Block, ...] | CompilationFailure: index = end continue + if re.match( + r"^ {0,3}(?:#{1,6}[ \t]*|---[ \t]*|\*{3,}[ \t]*|_{3,}[ \t]*)$", + line, + ): + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + text, + starts[index], + UnsupportedConstruct.NESTED_HEADING, + ) + end = index + 1 while end < len(lines) and lines[end].strip(): if ( _ATX_HEADING.fullmatch(lines[end]) or extractor._get_fence_marker(lines[end]) is not None or _LIST_ITEM.fullmatch(lines[end]) + or lines[end].lstrip().startswith((">", "<")) + or re.fullmatch(r" {0,3}(?:\*{3,}|_{3,}|-{3,})[ \t]*", lines[end]) or ( end + 1 < len(lines) and extractor._is_table_row(lines[end]) @@ -482,6 +537,18 @@ def compile_rich_markdown( normalized, block.start, ) + elif block.kind in { + SectionKind.LIST, + SectionKind.FENCED_CODE, + SectionKind.TABLE, + }: + ranges = ((0, len(full_source)),) + if rich_token_count(full_source) > capacity: + return _failure( + CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE, + normalized, + block.start, + ) else: ranges = _split_ranges(full_source, capacity) if not ranges: diff --git a/applications/compiler_runner.py b/applications/compiler_runner.py index ac908dd9..fcf62118 100644 --- a/applications/compiler_runner.py +++ b/applications/compiler_runner.py @@ -70,13 +70,13 @@ def _failure_from_document(value: object) -> CompilationFailure: ) -def compile_in_compiler_runner( +def compile_in_local_compiler_runner( source: bytes, config: MarkdownCompilerConfig, *, token_ceiling: int = MARKDOWN_RICH_TOKEN_CEILING, ) -> CompilationOutcome: - """Compile once in a fresh, state-free ContextEngine-owned process.""" + """Compile in an unleased local process that production must never call.""" if type(source) is not bytes: raise TypeError("compiler-runner source must be exact bytes") diff --git a/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md b/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md index bbd7a11e..522088b2 100644 --- a/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md +++ b/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md @@ -33,20 +33,27 @@ index and executes under the exact parent WorkerLease binding. `context-engine-markdown-v3` with its own canonicalization and compilation digest profiles. V1 and v2 remain frozen and byte-reproducible; v3 never silently reinterprets either version. -2. **Closed rich grammar.** V3 accepts UTF-8 Markdown containing YAML - frontmatter, ATX and setext headings, nested ordered or unordered lists, +2. **Closed rich grammar.** V3 accepts UTF-8 Markdown containing nonempty, + `---`-delimited YAML frontmatter, ATX and setext headings, nested ordered or + unordered lists, backtick or tilde fenced code blocks whose content may contain shorter fence runs, pipe tables, wikilinks, embeds, footnotes, HTML blocks, Obsidian - callouts, inline math, and ordinary paragraphs. CRLF and lone CR input are - normalized to LF before positions and digests are derived. CommonMark or - unrestricted HTML compatibility is not claimed. + callouts, inline math, and ordinary paragraphs. CRLF, lone CR, and LF are + treated uniformly for grammar recognition and line/column calculation, but + exact decoded source bytes are retained so every byte span round-trips + against the original UTF-8 input and representation digests distinguish + distinct inputs. Frontmatter payload bytes are retained but not interpreted + or used as authority; semantic YAML validation, CommonMark compatibility, + and unrestricted HTML compatibility are not claimed. An optional leading + UTF-8 BOM is retained for representation identity and treated only as a + transport marker preceding the first Fragment. 3. **Existing output contracts.** The runner emits and deserializes only the existing `ParsedDocument`, `CompiledFragment`, `SourceSpan`, `StructuralPath`, `CompilationProvenance`, or typed `CompilationFailure` contracts. Rich constructs are projected into those existing section kinds; no parallel document or Fragment model is introduced. 4. **Exact provenance.** Every Fragment carries an end-exclusive UTF-8 byte, - line, and column span. Its `source_text` is exactly the canonical input byte + line, and column span. Its `source_text` is exactly the original input byte slice at that span. Non-whitespace source bytes cannot be omitted. Table Fragments obey the same round-trip rule. Heading ancestry is derived during compilation and copied into the same budget-visible Fragment. @@ -57,17 +64,26 @@ index and executes under the exact parent WorkerLease binding. heading ancestry, exact source span, and structural lineage. If heading ancestry alone leaves no capacity, or an indivisible construct cannot be represented within the ceiling, compilation refuses all-or-nothing. -6. **Owned pure process.** The compiler-runner receives exact source bytes and - configuration, invokes the registered vendored parser region plus the - ContextEngine hierarchy/span/bounds kernel, and emits deterministic bytes. - It performs no network or database I/O and retains no independent state, - cache, index, or checkpoint. +6. **Owned pure process.** This issue delivers an unleased local/acceptance + process only. It receives exact source bytes and configuration, executes the + registered RAGFlow element-recognition helpers plus the ContextEngine-owned + grammar, hierarchy, raw-span, and bounds kernel, and emits deterministic + bytes. The copied upstream file remains deliberately unmodified so its hash + stays independently auditable; the adapter calls its fence and table + recognition methods directly. ContextEngine rewrites rich construct + classification, exact raw-byte position mapping, ancestry, typed output, + and splitting because the upstream return shape cannot express those + contracts. The process performs no network or database I/O and retains no + independent state, cache, index, or checkpoint. 7. **Activation remains deferred.** This decision proves the v3 pure transform and local acceptance reporting only. The active File import configuration, database publication functions, immutable Revision schemas, embeddings, and existing v1 Revision migration remain unchanged. Activating v3 for production publication requires a separate decision and complete atomic - publication and re-embedding evidence. + publication and re-embedding evidence. The Supply execution bridge in issue + #125 owns the future production invocation and must bind it to the exact + parent WorkerLease. No production module may call this issue's unleased + local subprocess helper. 8. **Runtime remains sealed.** Compilation changes no Runtime composition. After any future v3 activation, each published Fragment is still only a candidate until it crosses the exact diff --git a/engine/supply/markdown.py b/engine/supply/markdown.py index 18ced969..8a6124e8 100644 --- a/engine/supply/markdown.py +++ b/engine/supply/markdown.py @@ -84,7 +84,7 @@ def __post_init__(self) -> None: @dataclass(frozen=True, slots=True) class SourceSpan: - """End-exclusive source span over canonical normalized UTF-8 text.""" + """End-exclusive source span over the compiler's UTF-8 representation.""" start: SourcePoint end: SourcePoint @@ -330,10 +330,8 @@ class ParsedDocument: warnings: tuple[CompilationWarning, ...] = () def __post_init__(self) -> None: - if type(self.canonical_text) is not str or not self.canonical_text.endswith( - "\n" - ): - raise ValueError("parsed document requires final-newline canonical text") + if type(self.canonical_text) is not str or not self.canonical_text: + raise ValueError("parsed document requires nonempty canonical text") if type(self.sections) is not tuple or any( type(section) is not ParsedSection for section in self.sections ): @@ -346,6 +344,8 @@ def __post_init__(self) -> None: _require_sha256("compilation digest", self.compilation_digest) if type(self.provenance) is not CompilationProvenance: raise TypeError("parsed document provenance must be CompilationProvenance") + if not self.provenance.is_rich_v3 and not self.canonical_text.endswith("\n"): + raise ValueError("parsed document requires final-newline canonical text") if type(self.warnings) is not tuple or any( type(warning) is not CompilationWarning for warning in self.warnings ): @@ -981,12 +981,6 @@ def _validate_rich_content( sections: tuple[ParsedSection, ...], fragments: tuple[CompiledFragment, ...], ) -> None: - if ( - "\r" in canonical_text - or canonical_text.startswith("\ufeff") - or canonical_text.endswith("\n\n") - ): - raise ValueError("rich Markdown must use canonical transport text") if not sections or not fragments or len(sections) != len(fragments): raise ValueError("rich Markdown requires one Fragment per parsed section") canonical_bytes = canonical_text.encode("utf-8") @@ -1001,9 +995,13 @@ def _validate_rich_content( or section.position.start.byte_offset < prior_end ): raise ValueError("rich Fragment lineage must match source order") - if _expected_point(canonical_text, section.position.start.byte_offset) != ( + if _expected_rich_point( + canonical_text, section.position.start.byte_offset + ) != ( section.position.start - ) or _expected_point(canonical_text, section.position.end.byte_offset) != ( + ) or _expected_rich_point( + canonical_text, section.position.end.byte_offset + ) != ( section.position.end ): raise ValueError("rich source coordinates must match UTF-8 offsets") @@ -1017,16 +1015,27 @@ def _validate_rich_content( omitted = canonical_bytes[ prior_end : section.position.start.byte_offset ] - if any(byte not in b" \t\n" for byte in omitted): + if any(byte not in b" \t\r\n\xef\xbb\xbf" for byte in omitted): raise ValueError("rich sections cannot omit non-whitespace source") if fragment.contextual_text != _expected_contextual_text(fragment): raise ValueError("rich Fragment context must be exact heading ancestry") refs.add(fragment.fragment_ref) prior_end = section.position.end.byte_offset - if any(byte not in b" \t\n" for byte in canonical_bytes[prior_end:]): + if any(byte not in b" \t\r\n" for byte in canonical_bytes[prior_end:]): raise ValueError("rich sections cannot omit trailing source") +def _expected_rich_point(source_text: str, byte_offset: int) -> SourcePoint: + prefix = source_text.encode("utf-8")[:byte_offset].decode("utf-8") + logical = prefix.replace("\r\n", "\n").replace("\r", "\n") + last_newline = logical.rfind("\n") + return SourcePoint( + line=logical.count("\n") + 1, + column=len(logical[last_newline + 1 :]) + 1, + byte_offset=byte_offset, + ) + + def _validate_issue_22_content( canonical_text: str, sections: tuple[ParsedSection, ...], diff --git a/pyproject.toml b/pyproject.toml index 967560e5..1a0c5385 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ license-files = [ "NOTICE", "THIRD_PARTY_NOTICES.md", "third_party/ragflow/LICENSE.upstream", + "third_party/ragflow/LICENSE.python-markdown", ] requires-python = ">=3.13,<3.14" dependencies = [ diff --git a/tests/unit/test_compiler_runner_acceptance.py b/tests/unit/test_compiler_runner_acceptance.py index 40440159..09c9fe06 100644 --- a/tests/unit/test_compiler_runner_acceptance.py +++ b/tests/unit/test_compiler_runner_acceptance.py @@ -40,18 +40,18 @@ def test_tracked_rich_construct_corpus_compiles_all_or_nothing(fixture: str) -> @pytest.mark.parametrize("fixture", RICH_FIXTURES) -def test_every_fragment_span_round_trips_to_exact_canonical_utf8( +def test_every_fragment_span_round_trips_to_exact_original_utf8( fixture: str, ) -> None: - outcome = compile_rich_markdown(_source(fixture), CONFIG) + source = _source(fixture) + outcome = compile_rich_markdown(source, CONFIG) assert type(outcome) is ParsedDocument - canonical = outcome.canonical_text.encode("utf-8") for fragment in outcome.fragments: span = fragment.position - assert canonical[span.start.byte_offset : span.end.byte_offset].decode( - "utf-8" - ) == fragment.source_text + assert source[span.start.byte_offset : span.end.byte_offset] == ( + fragment.source_text.encode("utf-8") + ) def test_nested_setext_and_duplicate_headings_have_stable_ancestry() -> None: @@ -79,3 +79,36 @@ def test_nested_setext_and_duplicate_headings_have_stable_ancestry() -> None: "heading[2]", "paragraph[1]", ) + + +@pytest.mark.parametrize( + "frontmatter", + ( + ( + b"owner:\n" + b" name: compiler\n" + b" teams:\n" + b" - supply\n" + b"description: |\n" + b" Compiler metadata keeps\n" + b" ---\n" + b" its exact source lines.\n" + ), + b"- alpha\n- beta\n", + ), +) +def test_delimited_yaml_frontmatter_is_accepted_as_exact_source_fragment( + frontmatter: bytes, +) -> None: + source = b"---\n" + frontmatter + b"---\n# Handbook\n" + + outcome = compile_rich_markdown(source, CONFIG) + + assert type(outcome) is ParsedDocument + frontmatter_fragment = outcome.fragments[0] + expected = source[: source.index(b"#")].rstrip(b"\n").decode() + assert frontmatter_fragment.source_text == expected + span = frontmatter_fragment.position + assert source[span.start.byte_offset : span.end.byte_offset] == ( + frontmatter_fragment.source_text.encode("utf-8") + ) diff --git a/tests/unit/test_compiler_runner_bounds.py b/tests/unit/test_compiler_runner_bounds.py index 58cf5118..d4fe7838 100644 --- a/tests/unit/test_compiler_runner_bounds.py +++ b/tests/unit/test_compiler_runner_bounds.py @@ -1,7 +1,7 @@ from __future__ import annotations from adapters.parsers.ragflow_markdown import compile_rich_markdown, rich_token_count -from engine.supply import MarkdownCompilerConfig, ParsedDocument +from engine.supply import CompilationFailure, MarkdownCompilerConfig, ParsedDocument CONFIG = MarkdownCompilerConfig(version="markdown-config-v3") @@ -19,7 +19,6 @@ def test_oversize_blocks_split_under_hard_bound_with_exact_spans_and_ancestry() assert type(outcome) is ParsedDocument paragraph_fragments = outcome.fragments[2:] assert len(paragraph_fragments) > 1 - canonical = outcome.canonical_text.encode("utf-8") for fragment in paragraph_fragments: assert rich_token_count(fragment.contextual_text) <= ceiling assert tuple(heading.text for heading in fragment.parent_headings) == ( @@ -27,9 +26,34 @@ def test_oversize_blocks_split_under_hard_bound_with_exact_spans_and_ancestry() "Deep", ) span = fragment.position - assert canonical[span.start.byte_offset : span.end.byte_offset].decode( - "utf-8" - ) == fragment.source_text + assert source[span.start.byte_offset : span.end.byte_offset] == ( + fragment.source_text.encode("utf-8") + ) assert " ".join(fragment.source_text for fragment in paragraph_fragments) == ( outcome.canonical_text.split("\n\n", 2)[2].rstrip("\n") ) + + +def test_oversize_indivisible_code_block_refuses_instead_of_token_splitting() -> None: + ceiling = 32 + code = " ".join(f"operation_{index}()" for index in range(200)) + source = f"# Root\r\n\r\n```python\r\n{code}\r\n```\r\n".encode() + + outcome = compile_rich_markdown(source, CONFIG, token_ceiling=ceiling) + + assert type(outcome) is CompilationFailure + + +def test_every_emitted_fragment_obeys_the_ceiling() -> None: + ceiling = 12 + source = ( + "# Root\n\n" + " ".join(f"word{index}" for index in range(80)) + "\n" + ).encode() + + outcome = compile_rich_markdown(source, CONFIG, token_ceiling=ceiling) + + assert type(outcome) is ParsedDocument + assert all( + rich_token_count(fragment.contextual_text) <= ceiling + for fragment in outcome.fragments + ) diff --git a/tests/unit/test_compiler_runner_determinism.py b/tests/unit/test_compiler_runner_determinism.py index 48b16a8d..35314e69 100644 --- a/tests/unit/test_compiler_runner_determinism.py +++ b/tests/unit/test_compiler_runner_determinism.py @@ -2,20 +2,39 @@ from pathlib import Path +import pytest + from adapters.parsers.ragflow_markdown import compile_rich_markdown -from applications.compiler_runner import compile_in_compiler_runner +from applications.compiler_runner import compile_in_local_compiler_runner from engine.supply import MarkdownCompilerConfig, ParsedDocument FIXTURES = Path(__file__).parents[1] / "fixtures/markdown" CONFIG = MarkdownCompilerConfig(version="markdown-config-v3") +RICH_FIXTURES = ( + "rich-frontmatter.md", + "rich-headings-lists.md", + "rich-code-tables.md", + "rich-inline-html.md", + "rich-mixed-newlines.hex", +) + +def _source(name: str) -> bytes: + path = FIXTURES / name + if path.suffix == ".hex": + return bytes.fromhex(path.read_text(encoding="ascii")) + return path.read_bytes() -def test_rich_compilation_digest_is_stable_in_process_and_across_runner() -> None: - source = (FIXTURES / "rich-code-tables.md").read_bytes() + +@pytest.mark.parametrize("fixture", RICH_FIXTURES) +def test_rich_compilation_digest_is_stable_in_process_and_across_runner( + fixture: str, +) -> None: + source = _source(fixture) first = compile_rich_markdown(source, CONFIG) second = compile_rich_markdown(source, CONFIG) - subprocess_result = compile_in_compiler_runner(source, CONFIG) + subprocess_result = compile_in_local_compiler_runner(source, CONFIG) assert type(first) is ParsedDocument assert type(second) is ParsedDocument @@ -23,3 +42,23 @@ def test_rich_compilation_digest_is_stable_in_process_and_across_runner() -> Non assert first.compilation_digest == second.compilation_digest assert first.compilation_digest == subprocess_result.compilation_digest assert subprocess_result == first + + +def test_representation_digest_distinguishes_exact_trailing_newline_bytes() -> None: + variants = ( + b"# Exact\n\nBody", + b"# Exact\n\nBody\n", + b"# Exact\r\n\r\nBody\r\n", + b"# Exact\n\nBody\n\n", + ) + + outcomes = tuple(compile_rich_markdown(source, CONFIG) for source in variants) + + assert all(type(outcome) is ParsedDocument for outcome in outcomes) + documents = tuple( + outcome for outcome in outcomes if type(outcome) is ParsedDocument + ) + assert tuple(document.canonical_text.encode("utf-8") for document in documents) == ( + variants + ) + assert len({document.compilation_digest for document in documents}) == len(variants) diff --git a/tests/unit/test_compiler_runner_production_boundary.py b/tests/unit/test_compiler_runner_production_boundary.py new file mode 100644 index 00000000..82072c65 --- /dev/null +++ b/tests/unit/test_compiler_runner_production_boundary.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +import applications.compiler_runner as compiler_runner + +REPOSITORY_ROOT = Path(__file__).parents[2] +PRODUCTION_ROOTS = ("engine", "adapters", "applications") + + +def test_unleased_subprocess_helper_is_explicitly_local_only() -> None: + assert not hasattr(compiler_runner, "compile_in_compiler_runner") + assert hasattr(compiler_runner, "compile_in_local_compiler_runner") + + +def test_no_production_module_calls_the_local_compiler_runner() -> None: + forbidden_module = "applications.compiler_runner" + forbidden_symbol = "compile_in_local_compiler_runner" + callers: list[str] = [] + for root_name in PRODUCTION_ROOTS: + for path in (REPOSITORY_ROOT / root_name).rglob("*.py"): + if path == REPOSITORY_ROOT / "applications/compiler_runner.py": + continue + tree = ast.parse(path.read_bytes(), filename=str(path)) + for node in ast.walk(tree): + imported_from_runner = isinstance(node, ast.ImportFrom) and ( + node.module == forbidden_module + or any(alias.name == forbidden_symbol for alias in node.names) + ) + imported_runner = isinstance(node, ast.Import) and any( + alias.name == forbidden_module for alias in node.names + ) + if imported_from_runner or imported_runner: + callers.append(str(path.relative_to(REPOSITORY_ROOT))) + assert callers == [] diff --git a/tests/unit/test_compiler_runner_refusal.py b/tests/unit/test_compiler_runner_refusal.py index 3be7a19a..01c78ef1 100644 --- a/tests/unit/test_compiler_runner_refusal.py +++ b/tests/unit/test_compiler_runner_refusal.py @@ -2,7 +2,7 @@ import pytest -from applications.compiler_runner import compile_in_compiler_runner +from applications.compiler_runner import compile_in_local_compiler_runner from engine.supply import ( CompilationFailure, CompilationFailureCode, @@ -29,13 +29,29 @@ CompilationFailureCode.UNSUPPORTED_CONSTRUCT, ), (b"# Heading\n\ntruncated \xe4\xb8", CompilationFailureCode.INVALID_UTF8), + ( + b"# Heading\n\nplain paragraph\n> ordinary blockquote\n", + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + ), + ( + b"# Heading\n\n\n", + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + ), + ( + b"# Heading\n\n***\n", + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + ), + ( + b" ---\nkey: value\n---\n# Heading\n", + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + ), ], ) def test_malformed_source_returns_typed_failure_across_runner_boundary( source: bytes, expected_code: CompilationFailureCode, ) -> None: - outcome = compile_in_compiler_runner(source, CONFIG) + outcome = compile_in_local_compiler_runner(source, CONFIG) assert type(outcome) is CompilationFailure assert outcome.code is expected_code diff --git a/tests/unit/test_compiler_runner_tables.py b/tests/unit/test_compiler_runner_tables.py index e0acef4f..58e18f9f 100644 --- a/tests/unit/test_compiler_runner_tables.py +++ b/tests/unit/test_compiler_runner_tables.py @@ -31,3 +31,26 @@ def test_every_table_fragment_carries_a_round_tripping_source_span() -> None: assert canonical[span.start.byte_offset : span.end.byte_offset].decode( "utf-8" ) == fragment.source_text + + +def test_table_spans_round_trip_against_original_crlf_bytes() -> None: + source = ( + b"# Tables\r\n\r\n" + b"| Key | Value |\r\n" + b"| --- | --- |\r\n" + b"| alpha | ready |\r\n" + ) + + outcome = compile_rich_markdown(source, CONFIG) + + assert type(outcome) is ParsedDocument + table = next( + fragment + for fragment in outcome.fragments + if fragment.kind is SectionKind.TABLE + ) + span = table.position + assert source[span.start.byte_offset : span.end.byte_offset] == ( + table.source_text.encode("utf-8") + ) + assert b"\r\n" in table.source_text.encode("utf-8") diff --git a/tests/unit/test_third_party_ragflow_registration.py b/tests/unit/test_third_party_ragflow_registration.py index f24e5e7d..effc9041 100644 --- a/tests/unit/test_third_party_ragflow_registration.py +++ b/tests/unit/test_third_party_ragflow_registration.py @@ -2,6 +2,7 @@ import ast import hashlib +import json import re import tomllib from pathlib import Path @@ -50,6 +51,14 @@ def test_vendored_bytes_match_complete_pinned_registration() -> None: "https://github.com/stone16/context-engine/issues/124" ) assert registration["source_paths"] == ["deepdoc/parser/markdown_parser.py"] + assert registration["nested_dependencies"] == [ + { + "name": "Python-Markdown", + "version": "3.6", + "license": "BSD-3-Clause", + "license_path": "third_party/ragflow/LICENSE.python-markdown", + } + ] excluded_paths = registration["excluded_paths"] assert isinstance(excluded_paths, list) assert set(excluded_paths) >= REQUIRED_EXCLUSIONS @@ -80,14 +89,41 @@ def test_vendored_bytes_match_complete_pinned_registration() -> None: if path.is_file() and "__pycache__" not in path.relative_to(REGISTRATION_ROOT).parts and path.name - not in {"LICENSE.upstream", "MODIFICATIONS.md", "UPSTREAM.toml"} + not in { + "LICENSE.python-markdown", + "LICENSE.upstream", + "MODIFICATIONS.md", + "UPSTREAM.toml", + "sbom.cyclonedx.json", + } and "patches" not in path.relative_to(REGISTRATION_ROOT).parts } assert registered_paths == vendored_files assert (REGISTRATION_ROOT / "LICENSE.upstream").is_file() + assert hashlib.sha256( + (REGISTRATION_ROOT / "LICENSE.python-markdown").read_bytes() + ).hexdigest() == "7ba4eb6d10b32b2d11dce13821340351cdbbb30ba8ccc67841db2ffd86e79aca" assert (REGISTRATION_ROOT / "MODIFICATIONS.md").is_file() assert (REGISTRATION_ROOT / "patches").is_dir() assert (REPOSITORY_ROOT / "THIRD_PARTY_NOTICES.md").is_file() + sbom = json.loads( + (REGISTRATION_ROOT / "sbom.cyclonedx.json").read_text(encoding="utf-8") + ) + assert sbom["bomFormat"] == "CycloneDX" + assert sbom["metadata"]["component"]["bom-ref"] == ( + "context-engine:third-party:ragflow" + ) + assert { + (property_value["name"], property_value["value"]) + for property_value in sbom["metadata"]["properties"] + } >= { + ("context-engine:sbom:scope", "third_party/ragflow"), + ("context-engine:sbom:artifact-wide", "false"), + } + assert {component["name"] for component in sbom["components"]} == { + "Python-Markdown", + "RAGFlow Markdown parser", + } def test_vendored_subtree_imports_only_approved_dependencies() -> None: @@ -117,3 +153,31 @@ def test_vendored_subtree_imports_only_approved_dependencies() -> None: ) assert "Python-Markdown" in modifications assert "BSD 3-Clause" in modifications + + +def test_registered_parser_region_is_executed_by_the_ce_adapter() -> None: + adapter = REPOSITORY_ROOT / "adapters/parsers/ragflow_markdown.py" + tree = ast.parse(adapter.read_bytes(), filename=str(adapter)) + + imports = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) + and node.module == "third_party.ragflow.deepdoc.parser.markdown_parser" + ] + assert len(imports) == 1 + assert [alias.name for alias in imports[0].names] == [ + "MarkdownElementExtractor" + ] + called_helpers = { + node.func.attr + for node in ast.walk(tree) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + } + assert { + "_get_fence_marker", + "_is_closing_fence", + "_is_table_row", + "_is_table_separator_row", + "_table_cells", + } <= called_helpers diff --git a/third_party/ragflow/LICENSE.python-markdown b/third_party/ragflow/LICENSE.python-markdown new file mode 100644 index 00000000..6249d60c --- /dev/null +++ b/third_party/ragflow/LICENSE.python-markdown @@ -0,0 +1,30 @@ +BSD 3-Clause License + +Copyright 2007, 2008 The Python Markdown Project (v. 1.7 and later) +Copyright 2004, 2005, 2006 Yuri Takhteyev (v. 0.2-1.6b) +Copyright 2004 Manfred Stienstra (the original version) + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/third_party/ragflow/MODIFICATIONS.md b/third_party/ragflow/MODIFICATIONS.md index 921c96c5..c70cb868 100644 --- a/third_party/ragflow/MODIFICATIONS.md +++ b/third_party/ragflow/MODIFICATIONS.md @@ -12,17 +12,28 @@ root `NOTICE` file. The copied file imports only Python standard-library modules and Python-Markdown. Python-Markdown is licensed under the BSD 3-Clause License. -No other nested third-party dependency is imported by the copied region. +Its verbatim license for the pinned 3.6 dependency is retained as +`LICENSE.python-markdown`. No other nested third-party dependency is imported +by the copied region. -## ContextEngine modifications +## Executed reuse and ContextEngine-owned behavior -The upstream bytes are retained under `deepdoc/parser/markdown_parser.py` for -auditable provenance. ContextEngine adapters patch behavior through the owned -compiler-runner and do not import RAGFlow package initializers or its `rag/nlp` -dependency carriers. Local changes add exact UTF-8 spans, heading ancestry, -hard bounds, versioned deterministic output, and typed refusal at the -ContextEngine seam. +The exact upstream file is copied and executed: the ContextEngine adapter +constructs its `MarkdownElementExtractor` and calls the upstream fence-marker, +closing-fence, table-row, table-separator, and table-cell recognition methods. +No RAGFlow package initializer or `rag/nlp` dependency carrier is copied or +imported. -The empty `patches/` directory is reserved for future upgrades that change the -registered upstream file itself. The current integration wraps the copied file, -so no source patch applies. +The broader compilation pipeline is ContextEngine-owned. It implements the +closed rich grammar, raw-input UTF-8 spans, heading ancestry, hard bounds, +versioned deterministic output, and typed refusal because the upstream parser +does not return the existing ContextEngine contracts and lacks exact byte-span +and hard-bound semantics. This is intentionally narrow reuse of one verified +Apache-2.0 file, not a claim that the entire upstream parser pipeline executes. + +`sbom.cyclonedx.json` is the deterministic component inventory for the copied +parser and its sole nested dependency. Both wheel and source distribution +artifacts include it alongside the applicable license texts. + +The empty `patches/` directory records that no textual source patch applies: +the current integration wraps and directly executes selected copied helpers. diff --git a/third_party/ragflow/UPSTREAM.toml b/third_party/ragflow/UPSTREAM.toml index 569bca79..7cf5ab0f 100644 --- a/third_party/ragflow/UPSTREAM.toml +++ b/third_party/ragflow/UPSTREAM.toml @@ -8,6 +8,9 @@ excluded_paths = [ ] reuse_mode = "copy-patch" approval = "https://github.com/stone16/context-engine/issues/124" +nested_dependencies = [ + { name = "Python-Markdown", version = "3.6", license = "BSD-3-Clause", license_path = "third_party/ragflow/LICENSE.python-markdown" }, +] [[files]] upstream_path = "deepdoc/parser/markdown_parser.py" diff --git a/third_party/ragflow/sbom.cyclonedx.json b/third_party/ragflow/sbom.cyclonedx.json new file mode 100644 index 00000000..011d5c3c --- /dev/null +++ b/third_party/ragflow/sbom.cyclonedx.json @@ -0,0 +1,50 @@ +{ + "bomFormat": "CycloneDX", + "metadata": { + "component": { + "bom-ref": "context-engine:third-party:ragflow", + "name": "ContextEngine registered RAGFlow subtree", + "type": "library" + }, + "properties": [ + { + "name": "context-engine:sbom:scope", + "value": "third_party/ragflow" + }, + { + "name": "context-engine:sbom:artifact-wide", + "value": "false" + } + ] + }, + "components": [ + { + "licenses": [ + { + "license": { + "id": "Apache-2.0" + } + } + ], + "name": "RAGFlow Markdown parser", + "purl": "pkg:github/infiniflow/ragflow@4391e03886b996201f3b8818f671b19eb24d0f7b#deepdoc/parser/markdown_parser.py", + "type": "library", + "version": "4391e03886b996201f3b8818f671b19eb24d0f7b" + }, + { + "licenses": [ + { + "license": { + "id": "BSD-3-Clause" + } + } + ], + "name": "Python-Markdown", + "purl": "pkg:pypi/markdown@3.6", + "type": "library", + "version": "3.6" + } + ], + "specVersion": "1.6", + "version": 1 +} From 8e177348ab33903f083bbb804141ebfe06e3714f Mon Sep 17 00:00:00 2001 From: stone Date: Thu, 30 Jul 2026 01:44:19 +0800 Subject: [PATCH 05/16] fix: seal rich compiler invariants (#124) --- adapters/parsers/ragflow_markdown.py | 244 ++++++-- applications/compiler_runner.py | 72 ++- .../compiler-runner-acceptance-v1.schema.json | 11 +- engine/supply/markdown.py | 374 +++++++++++- tests/fixtures/markdown/rich-expanded.md | 27 + tests/unit/test_compiler_runner_acceptance.py | 537 ++++++++++++++++++ .../test_compiler_runner_acceptance_report.py | 15 +- tests/unit/test_compiler_runner_bounds.py | 45 +- .../unit/test_compiler_runner_determinism.py | 38 +- ...est_compiler_runner_production_boundary.py | 6 + tests/unit/test_compiler_runner_refusal.py | 191 ++++++- tests/unit/test_compiler_runner_tables.py | 75 ++- .../test_third_party_ragflow_registration.py | 50 +- 13 files changed, 1576 insertions(+), 109 deletions(-) create mode 100644 tests/fixtures/markdown/rich-expanded.md diff --git a/adapters/parsers/ragflow_markdown.py b/adapters/parsers/ragflow_markdown.py index 2324d3d1..2ada9904 100644 --- a/adapters/parsers/ragflow_markdown.py +++ b/adapters/parsers/ragflow_markdown.py @@ -11,7 +11,6 @@ MARKDOWN_COMPILER_V3_VERSION, MARKDOWN_RICH_CANONICALIZATION_PROFILE, MARKDOWN_RICH_COMPILATION_DIGEST_PROFILE, - MARKDOWN_RICH_TOKEN_CEILING, CompilationFailure, CompilationFailureCode, CompilationOutcome, @@ -25,6 +24,7 @@ SourceSpan, StructuralPath, UnsupportedConstruct, + unsupported_rich_markdown_inline, ) from third_party.ragflow.deepdoc.parser.markdown_parser import MarkdownElementExtractor @@ -39,6 +39,7 @@ r"th|td|p|ul|ol|li)\b[^>]*>", re.IGNORECASE, ) +_ANGLE_LITERAL: Final = re.compile(r"<[^<>\r\n]+>") @dataclass(frozen=True, slots=True) @@ -46,6 +47,7 @@ class _Block: kind: SectionKind start: int end: int + indivisible: bool = False level: int | None = None list_ordered: bool | None = None list_items: tuple[str, ...] = () @@ -160,6 +162,12 @@ def _table_cells(extractor: _ElementExtractor, line: str) -> tuple[str, ...]: return tuple(extractor._table_cells(line)) +def _is_table_source_line(line: str) -> bool: + """Return whether a nonblank line remains part of an opened pipe table.""" + + return "|" in line + + def _frontmatter_end(lines: list[str]) -> int | None: if not lines or lines[0] != "---": return None @@ -204,6 +212,14 @@ def _blocks(text: str) -> tuple[_Block, ...] | CompilationFailure: atx = _ATX_HEADING.fullmatch(line) if atx is not None: + construct = unsupported_rich_markdown_inline(atx.group(2).strip()) + if construct is not None: + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + text, + starts[index], + construct, + ) blocks.append( _Block( kind=SectionKind.HEADING, @@ -214,34 +230,35 @@ def _blocks(text: str) -> tuple[_Block, ...] | CompilationFailure: ) index += 1 continue - if index + 1 < len(lines) and _SETEXT.fullmatch(lines[index + 1]): - blocks.append( - _Block( - kind=SectionKind.HEADING, - start=starts[index], - end=_line_end(lines, starts, index + 1), - level=1 if lines[index + 1].lstrip().startswith("=") else 2, - ) - ) - index += 2 - continue - fence = extractor._get_fence_marker(line) if fence is not None: fence_character, fence_length = fence + language = line.lstrip()[fence_length:].strip() or None closing = index + 1 while closing < len(lines) and not extractor._is_closing_fence( lines[closing], fence_character, fence_length ): closing += 1 if closing >= len(lines): + if language is None and any( + candidate.strip() + for candidate in lines[index + 1 :] + ): + blocks.append( + _Block( + kind=SectionKind.PARAGRAPH, + start=starts[index], + end=_line_end(lines, starts, len(lines) - 1), + ) + ) + index = len(lines) + continue return _failure( CompilationFailureCode.UNSUPPORTED_CONSTRUCT, text, starts[index], UnsupportedConstruct.CODE_BLOCK, ) - language = line.lstrip()[fence_length:].strip() or None if language is not None and ( len(language) > MARKDOWN_CODE_LANGUAGE_MAX_LENGTH or any(character.isspace() for character in language) @@ -272,33 +289,70 @@ def _blocks(text: str) -> tuple[_Block, ...] | CompilationFailure: index = closing + 1 continue + if index + 1 < len(lines) and _SETEXT.fullmatch(lines[index + 1]): + construct = unsupported_rich_markdown_inline(line.strip()) + if construct is not None: + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + text, + starts[index], + construct, + ) + blocks.append( + _Block( + kind=SectionKind.HEADING, + start=starts[index], + end=_line_end(lines, starts, index + 1), + level=1 if lines[index + 1].lstrip().startswith("=") else 2, + ) + ) + index += 2 + continue + if ( index + 1 < len(lines) and extractor._is_table_row(line) and extractor._is_table_separator_row(lines[index + 1]) ): header = _table_cells(extractor, line) + separator = _table_cells(extractor, lines[index + 1]) width = len(header) rows: list[tuple[str, ...]] = [] end = index + 2 - while end < len(lines) and extractor._is_table_row(lines[end]): + while end < len(lines) and _is_table_source_line(lines[end]): row = _table_cells(extractor, lines[end]) - if len(row) != width: - return _failure( - CompilationFailureCode.UNSUPPORTED_CONSTRUCT, - text, - starts[end], - UnsupportedConstruct.TABLE, - ) rows.append(row) end += 1 - if not rows or any(not cell for row in (header, *rows) for cell in row): - return _failure( - CompilationFailureCode.UNSUPPORTED_CONSTRUCT, - text, - starts[index], - UnsupportedConstruct.TABLE, + if ( + not rows + or width < 2 + or len(separator) != width + or any( + len(row) != width or any(not cell for cell in row) + for row in (header, *rows) ) + ): + for table_index in range(index, end): + construct = unsupported_rich_markdown_inline( + lines[table_index] + ) + if construct is not None: + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + text, + starts[table_index], + construct, + ) + blocks.append( + _Block( + kind=SectionKind.PARAGRAPH, + start=starts[index], + end=_line_end(lines, starts, end - 1), + indivisible=True, + ) + ) + index = end + continue blocks.append( _Block( kind=SectionKind.TABLE, @@ -320,10 +374,29 @@ def _blocks(text: str) -> tuple[_Block, ...] | CompilationFailure: candidate = lines[end] match = _LIST_ITEM.fullmatch(candidate) if match is not None: - items.append(match.group(2)) + item = match.group(2).rstrip(" \t") + construct = unsupported_rich_markdown_inline(item) + if construct not in {None, UnsupportedConstruct.LIST}: + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + text, + starts[end], + construct, + ) + items.append(item) end += 1 continue if candidate.strip() and candidate.startswith((" ", "\t")): + construct = unsupported_rich_markdown_inline( + candidate.lstrip() + ) + if construct is not None: + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + text, + starts[end], + construct, + ) end += 1 continue break @@ -340,6 +413,16 @@ def _blocks(text: str) -> tuple[_Block, ...] | CompilationFailure: continue if line.lstrip().startswith("<"): + if _ANGLE_LITERAL.fullmatch(line.strip()) is not None: + blocks.append( + _Block( + kind=SectionKind.PARAGRAPH, + start=starts[index], + end=_line_end(lines, starts, index), + ) + ) + index += 1 + continue html_open = _HTML_OPEN.match(line) if html_open is None: return _failure( @@ -374,16 +457,24 @@ def _blocks(text: str) -> tuple[_Block, ...] | CompilationFailure: continue if line.lstrip().startswith(">"): - if _CALLOUT.fullmatch(line) is None: - return _failure( - CompilationFailureCode.UNSUPPORTED_CONSTRUCT, - text, - starts[index], - UnsupportedConstruct.BLOCKQUOTE, - ) end = index + 1 while end < len(lines) and lines[end].lstrip().startswith(">"): end += 1 + for quote_index in range(index, end): + quoted = lines[quote_index].lstrip()[1:].lstrip() + callout = _CALLOUT.fullmatch(lines[quote_index]) is not None + construct = ( + None + if callout + else unsupported_rich_markdown_inline(quoted) + ) + if construct is not None: + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + text, + starts[quote_index], + construct, + ) blocks.append( _Block( kind=SectionKind.PARAGRAPH, @@ -395,7 +486,7 @@ def _blocks(text: str) -> tuple[_Block, ...] | CompilationFailure: continue if re.match( - r"^ {0,3}(?:#{1,6}[ \t]*|---[ \t]*|\*{3,}[ \t]*|_{3,}[ \t]*)$", + r"^ {0,3}#{1,6}[ \t]*$", line, ): return _failure( @@ -405,6 +496,20 @@ def _blocks(text: str) -> tuple[_Block, ...] | CompilationFailure: UnsupportedConstruct.NESTED_HEADING, ) + if re.fullmatch( + r" {0,3}(?:(?:\*[ \t]*){3,}|(?:_[ \t]*){3,}|(?:-[ \t]*){3,})", + line, + ): + blocks.append( + _Block( + kind=SectionKind.PARAGRAPH, + start=starts[index], + end=_line_end(lines, starts, index), + ) + ) + index += 1 + continue + end = index + 1 while end < len(lines) and lines[end].strip(): if ( @@ -412,7 +517,11 @@ def _blocks(text: str) -> tuple[_Block, ...] | CompilationFailure: or extractor._get_fence_marker(lines[end]) is not None or _LIST_ITEM.fullmatch(lines[end]) or lines[end].lstrip().startswith((">", "<")) - or re.fullmatch(r" {0,3}(?:\*{3,}|_{3,}|-{3,})[ \t]*", lines[end]) + or re.fullmatch( + r" {0,3}(?:(?:\*[ \t]*){3,}|(?:_[ \t]*){3,}|" + r"(?:-[ \t]*){3,})", + lines[end], + ) or ( end + 1 < len(lines) and extractor._is_table_row(lines[end]) @@ -421,6 +530,17 @@ def _blocks(text: str) -> tuple[_Block, ...] | CompilationFailure: ): break end += 1 + for paragraph_index in range(index, end): + construct = unsupported_rich_markdown_inline( + lines[paragraph_index] + ) + if construct is not None: + return _failure( + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + text, + starts[paragraph_index], + construct, + ) blocks.append( _Block( kind=SectionKind.PARAGRAPH, @@ -433,7 +553,7 @@ def _blocks(text: str) -> tuple[_Block, ...] | CompilationFailure: def _heading_text(source: str, level: int) -> str: - lines = source.split("\n") + lines = source.splitlines() if len(lines) == 2 and _SETEXT.fullmatch(lines[1]): return lines[0].strip() match = _ATX_HEADING.fullmatch(source) @@ -481,8 +601,6 @@ def _section( def compile_rich_markdown( source: bytes, config: MarkdownCompilerConfig, - *, - token_ceiling: int = MARKDOWN_RICH_TOKEN_CEILING, ) -> CompilationOutcome: """Compile exact bytes through the explicit rich v3 representation.""" @@ -492,12 +610,19 @@ def compile_rich_markdown( raise TypeError("rich Markdown compiler config must be exact") if config.version != "markdown-config-v3": raise ValueError("rich Markdown compiler requires markdown-config-v3") - if type(token_ceiling) is not int or token_ceiling < 1: - raise ValueError("rich Markdown token ceiling must be positive") + assert config.token_ceiling is not None + token_ceiling = config.token_ceiling normalized = _normalize(source) if isinstance(normalized, CompilationFailure): return normalized - blocks = _blocks(normalized) + try: + blocks = _blocks(normalized) + except Exception: + return _failure( + CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE, + normalized, + 0, + ) if isinstance(blocks, CompilationFailure): return blocks if not blocks: @@ -537,7 +662,7 @@ def compile_rich_markdown( normalized, block.start, ) - elif block.kind in { + elif block.indivisible or block.kind in { SectionKind.LIST, SectionKind.FENCED_CODE, SectionKind.TABLE, @@ -569,7 +694,14 @@ def compile_rich_markdown( block.start + relative_start, block.start + relative_end, ) - section = _section(block, source_text, path, position) + try: + section = _section(block, source_text, path, position) + except Exception: + return _failure( + CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE, + normalized, + block.start + relative_start, + ) ordinals[block.kind] = ordinals.get(block.kind, 0) + 1 contextual = f"{ancestry}\n\n{source_text}" if ancestry else source_text phrases = tuple(dict.fromkeys((source_text, section.text))) @@ -595,10 +727,18 @@ def compile_rich_markdown( config_version=config.version, canonicalization_profile=MARKDOWN_RICH_CANONICALIZATION_PROFILE, compilation_digest_profile=MARKDOWN_RICH_COMPILATION_DIGEST_PROFILE, + token_ceiling=token_ceiling, ) - return ParsedDocument.rich_v3( - canonical_text=normalized, - sections=tuple(sections), - fragments=tuple(fragments), - provenance=provenance, - ) + try: + return ParsedDocument.rich_v3( + canonical_text=normalized, + sections=tuple(sections), + fragments=tuple(fragments), + provenance=provenance, + ) + except Exception: + return _failure( + CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE, + normalized, + 0, + ) diff --git a/applications/compiler_runner.py b/applications/compiler_runner.py index fcf62118..a2acd70c 100644 --- a/applications/compiler_runner.py +++ b/applications/compiler_runner.py @@ -29,6 +29,13 @@ _RUNNER_MODULE: Final = "applications.compiler_runner" +def _boundary_failure() -> CompilationFailure: + return CompilationFailure( + code=CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE, + position=None, + ) + + def _failure_document(failure: CompilationFailure) -> dict[str, object]: return { "code": failure.code.value, @@ -73,8 +80,6 @@ def _failure_from_document(value: object) -> CompilationFailure: def compile_in_local_compiler_runner( source: bytes, config: MarkdownCompilerConfig, - *, - token_ceiling: int = MARKDOWN_RICH_TOKEN_CEILING, ) -> CompilationOutcome: """Compile in an unleased local process that production must never call.""" @@ -82,6 +87,8 @@ def compile_in_local_compiler_runner( raise TypeError("compiler-runner source must be exact bytes") if type(config) is not MarkdownCompilerConfig: raise TypeError("compiler-runner config must be exact") + if config.token_ceiling is None: + raise ValueError("compiler-runner requires rich Markdown config") completed = subprocess.run( [ sys.executable, @@ -91,30 +98,44 @@ def compile_in_local_compiler_runner( "--config", config.version, "--token-ceiling", - str(token_ceiling), + str(config.token_ceiling), ], input=source, capture_output=True, check=False, ) if completed.returncode != 0: - raise RuntimeError("compiler-runner process failed") - envelope = json.loads(completed.stdout) + return _boundary_failure() + try: + envelope = json.loads(completed.stdout) + except (UnicodeDecodeError, json.JSONDecodeError): + return _boundary_failure() if type(envelope) is not dict: - raise RuntimeError("compiler-runner emitted an invalid envelope") + return _boundary_failure() document = cast(dict[str, object], envelope) if document.get("outcome") == "parsed": encoded = document.get("document") if type(encoded) is not str: - raise RuntimeError("compiler-runner parsed payload is invalid") - return deserialize_parsed_document(base64.b64decode(encoded, validate=True)) + return _boundary_failure() + try: + return deserialize_parsed_document( + base64.b64decode(encoded, validate=True) + ) + except (ValueError, TypeError): + return _boundary_failure() if document.get("outcome") == "failure": - return _failure_from_document(document.get("failure")) - raise RuntimeError("compiler-runner outcome is invalid") + try: + return _failure_from_document(document.get("failure")) + except (KeyError, ValueError, TypeError): + return _boundary_failure() + return _boundary_failure() -def _emit(source: bytes, config: MarkdownCompilerConfig, token_ceiling: int) -> None: - outcome = compile_rich_markdown(source, config, token_ceiling=token_ceiling) +def _emit(source: bytes, config: MarkdownCompilerConfig) -> None: + try: + outcome = compile_rich_markdown(source, config) + except Exception: + outcome = _boundary_failure() if type(outcome) is ParsedDocument: envelope: dict[str, object] = { "outcome": "parsed", @@ -180,6 +201,7 @@ def _acceptance_report(root: Path, token_ceiling: int) -> dict[str, object]: digests: list[str] = [] maximum = 0 histogram = {name: 0 for name in _CONSTRUCT_PATTERNS} + refusal_histogram: dict[str, int] = {} for path in _safe_markdown_files(root): source = path.read_bytes() try: @@ -191,8 +213,10 @@ def _acceptance_report(root: Path, token_ceiling: int) -> dict[str, object]: histogram[name] += len(pattern.findall(normalized)) outcome = compile_rich_markdown( source, - MarkdownCompilerConfig("markdown-config-v3"), - token_ceiling=token_ceiling, + MarkdownCompilerConfig( + "markdown-config-v3", + token_ceiling=token_ceiling, + ), ) if type(outcome) is ParsedDocument: accepted += 1 @@ -206,16 +230,28 @@ def _acceptance_report(root: Path, token_ceiling: int) -> dict[str, object]: ) else: refused += 1 + assert type(outcome) is CompilationFailure + category = outcome.code.value + if outcome.construct is not None: + category = f"{category}:{outcome.construct.value}" + refusal_histogram[category] = refusal_histogram.get(category, 0) + 1 aggregate = hashlib.sha256() for digest in sorted(digests): aggregate.update(bytes.fromhex(digest)) total = accepted + refused + acceptance_rate = f"{accepted / total:.6f}" if total else "0.000000" return { "schemaVersion": "compiler-runner-acceptance-v1", "compilerVersion": "context-engine-markdown-v3", "configVersion": "markdown-config-v3", - "documents": {"accepted": accepted, "refused": refused, "total": total}, + "documents": { + "accepted": accepted, + "acceptanceRate": acceptance_rate, + "refused": refused, + "total": total, + }, "constructHistogram": histogram, + "refusalHistogram": refusal_histogram, "tokenCeiling": token_ceiling, "maxFragmentTokenCount": maximum, "aggregateCompilationDigest": aggregate.hexdigest(), @@ -237,8 +273,10 @@ def main() -> None: if args.compile: _emit( sys.stdin.buffer.read(), - MarkdownCompilerConfig(args.config), - cast(int, args.token_ceiling), + MarkdownCompilerConfig( + args.config, + token_ceiling=cast(int, args.token_ceiling), + ), ) return if args.acceptance_report: diff --git a/docs/contracts/compiler-runner-acceptance-v1.schema.json b/docs/contracts/compiler-runner-acceptance-v1.schema.json index b3e87a34..55a21550 100644 --- a/docs/contracts/compiler-runner-acceptance-v1.schema.json +++ b/docs/contracts/compiler-runner-acceptance-v1.schema.json @@ -11,6 +11,7 @@ "constructHistogram", "documents", "maxFragmentTokenCount", + "refusalHistogram", "schemaVersion", "tokenCeiling" ], @@ -56,13 +57,21 @@ "documents": { "type": "object", "additionalProperties": false, - "required": ["accepted", "refused", "total"], + "required": ["accepted", "acceptanceRate", "refused", "total"], "properties": { "accepted": {"type": "integer", "minimum": 0}, + "acceptanceRate": { + "type": "string", + "pattern": "^(?:0|1)\\.[0-9]{6}$" + }, "refused": {"type": "integer", "minimum": 0}, "total": {"type": "integer", "minimum": 0} } }, + "refusalHistogram": { + "type": "object", + "additionalProperties": {"type": "integer", "minimum": 1} + }, "maxFragmentTokenCount": {"type": "integer", "minimum": 0}, "schemaVersion": {"const": "compiler-runner-acceptance-v1"}, "tokenCeiling": {"type": "integer", "minimum": 1} diff --git a/engine/supply/markdown.py b/engine/supply/markdown.py index 8a6124e8..45106689 100644 --- a/engine/supply/markdown.py +++ b/engine/supply/markdown.py @@ -23,7 +23,7 @@ MARKDOWN_RICH_CANONICALIZATION_PROFILE: Final = "markdown-rich-structural-v3" MARKDOWN_RICH_COMPILATION_DIGEST_PROFILE: Final = "rfc8785-sha256-v3" MARKDOWN_CODE_LANGUAGE_MAX_LENGTH: Final = 64 -MARKDOWN_RICH_TOKEN_CEILING: Final = 512 +MARKDOWN_RICH_TOKEN_CEILING: Final = 2048 _COMPILATION_DIGEST_V1_DOMAIN: Final = b"context-engine.markdown-compilation.v1\x00" _COMPILATION_DIGEST_DOMAIN: Final = b"context-engine.markdown-compilation.v2\x00" _COMPILATION_DIGEST_V3_DOMAIN: Final = b"context-engine.markdown-compilation.v3\x00" @@ -60,9 +60,17 @@ class MarkdownCompilerConfig: """Explicit representation-affecting compiler configuration identity.""" version: str + token_ceiling: int | None = None def __post_init__(self) -> None: _require_version(self.version) + if self.version == "markdown-config-v3": + if self.token_ceiling is None: + object.__setattr__(self, "token_ceiling", MARKDOWN_RICH_TOKEN_CEILING) + elif type(self.token_ceiling) is not int or self.token_ceiling < 1: + raise ValueError("rich Markdown token ceiling must be positive") + elif self.token_ceiling is not None: + raise ValueError("only rich Markdown config records a token ceiling") @dataclass(frozen=True, slots=True) @@ -153,12 +161,7 @@ class ParsedSection: def __post_init__(self) -> None: if type(self.kind) is not SectionKind: raise TypeError("parsed section kind must be SectionKind") - if ( - type(self.text) is not str - or not self.text - or self.text.isspace() - or self.text != self.text.strip() - ): + if type(self.text) is not str or not self.text or self.text.isspace(): raise ValueError("parsed section text must be exact nonblank text") if type(self.path) is not StructuralPath: raise TypeError("parsed section path must be StructuralPath") @@ -194,9 +197,8 @@ def __post_init__(self) -> None: if self.kind is SectionKind.TABLE: if not self.table_header or not self.table_rows: raise ValueError("table sections require a header and rows") - width = len(self.table_header) - if width < 1 or any(len(row) != width for row in self.table_rows): - raise ValueError("table rows must match the header width") + if len(self.table_header) < 1: + raise ValueError("table header must carry at least one cell") if any( type(cell) is not str or not cell or cell.isspace() for row in (self.table_header, *self.table_rows) @@ -285,6 +287,7 @@ class CompilationProvenance: canonicalization_profile: str = MARKDOWN_CANONICALIZATION_V1_PROFILE content_hash_profile: str = MARKDOWN_CONTENT_HASH_PROFILE compilation_digest_profile: str = MARKDOWN_COMPILATION_DIGEST_V1_PROFILE + token_ceiling: int | None = None def __post_init__(self) -> None: _require_version(self.compiler_version) @@ -307,6 +310,11 @@ def __post_init__(self) -> None: raise ValueError("Markdown canonicalization and digest profiles must match") if self.content_hash_profile != MARKDOWN_CONTENT_HASH_PROFILE: raise ValueError("content hash profile must use the active version") + if self.is_rich_v3: + if type(self.token_ceiling) is not int or self.token_ceiling < 1: + raise ValueError("rich provenance requires a positive token ceiling") + elif self.token_ceiling is not None: + raise ValueError("only rich provenance records a token ceiling") @property def is_structural_v2(self) -> bool: @@ -353,7 +361,13 @@ def __post_init__(self) -> None: if self.warnings: raise ValueError("the active Markdown compiler emits no warnings") if self.provenance.is_rich_v3: - _validate_rich_content(self.canonical_text, self.sections, self.fragments) + assert self.provenance.token_ceiling is not None + _validate_rich_content( + self.canonical_text, + self.sections, + self.fragments, + self.provenance.token_ceiling, + ) elif self.provenance.is_structural_v2: _validate_structural_content( self.canonical_text, @@ -520,6 +534,39 @@ class UnsupportedConstruct(StrEnum): r"(?`{3,}|~{3,})(?:.*)$" +) +_RICH_WIKILINK_PATTERN: Final = re.compile(r"!?\[\[[^]\r\n]+]]") +_RICH_FOOTNOTE_PATTERN: Final = re.compile(r"\[\^[^]\r\n]+](?::)?") +_RICH_INLINE_MATH_PATTERN: Final = re.compile( + r"(?\s]+>", + re.IGNORECASE, +) +_RICH_INLINE_LINK_PATTERN: Final = re.compile( + r"!?\[[^]\r\n]*]\([^()\r\n]+\)" +) +_RICH_REFERENCE_LINK_PATTERN: Final = re.compile( + r"(?:!?\[[^]\r\n]*]\[[^]\r\n]*]|\[[^]\r\n]+]:[ \t]*\S+)" +) +_RICH_STRIKETHROUGH_PATTERN: Final = re.compile(r"~~(?=\S).+?(?<=\S)~~") +_RICH_ANGLE_LITERAL_PATTERN: Final = re.compile(r"<[^<>\r\n]+>") +_RICH_HTML_OPEN_PATTERN: Final = re.compile( + r"^ {0,3}<(?Psection|div|details|summary|table|thead|tbody|tr|" + r"th|td|p|ul|ol|li)\b[^>]*>", + re.IGNORECASE, +) def unsupported_markdown_construct( @@ -572,6 +619,31 @@ def unsupported_markdown_construct( return None +def unsupported_rich_markdown_inline(line: str) -> UnsupportedConstruct | None: + """Classify inline syntax outside the accepted rich-v3 grammar.""" + + if type(line) is not str: + raise TypeError("rich Markdown inline classification requires exact text") + masked = line.rstrip(" \t") + if masked.endswith("\\"): + masked = masked[:-1] + "x" + for pattern in ( + _RICH_WIKILINK_PATTERN, + _RICH_FOOTNOTE_PATTERN, + _RICH_INLINE_MATH_PATTERN, + _RICH_INLINE_CODE_PATTERN, + _EMPHASIS_PATTERN, + _RICH_AUTOLINK_PATTERN, + _RICH_INLINE_LINK_PATTERN, + _RICH_REFERENCE_LINK_PATTERN, + _RICH_STRIKETHROUGH_PATTERN, + _RICH_ANGLE_LITERAL_PATTERN, + ): + masked = pattern.sub(lambda match: "x" * len(match.group()), masked) + construct = unsupported_markdown_construct(masked, supported_heading=False) + return None if construct is UnsupportedConstruct.LIST else construct + + @dataclass(frozen=True, slots=True) class CompilationFailure: """Typed all-or-nothing failure; it never carries partial ParsedDocument data.""" @@ -696,6 +768,10 @@ def _compilation_document( for warning in warnings ], } + if provenance.is_rich_v3: + provenance_value = document["provenance"] + assert isinstance(provenance_value, dict) + provenance_value["tokenCeiling"] = provenance.token_ceiling if provenance.is_structural_v2 or provenance.is_rich_v3: document["fragments"] = [ _fragment_document(fragment) for fragment in fragments @@ -790,6 +866,201 @@ def _expected_search_phrases( return tuple(dict.fromkeys(values)) +def _expected_rich_search_phrases( + section: ParsedSection, + source_text: str, +) -> tuple[str, ...]: + return tuple(dict.fromkeys((source_text, section.text))) + + +def _rich_table_cells(line: str) -> tuple[str, ...]: + stripped = line.strip() + if stripped.startswith("|"): + stripped = stripped[1:] + if stripped.endswith("|"): + stripped = stripped[:-1] + return tuple(cell.strip() for cell in stripped.split("|")) + + +def _rich_table_source_ranges(source: str) -> tuple[tuple[int, int], ...]: + raw_lines = source.splitlines(keepends=True) or (source,) + lines: list[str] = [] + starts: list[int] = [] + offset = 0 + for index, raw_line in enumerate(raw_lines): + content = raw_line.removesuffix("\n").removesuffix("\r") + bom_width = 1 if index == 0 and content.startswith("\ufeff") else 0 + lines.append(content[bom_width:]) + starts.append(offset + bom_width) + offset += len(raw_line) + ranges: list[tuple[int, int]] = [] + index = 0 + while index + 1 < len(lines): + header = _rich_table_cells(lines[index]) + separator = _rich_table_cells(lines[index + 1]) + if ( + "|" not in lines[index] + or len(header) < 2 + or not any(header) + or len(separator) < 2 + or not all( + re.fullmatch(r":?-+:?", cell.replace(" ", "")) is not None + for cell in separator + ) + ): + index += 1 + continue + end = index + 2 + while end < len(lines) and "|" in lines[end]: + end += 1 + ranges.append( + ( + starts[index], + starts[end - 1] + len(lines[end - 1]), + ) + ) + index = end + return tuple(ranges) + + +def _expected_rich_section_kind(source: str) -> SectionKind: + lines = source.splitlines() + if len(lines) >= 2 and lines[0] == "---" and lines[-1] == "---": + return SectionKind.PARAGRAPH + if ( + _RICH_ATX_HEADING_PATTERN.fullmatch(source) is not None + or len(lines) == 2 + and _RICH_SETEXT_PATTERN.fullmatch(lines[1]) is not None + ): + return SectionKind.HEADING + fence = _RICH_FENCE_PATTERN.match(lines[0]) if lines else None + if fence is not None and len(lines) >= 3: + marker = fence.group("fence") + if re.fullmatch( + rf"[ \t]{{0,3}}{re.escape(marker[0])}{{{len(marker)},}}[ \t]*", + lines[-1], + ): + return SectionKind.FENCED_CODE + if lines and _RICH_LIST_ITEM_PATTERN.fullmatch(lines[0]) is not None: + return SectionKind.LIST + table_ranges = _rich_table_source_ranges(source) + if table_ranges == ((0, len(source)),): + header_cells = _rich_table_cells(lines[0]) + separator_cells = _rich_table_cells(lines[1]) + header_width = len(header_cells) + source_rows = tuple(_rich_table_cells(line) for line in lines[2:]) + if ( + source_rows + and header_width >= 2 + and len(separator_cells) == header_width + and all(header_cells) + and all(len(row) == header_width and all(row) for row in source_rows) + ): + return SectionKind.TABLE + return SectionKind.PARAGRAPH + + +def _validate_rich_closed_grammar(section: ParsedSection, source: str) -> None: + if section.kind is not _expected_rich_section_kind(source): + raise ValueError("rich section kind must match its source grammar") + lines = source.splitlines() + metadata_valid = True + if section.kind is SectionKind.LIST: + item_matches = tuple( + match + for line in lines + if (match := _RICH_LIST_ITEM_PATTERN.fullmatch(line)) is not None + ) + metadata_valid = ( + bool(item_matches) + and section.list_ordered + is (re.match(r"[0-9]+[.)]", lines[0].lstrip()) is not None) + and section.list_items + == tuple(match.group(2).rstrip(" \t") for match in item_matches) + ) + elif section.kind is SectionKind.FENCED_CODE: + fence = _RICH_FENCE_PATTERN.match(lines[0]) if lines else None + assert fence is not None + marker = fence.group("fence") + expected_language = lines[0].lstrip()[len(marker) :].strip() or None + expected_body = "\n".join(lines[1:-1]) + metadata_valid = ( + section.code_language == expected_language + and section.code_body == expected_body + ) + elif section.kind is SectionKind.TABLE: + cells = tuple(_rich_table_cells(line) for line in lines) + metadata_valid = ( + len(cells) >= 3 + and section.table_header == cells[0] + and section.table_rows == cells[2:] + ) + if not metadata_valid: + raise ValueError("rich section metadata must match its source grammar") + if section.kind is SectionKind.FENCED_CODE: + return + if len(lines) >= 2 and lines[0] == "---" and lines[-1] == "---": + return + html_open = _RICH_HTML_OPEN_PATTERN.match(lines[0]) if lines else None + if ( + html_open is not None + and re.search( + rf"", + source, + re.IGNORECASE, + ) + ): + return + if ( + section.kind is SectionKind.PARAGRAPH + and len(lines) == 1 + and _THEMATIC_BREAK_PATTERN.fullmatch(lines[0]) is not None + ): + return + if ( + section.kind is SectionKind.PARAGRAPH + and lines + and _RICH_FENCE_PATTERN.match(lines[0]) is not None + ): + marker = _RICH_FENCE_PATTERN.match(lines[0]) + assert marker is not None + fence_text = marker.group("fence") + language = lines[0].lstrip()[len(fence_text) :].strip() + if language or not any(line.strip() for line in lines[1:]): + raise ValueError("rich section source must match the closed grammar") + return + if ( + section.kind is SectionKind.PARAGRAPH + and _rich_table_source_ranges(source) == ((0, len(source)),) + ): + if any( + unsupported_rich_markdown_inline(line) is not None + for line in lines + ): + raise ValueError("rich section source must match the closed grammar") + return + for line in lines: + inspected = line + if section.kind is SectionKind.HEADING: + match = _RICH_ATX_HEADING_PATTERN.fullmatch(line) + if match is not None: + inspected = match.group(2).strip() + elif _RICH_SETEXT_PATTERN.fullmatch(line) is not None: + continue + elif section.kind is SectionKind.LIST: + match = _RICH_LIST_ITEM_PATTERN.fullmatch(line) + if match is not None: + inspected = match.group(2) + elif line.startswith((" ", "\t")): + inspected = line.lstrip() + elif line.lstrip().startswith(">"): + inspected = line.lstrip()[1:].lstrip() + if inspected.startswith("[!"): + inspected = _RICH_FOOTNOTE_PATTERN.sub("x", inspected, count=1) + if unsupported_rich_markdown_inline(inspected) is not None: + raise ValueError("rich section source must match the closed grammar") + + def _validate_section_source(section: ParsedSection, source_text: str) -> None: lines = source_text.split("\n") if section.kind is SectionKind.HEADING: @@ -980,12 +1251,35 @@ def _validate_rich_content( canonical_text: str, sections: tuple[ParsedSection, ...], fragments: tuple[CompiledFragment, ...], + token_ceiling: int, ) -> None: if not sections or not fragments or len(sections) != len(fragments): raise ValueError("rich Markdown requires one Fragment per parsed section") canonical_bytes = canonical_text.encode("utf-8") prior_end = 0 refs: set[str] = set() + headings: list[ParsedSection] = [] + counters: dict[tuple[tuple[str, ...], SectionKind], int] = {} + kind_ordinals: dict[SectionKind, int] = {} + for table_start_character, table_end_character in _rich_table_source_ranges( + canonical_text + ): + table_start = len( + canonical_text[:table_start_character].encode("utf-8") + ) + table_end = len(canonical_text[:table_end_character].encode("utf-8")) + overlapping_sections = tuple( + section + for section in sections + if section.position.start.byte_offset < table_end + and section.position.end.byte_offset > table_start + ) + if ( + len(overlapping_sections) != 1 + or overlapping_sections[0].position.start.byte_offset != table_start + or overlapping_sections[0].position.end.byte_offset != table_end + ): + raise ValueError("rich table source must remain atomic") for section, fragment in zip(sections, fragments, strict=True): if ( fragment.kind is not section.kind @@ -1012,6 +1306,59 @@ def _validate_rich_content( raise ValueError("rich Fragment source text must match its span") if section.text != source and section.kind is not SectionKind.HEADING: raise ValueError("rich section text must match its span") + _validate_rich_closed_grammar(section, source) + if section.kind is SectionKind.HEADING: + assert section.level is not None + heading_lines = source.splitlines() + setext = ( + len(heading_lines) == 2 + and re.fullmatch(r"^ {0,3}(=+|-+)[ \t]*$", heading_lines[1]) + is not None + ) + atx = re.fullmatch( + r"^ {0,3}(#{1,6})[ \t]+(.+?)[ \t]*#*[ \t]*$", + source, + ) + expected_heading_text = ( + heading_lines[0].strip() + if setext + else atx.group(2).strip() + if atx is not None + else None + ) + expected_level = ( + 1 + if setext and heading_lines[1].lstrip().startswith("=") + else 2 + if setext + else len(atx.group(1)) + if atx is not None + else None + ) + if section.text != expected_heading_text or section.level != expected_level: + raise ValueError("rich heading metadata must match its source") + headings = [ + heading + for heading in headings + if heading.level is not None and heading.level < section.level + ] + parents = tuple(headings) + parent_path = parents[-1].path.segments if parents else ("document",) + counter_key = (parent_path, section.kind) + counters[counter_key] = counters.get(counter_key, 0) + 1 + expected_path = StructuralPath( + parent_path + (f"{section.kind.value}[{counters[counter_key]}]",) + ) + kind_ordinals[section.kind] = kind_ordinals.get(section.kind, 0) + 1 + expected_ref = f"fragment:{section.kind.value}:{kind_ordinals[section.kind]}" + if ( + section.path != expected_path + or fragment.parent_headings != parents + or fragment.fragment_ref != expected_ref + or fragment.search_phrases + != _expected_rich_search_phrases(section, source) + ): + raise ValueError("rich Fragment derivation must be exact") omitted = canonical_bytes[ prior_end : section.position.start.byte_offset ] @@ -1019,6 +1366,10 @@ def _validate_rich_content( raise ValueError("rich sections cannot omit non-whitespace source") if fragment.contextual_text != _expected_contextual_text(fragment): raise ValueError("rich Fragment context must be exact heading ancestry") + if len(re.findall(r"\S+", fragment.contextual_text)) > token_ceiling: + raise ValueError("rich Fragment exceeds its provenance-bound ceiling") + if section.kind is SectionKind.HEADING: + headings.append(section) refs.add(fragment.fragment_ref) prior_end = section.position.end.byte_offset if any(byte not in b" \t\r\n" for byte in canonical_bytes[prior_end:]): @@ -1197,6 +1548,7 @@ def deserialize_parsed_document(payload: bytes) -> ParsedDocument: compilation_digest_profile=cast( str, provenance_document["compilationDigestProfile"] ), + token_ceiling=cast(int | None, provenance_document.get("tokenCeiling")), ) sections = tuple( _section_from_document(value) diff --git a/tests/fixtures/markdown/rich-expanded.md b/tests/fixtures/markdown/rich-expanded.md new file mode 100644 index 00000000..d5351122 --- /dev/null +++ b/tests/fixtures/markdown/rich-expanded.md @@ -0,0 +1,27 @@ +# Expanded grammar + +> Ordinary blockquote with **strong**, *emphasis*, `inline code`, $x^2$, and ~~strike~~. + +[inline link](https://example.test) and ![inline image](image.png). + +[reference link][reference] and ![reference image][reference]. + +[reference]: https://example.test + + + +Next line uses a hard break.\ +Final line. + +* * * + +| A | B | +| --- | --- | +| ragged | + +| A | B | +| --- | --- | +| empty | | + +``` +Literal unmatched fence content. diff --git a/tests/unit/test_compiler_runner_acceptance.py b/tests/unit/test_compiler_runner_acceptance.py index 09c9fe06..7dba44e0 100644 --- a/tests/unit/test_compiler_runner_acceptance.py +++ b/tests/unit/test_compiler_runner_acceptance.py @@ -1,15 +1,25 @@ from __future__ import annotations +from dataclasses import replace from pathlib import Path import pytest from adapters.parsers.ragflow_markdown import compile_rich_markdown from engine.supply import ( + MARKDOWN_COMPILER_V3_VERSION, + MARKDOWN_RICH_CANONICALIZATION_PROFILE, + MARKDOWN_RICH_COMPILATION_DIGEST_PROFILE, CompilationFailure, + CompilationProvenance, + CompiledFragment, MarkdownCompilerConfig, ParsedDocument, + ParsedSection, SectionKind, + SourcePoint, + SourceSpan, + StructuralPath, ) FIXTURES = Path(__file__).parents[1] / "fixtures/markdown" @@ -18,6 +28,7 @@ "rich-frontmatter.md", "rich-headings-lists.md", "rich-code-tables.md", + "rich-expanded.md", "rich-inline-html.md", "rich-mixed-newlines.hex", ) @@ -81,6 +92,532 @@ def test_nested_setext_and_duplicate_headings_have_stable_ancestry() -> None: ) +def test_lone_cr_setext_heading_compiles_with_exact_span_and_ancestry() -> None: + source = "标题🙂\r====\r\r段落\r".encode() + + outcome = compile_rich_markdown(source, CONFIG) + + assert type(outcome) is ParsedDocument + heading, paragraph = outcome.fragments + assert heading.kind is SectionKind.HEADING + assert heading.source_text == "标题🙂\r====" + assert paragraph.parent_headings == (outcome.sections[0],) + for fragment in outcome.fragments: + span = fragment.position + assert source[span.start.byte_offset : span.end.byte_offset] == ( + fragment.source_text.encode("utf-8") + ) + + +@pytest.mark.parametrize( + "source", + ( + b"# T\n\n- body \n", + b"# T\n\n```\nbody\n``` \n", + b"# T\n\n - body \n", + b"# T\n\n ```\nbody\n ``` \n", + ), +) +def test_valid_blocks_retain_trailing_whitespace_without_crashing( + source: bytes, +) -> None: + outcome = compile_rich_markdown(source, CONFIG) + + assert type(outcome) is ParsedDocument + fragment = outcome.fragments[-1] + span = fragment.position + assert source[span.start.byte_offset : span.end.byte_offset] == ( + fragment.source_text.encode("utf-8") + ) + + +def test_fence_precedes_setext_recognition_when_first_body_line_is_rule_like() -> None: + source = b"# T\n\n```text\n---\nbody\n```\n" + + outcome = compile_rich_markdown(source, CONFIG) + + assert type(outcome) is ParsedDocument + fence = outcome.fragments[-1] + assert fence.kind is SectionKind.FENCED_CODE + span = fence.position + assert source[span.start.byte_offset : span.end.byte_offset] == ( + fence.source_text.encode("utf-8") + ) + + +@pytest.mark.parametrize( + "source", + ( + b"---\nkey: value\n---\n# T\n", + b"T\n===\n\n## Child\n", + b"# T\n\n- parent\n 1. child\n", + b"# T\n\n~~~text\nbody\n~~~\n", + b"# T\n\n| A | B |\n| --- | --- |\n| x | y |\n", + b"# T\n\n[[target|label]] and ![[asset]]\n", + b"# T\n\nText[^1].\n\n[^1]: Note.\n", + b"# T\n\n
body
\n", + b"# T\n\n> [!NOTE]\n> callout body\n", + b"# T\n\n> quoted paragraph\n", + b"# T\n\n$x^2$\n", + b"# T\n\n**strong** and *emphasis*\n", + b"# T\n\nUse `inline code`.\n", + b"# T\n\n[link](https://example.test) and ![image](image.png)\n", + b"# T\n\n[link][ref] and ![image][ref]\n\n[ref]: https://example.test\n", + b"# T\n\n~~strikethrough~~\n", + b"# T\n\n\n", + b"# T\n\nhard break \nnext line\n", + b"# T\n\nhard break\\\nnext line\n", + b"# T\n\n---\n", + b"# T\n\n* * *\n", + b"# T\n\n_ _ _\n", + b"# T\n\nordinary paragraph\n", + b"# T\n\n| A | B |\n| --- | --- |\n| x |\n", + b"# T\n\n| A | B |\n| --- | --- |\n| x | |\n", + b"# T\n\n```\nliteral unmatched fence\n", + ), +) +def test_every_adr_listed_rich_construct_is_explicitly_accepted( + source: bytes, +) -> None: + outcome = compile_rich_markdown(source, CONFIG) + + assert type(outcome) is ParsedDocument + assert outcome.fragments + + +def test_rich_constructor_rejects_forged_lineage_and_ancestry() -> None: + compiled = compile_rich_markdown(b"# Root\n\nBody\n", CONFIG) + assert type(compiled) is ParsedDocument + heading, body = compiled.sections + heading_fragment, body_fragment = compiled.fragments + forged_heading = replace( + heading, + text="WRONG", + path=type(heading.path)(("invented", "heading[7]")), + ) + forged_body = replace( + body, + path=type(body.path)(("invented", "paragraph[9]")), + ) + forged_fragments = ( + replace( + heading_fragment, + fragment_ref="fragment:heading:9", + path=forged_heading.path, + search_phrases=("poison-heading",), + ), + replace( + body_fragment, + fragment_ref="fragment:paragraph:99", + path=forged_body.path, + parent_headings=(), + contextual_text=body_fragment.source_text, + search_phrases=("poison-body",), + ), + ) + + expected_error = "rich (?:heading metadata|Fragment derivation)" + with pytest.raises(ValueError, match=expected_error): + ParsedDocument.rich_v3( + canonical_text=compiled.canonical_text, + sections=(forged_heading, forged_body), + fragments=forged_fragments, + provenance=compiled.provenance, + ) + + +def test_rich_constructor_rederives_section_kind_from_exact_source() -> None: + compiled = compile_rich_markdown(b"# Root\n", CONFIG) + assert type(compiled) is ParsedDocument + heading = compiled.sections[0] + fragment = compiled.fragments[0] + forged_path = StructuralPath(("document", "paragraph[1]")) + + with pytest.raises(ValueError, match="rich section kind"): + ParsedDocument.rich_v3( + canonical_text=compiled.canonical_text, + sections=( + replace( + heading, + kind=SectionKind.PARAGRAPH, + text=fragment.source_text, + path=forged_path, + level=None, + ), + ), + fragments=( + replace( + fragment, + fragment_ref="fragment:paragraph:1", + kind=SectionKind.PARAGRAPH, + path=forged_path, + search_phrases=(fragment.source_text,), + ), + ), + provenance=compiled.provenance, + ) + + +def test_rich_constructor_rederives_table_kind_from_exact_source() -> None: + compiled = compile_rich_markdown( + b"| A | B |\n| --- | --- |\n| x | y |\n", + CONFIG, + ) + assert type(compiled) is ParsedDocument + table = compiled.sections[0] + fragment = compiled.fragments[0] + forged_path = StructuralPath(("document", "paragraph[1]")) + + with pytest.raises(ValueError, match="rich section kind"): + ParsedDocument.rich_v3( + canonical_text=compiled.canonical_text, + sections=( + replace( + table, + kind=SectionKind.PARAGRAPH, + path=forged_path, + table_header=(), + table_rows=(), + ), + ), + fragments=( + replace( + fragment, + fragment_ref="fragment:paragraph:1", + kind=SectionKind.PARAGRAPH, + path=forged_path, + ), + ), + provenance=compiled.provenance, + ) + + +def test_rich_constructor_rejects_forged_table_kind_for_ragged_source() -> None: + compiled = compile_rich_markdown( + b"| A | B |\n| --- | --- |\n| x |\n", + CONFIG, + ) + assert type(compiled) is ParsedDocument + paragraph = compiled.sections[0] + fragment = compiled.fragments[0] + forged_path = StructuralPath(("document", "table[1]")) + + with pytest.raises(ValueError, match="rich section kind"): + ParsedDocument.rich_v3( + canonical_text=compiled.canonical_text, + sections=( + replace( + paragraph, + kind=SectionKind.TABLE, + path=forged_path, + table_header=("A", "B"), + table_rows=(("x",),), + ), + ), + fragments=( + replace( + fragment, + fragment_ref="fragment:table:1", + kind=SectionKind.TABLE, + path=forged_path, + ), + ), + provenance=compiled.provenance, + ) + + +def test_rich_constructor_rejects_language_bearing_unmatched_fence() -> None: + compiled = compile_rich_markdown(b"Plaintext\nbody\n", CONFIG) + assert type(compiled) is ParsedDocument + paragraph = compiled.sections[0] + fragment = compiled.fragments[0] + forged_source = "```python\nbody" + assert len(forged_source.encode("utf-8")) == fragment.position.end.byte_offset + + with pytest.raises(ValueError, match="closed grammar"): + ParsedDocument.rich_v3( + canonical_text=f"{forged_source}\n", + sections=(replace(paragraph, text=forged_source),), + fragments=( + replace( + fragment, + source_text=forged_source, + contextual_text=forged_source, + search_phrases=(forged_source,), + ), + ), + provenance=compiled.provenance, + ) + + +def test_rich_constructor_rejects_unlisted_construct_in_forged_document() -> None: + compiled = compile_rich_markdown(b"Plain\n", CONFIG) + assert type(compiled) is ParsedDocument + paragraph = compiled.sections[0] + fragment = compiled.fragments[0] + forged_source = "&" + + with pytest.raises(ValueError, match="closed grammar"): + ParsedDocument.rich_v3( + canonical_text=f"{forged_source}\n", + sections=(replace(paragraph, text=forged_source),), + fragments=( + replace( + fragment, + source_text=forged_source, + contextual_text=forged_source, + search_phrases=(forged_source,), + ), + ), + provenance=compiled.provenance, + ) + + +def test_rich_constructor_rejects_unlisted_construct_in_ragged_table() -> None: + compiled = compile_rich_markdown( + b"| A | B |\n| --- | --- |\n| x | value |\n| ragged |\n", + CONFIG, + ) + assert type(compiled) is ParsedDocument + paragraph = compiled.sections[0] + fragment = compiled.fragments[0] + forged_source = fragment.source_text.replace("value", "&") + + with pytest.raises(ValueError, match="closed grammar"): + ParsedDocument.rich_v3( + canonical_text=f"{forged_source}\n", + sections=(replace(paragraph, text=forged_source),), + fragments=( + replace( + fragment, + source_text=forged_source, + contextual_text=forged_source, + search_phrases=(forged_source,), + ), + ), + provenance=compiled.provenance, + ) + + +def test_rich_constructor_does_not_treat_arbitrary_pipe_lines_as_table() -> None: + compiled = compile_rich_markdown( + b"first | line\nsecond | line\nplain value\n", + CONFIG, + ) + assert type(compiled) is ParsedDocument + paragraph = compiled.sections[0] + fragment = compiled.fragments[0] + forged_source = fragment.source_text.replace("value", "&") + + with pytest.raises(ValueError, match="closed grammar"): + ParsedDocument.rich_v3( + canonical_text=f"{forged_source}\n", + sections=(replace(paragraph, text=forged_source),), + fragments=( + replace( + fragment, + source_text=forged_source, + contextual_text=forged_source, + search_phrases=(forged_source,), + ), + ), + provenance=compiled.provenance, + ) + + +def test_rich_constructor_rejects_split_fragments_of_one_atomic_ragged_table() -> None: + canonical_text = ( + "| A | B |\n" + "| --- | --- |\n" + "| one two three | four five six |\n" + "| ragged seven eight |\n" + ) + source_parts = ( + "| A | B |\n| --- |", + "--- |\n| one two three | four", + "five six |\n| ragged seven eight |", + ) + + def point(byte_offset: int) -> SourcePoint: + prefix = canonical_text.encode("utf-8")[:byte_offset].decode("utf-8") + return SourcePoint( + line=prefix.count("\n") + 1, + column=len(prefix.rsplit("\n", maxsplit=1)[-1]) + 1, + byte_offset=byte_offset, + ) + + sections: list[ParsedSection] = [] + fragments: list[CompiledFragment] = [] + search_start = 0 + for ordinal, source_part in enumerate(source_parts, start=1): + start = canonical_text.index(source_part, search_start) + end = start + len(source_part) + search_start = end + path = StructuralPath(("document", f"paragraph[{ordinal}]")) + position = SourceSpan(start=point(start), end=point(end)) + section = ParsedSection( + kind=SectionKind.PARAGRAPH, + text=source_part, + path=path, + position=position, + ) + sections.append(section) + fragments.append( + CompiledFragment( + fragment_ref=f"fragment:paragraph:{ordinal}", + kind=SectionKind.PARAGRAPH, + path=path, + position=position, + source_text=source_part, + contextual_text=source_part, + parent_headings=(), + search_phrases=(source_part,), + ) + ) + + with pytest.raises(ValueError, match="rich table source must remain atomic"): + ParsedDocument.rich_v3( + canonical_text=canonical_text, + sections=tuple(sections), + fragments=tuple(fragments), + provenance=CompilationProvenance( + compiler_version=MARKDOWN_COMPILER_V3_VERSION, + config_version="markdown-config-v3", + canonicalization_profile=MARKDOWN_RICH_CANONICALIZATION_PROFILE, + compilation_digest_profile=MARKDOWN_RICH_COMPILATION_DIGEST_PROFILE, + token_ceiling=8, + ), + ) + + +def test_rich_constructor_rejects_ragged_table_forged_as_table_then_paragraph() -> None: + table_source = "| A | B |\n| --- | --- |\n| x | y |" + ragged_source = "| ragged |" + canonical_text = f"{table_source}\n{ragged_source}\n" + table_document = compile_rich_markdown(table_source.encode("utf-8"), CONFIG) + ragged_document = compile_rich_markdown(ragged_source.encode("utf-8"), CONFIG) + assert type(table_document) is ParsedDocument + assert type(ragged_document) is ParsedDocument + table_section = table_document.sections[0] + table_fragment = table_document.fragments[0] + ragged_start = len(f"{table_source}\n".encode()) + ragged_end = ragged_start + len(ragged_source.encode("utf-8")) + ragged_path = StructuralPath(("document", "paragraph[1]")) + ragged_position = SourceSpan( + start=SourcePoint(line=4, column=1, byte_offset=ragged_start), + end=SourcePoint( + line=4, + column=len(ragged_source) + 1, + byte_offset=ragged_end, + ), + ) + ragged_section = replace( + ragged_document.sections[0], + path=ragged_path, + position=ragged_position, + ) + ragged_fragment = replace( + ragged_document.fragments[0], + path=ragged_path, + position=ragged_position, + ) + + with pytest.raises(ValueError, match="rich table source must remain atomic"): + ParsedDocument.rich_v3( + canonical_text=canonical_text, + sections=(table_section, ragged_section), + fragments=(table_fragment, ragged_fragment), + provenance=table_document.provenance, + ) + + +@pytest.mark.parametrize( + ("original", "forged_source"), + ( + (b"> plain\n", "> &"), + (b"- item\n plain\n", "- item\n &"), + ), +) +def test_rich_constructor_rejects_unlisted_construct_in_nested_context( + original: bytes, + forged_source: str, +) -> None: + compiled = compile_rich_markdown(original, CONFIG) + assert type(compiled) is ParsedDocument + section = compiled.sections[0] + fragment = compiled.fragments[0] + assert len(forged_source.encode()) == fragment.position.end.byte_offset + + with pytest.raises(ValueError, match="closed grammar"): + ParsedDocument.rich_v3( + canonical_text=f"{forged_source}\n", + sections=(replace(section, text=forged_source),), + fragments=( + replace( + fragment, + source_text=forged_source, + contextual_text=forged_source, + search_phrases=(forged_source,), + ), + ), + provenance=compiled.provenance, + ) + + +def test_rich_constructor_rederives_list_metadata() -> None: + compiled = compile_rich_markdown(b"- first\n 1. child\n", CONFIG) + assert type(compiled) is ParsedDocument + + with pytest.raises(ValueError, match="rich section metadata"): + ParsedDocument.rich_v3( + canonical_text=compiled.canonical_text, + sections=(replace(compiled.sections[0], list_items=("forged",)),), + fragments=compiled.fragments, + provenance=compiled.provenance, + ) + + +def test_rich_constructor_rederives_code_metadata() -> None: + compiled = compile_rich_markdown(b"```python\nbody\n```\n", CONFIG) + assert type(compiled) is ParsedDocument + + with pytest.raises(ValueError, match="rich section metadata"): + ParsedDocument.rich_v3( + canonical_text=compiled.canonical_text, + sections=( + replace( + compiled.sections[0], + code_language="forged", + code_body="forged", + ), + ), + fragments=compiled.fragments, + provenance=compiled.provenance, + ) + + +def test_rich_constructor_rederives_table_metadata() -> None: + compiled = compile_rich_markdown( + b"| A | B |\n| --- | --- |\n| x | y |\n", + CONFIG, + ) + assert type(compiled) is ParsedDocument + + with pytest.raises(ValueError, match="rich section metadata"): + ParsedDocument.rich_v3( + canonical_text=compiled.canonical_text, + sections=( + replace( + compiled.sections[0], + table_header=("forged",), + table_rows=(("forged",),), + ), + ), + fragments=compiled.fragments, + provenance=compiled.provenance, + ) + + @pytest.mark.parametrize( "frontmatter", ( diff --git a/tests/unit/test_compiler_runner_acceptance_report.py b/tests/unit/test_compiler_runner_acceptance_report.py index be5b81bf..9d07bf42 100644 --- a/tests/unit/test_compiler_runner_acceptance_report.py +++ b/tests/unit/test_compiler_runner_acceptance_report.py @@ -16,6 +16,10 @@ def test_acceptance_report_is_count_only_deterministic_and_written_under_ignore( "# Two\n\n| A | B |\n| --- | --- |\n| x | y |\n", encoding="utf-8", ) + (corpus / "refused.md").write_text( + "# Refused\n\n&\n", + encoding="utf-8", + ) output = tmp_path / ".context-engine/compiler-runner-acceptance.json" command = [ sys.executable, @@ -34,11 +38,20 @@ def test_acceptance_report_is_count_only_deterministic_and_written_under_ignore( report = json.loads(first_bytes) assert report["schemaVersion"] == "compiler-runner-acceptance-v1" - assert report["documents"] == {"accepted": 2, "refused": 0, "total": 2} + assert report["documents"] == { + "accepted": 2, + "acceptanceRate": "0.666667", + "refused": 1, + "total": 3, + } + assert report["refusalHistogram"] == { + "unsupported_construct:entity": 1, + } assert report["aggregateCompilationDigest"] assert report["maxFragmentTokenCount"] <= report["tokenCeiling"] assert report["constructHistogram"]["tables"] == 1 assert str(corpus) not in first_bytes.decode("utf-8") assert "one.md" not in first_bytes.decode("utf-8") + assert "refused.md" not in first_bytes.decode("utf-8") assert first_bytes == output.read_bytes() assert first.stdout == second.stdout diff --git a/tests/unit/test_compiler_runner_bounds.py b/tests/unit/test_compiler_runner_bounds.py index d4fe7838..f409f074 100644 --- a/tests/unit/test_compiler_runner_bounds.py +++ b/tests/unit/test_compiler_runner_bounds.py @@ -1,5 +1,7 @@ from __future__ import annotations +import pytest + from adapters.parsers.ragflow_markdown import compile_rich_markdown, rich_token_count from engine.supply import CompilationFailure, MarkdownCompilerConfig, ParsedDocument @@ -14,9 +16,14 @@ def test_oversize_blocks_split_under_hard_bound_with_exact_spans_and_ancestry() + "\n" ).encode() - outcome = compile_rich_markdown(source, CONFIG, token_ceiling=ceiling) + config = MarkdownCompilerConfig( + version="markdown-config-v3", + token_ceiling=ceiling, + ) + outcome = compile_rich_markdown(source, config) assert type(outcome) is ParsedDocument + assert outcome.provenance.token_ceiling == ceiling paragraph_fragments = outcome.fragments[2:] assert len(paragraph_fragments) > 1 for fragment in paragraph_fragments: @@ -39,7 +46,11 @@ def test_oversize_indivisible_code_block_refuses_instead_of_token_splitting() -> code = " ".join(f"operation_{index}()" for index in range(200)) source = f"# Root\r\n\r\n```python\r\n{code}\r\n```\r\n".encode() - outcome = compile_rich_markdown(source, CONFIG, token_ceiling=ceiling) + config = MarkdownCompilerConfig( + version="markdown-config-v3", + token_ceiling=ceiling, + ) + outcome = compile_rich_markdown(source, config) assert type(outcome) is CompilationFailure @@ -50,10 +61,38 @@ def test_every_emitted_fragment_obeys_the_ceiling() -> None: "# Root\n\n" + " ".join(f"word{index}" for index in range(80)) + "\n" ).encode() - outcome = compile_rich_markdown(source, CONFIG, token_ceiling=ceiling) + config = MarkdownCompilerConfig( + version="markdown-config-v3", + token_ceiling=ceiling, + ) + outcome = compile_rich_markdown(source, config) assert type(outcome) is ParsedDocument assert all( rich_token_count(fragment.contextual_text) <= ceiling for fragment in outcome.fragments ) + + +def test_rich_token_ceiling_is_validated_by_configuration() -> None: + with pytest.raises(ValueError, match="token ceiling"): + MarkdownCompilerConfig(version="markdown-config-v3", token_ceiling=0) + + +def test_ceiling_is_serialized_as_representation_provenance() -> None: + source = b"# Root\n\n" + b"word " * 80 + narrow = compile_rich_markdown( + source, + MarkdownCompilerConfig(version="markdown-config-v3", token_ceiling=16), + ) + wide = compile_rich_markdown( + source, + MarkdownCompilerConfig(version="markdown-config-v3", token_ceiling=64), + ) + + assert type(narrow) is ParsedDocument + assert type(wide) is ParsedDocument + assert narrow.provenance.token_ceiling == 16 + assert wide.provenance.token_ceiling == 64 + assert narrow.provenance != wide.provenance + assert narrow.compilation_digest != wide.compilation_digest diff --git a/tests/unit/test_compiler_runner_determinism.py b/tests/unit/test_compiler_runner_determinism.py index 35314e69..11e92a99 100644 --- a/tests/unit/test_compiler_runner_determinism.py +++ b/tests/unit/test_compiler_runner_determinism.py @@ -1,12 +1,19 @@ from __future__ import annotations +import base64 +import json +import subprocess +import sys from pathlib import Path import pytest from adapters.parsers.ragflow_markdown import compile_rich_markdown -from applications.compiler_runner import compile_in_local_compiler_runner -from engine.supply import MarkdownCompilerConfig, ParsedDocument +from engine.supply import ( + MarkdownCompilerConfig, + ParsedDocument, + deserialize_parsed_document, +) FIXTURES = Path(__file__).parents[1] / "fixtures/markdown" CONFIG = MarkdownCompilerConfig(version="markdown-config-v3") @@ -14,6 +21,7 @@ "rich-frontmatter.md", "rich-headings-lists.md", "rich-code-tables.md", + "rich-expanded.md", "rich-inline-html.md", "rich-mixed-newlines.hex", ) @@ -34,7 +42,31 @@ def test_rich_compilation_digest_is_stable_in_process_and_across_runner( first = compile_rich_markdown(source, CONFIG) second = compile_rich_markdown(source, CONFIG) - subprocess_result = compile_in_local_compiler_runner(source, CONFIG) + command = [ + sys.executable, + "-m", + "applications.compiler_runner", + "--compile", + "--config", + CONFIG.version, + ] + first_process = subprocess.run( + command, + input=source, + capture_output=True, + check=True, + ) + second_process = subprocess.run( + command, + input=source, + capture_output=True, + check=True, + ) + assert first_process.stdout == second_process.stdout + envelope = json.loads(first_process.stdout) + subprocess_result = deserialize_parsed_document( + base64.b64decode(envelope["document"], validate=True) + ) assert type(first) is ParsedDocument assert type(second) is ParsedDocument diff --git a/tests/unit/test_compiler_runner_production_boundary.py b/tests/unit/test_compiler_runner_production_boundary.py index 82072c65..1c810d09 100644 --- a/tests/unit/test_compiler_runner_production_boundary.py +++ b/tests/unit/test_compiler_runner_production_boundary.py @@ -27,6 +27,12 @@ def test_no_production_module_calls_the_local_compiler_runner() -> None: imported_from_runner = isinstance(node, ast.ImportFrom) and ( node.module == forbidden_module or any(alias.name == forbidden_symbol for alias in node.names) + or ( + node.module == "applications" + and any( + alias.name == "compiler_runner" for alias in node.names + ) + ) ) imported_runner = isinstance(node, ast.Import) and any( alias.name == forbidden_module for alias in node.names diff --git a/tests/unit/test_compiler_runner_refusal.py b/tests/unit/test_compiler_runner_refusal.py index 01c78ef1..9bc622f4 100644 --- a/tests/unit/test_compiler_runner_refusal.py +++ b/tests/unit/test_compiler_runner_refusal.py @@ -1,12 +1,19 @@ from __future__ import annotations +import json +import subprocess + import pytest +import applications.compiler_runner as compiler_runner +from adapters.parsers.ragflow_markdown import compile_rich_markdown from applications.compiler_runner import compile_in_local_compiler_runner from engine.supply import ( CompilationFailure, CompilationFailureCode, MarkdownCompilerConfig, + ParsedDocument, + UnsupportedConstruct, ) CONFIG = MarkdownCompilerConfig(version="markdown-config-v3") @@ -29,22 +36,10 @@ CompilationFailureCode.UNSUPPORTED_CONSTRUCT, ), (b"# Heading\n\ntruncated \xe4\xb8", CompilationFailureCode.INVALID_UTF8), - ( - b"# Heading\n\nplain paragraph\n> ordinary blockquote\n", - CompilationFailureCode.UNSUPPORTED_CONSTRUCT, - ), ( b"# Heading\n\n\n", CompilationFailureCode.UNSUPPORTED_CONSTRUCT, ), - ( - b"# Heading\n\n***\n", - CompilationFailureCode.UNSUPPORTED_CONSTRUCT, - ), - ( - b" ---\nkey: value\n---\n# Heading\n", - CompilationFailureCode.UNSUPPORTED_CONSTRUCT, - ), ], ) def test_malformed_source_returns_typed_failure_across_runner_boundary( @@ -56,3 +51,175 @@ def test_malformed_source_returns_typed_failure_across_runner_boundary( assert type(outcome) is CompilationFailure assert outcome.code is expected_code assert outcome.position is not None + + +@pytest.mark.parametrize( + ("source", "construct"), + ( + (b"# T\n\n&\n", UnsupportedConstruct.ENTITY), + (b"# T\n\nescaped\\*text\n", UnsupportedConstruct.ESCAPE), + (b"# T\n\n indented code\n", UnsupportedConstruct.CODE_BLOCK), + (b"# T\n\n> &\n", UnsupportedConstruct.ENTITY), + (b"# T\n\n- item\n &\n", UnsupportedConstruct.ENTITY), + ), +) +def test_unlisted_inline_constructs_are_typed_refusals( + source: bytes, + construct: UnsupportedConstruct, +) -> None: + outcome = compile_in_local_compiler_runner(source, CONFIG) + + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_CONSTRUCT + assert outcome.construct is construct + + +def test_unlisted_construct_inside_atomic_ragged_table_is_typed_refusal() -> None: + source = ( + b"| A | B |\n" + b"| --- | --- |\n" + b"| x | & |\n" + b"| ragged |\n" + ) + + outcome = compile_rich_markdown(source, CONFIG) + + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_CONSTRUCT + assert outcome.construct is UnsupportedConstruct.ENTITY + + +def test_runner_boundary_converts_unexpected_compiler_exception_to_typed_failure( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + def raise_unexpected( + source: bytes, + config: MarkdownCompilerConfig, + ) -> object: + raise ValueError("internal parser defect") + + monkeypatch.setattr(compiler_runner, "compile_rich_markdown", raise_unexpected) + + compiler_runner._emit(b"# T\n", CONFIG) + + outcome = compiler_runner._failure_from_document( + json.loads(capsys.readouterr().out)["failure"] + ) + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE + assert outcome.position is None + + +def test_runner_api_converts_child_process_failure_to_typed_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + subprocess, + "run", + lambda *args, **kwargs: subprocess.CompletedProcess( + args=args, + returncode=1, + stdout=b"", + stderr=b"internal parser defect", + ), + ) + + outcome = compile_in_local_compiler_runner(b"# T\n", CONFIG) + + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE + assert outcome.position is None + + +def test_direct_compiler_converts_domain_constructor_rejection_to_typed_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def reject_constructor(*args: object, **kwargs: object) -> ParsedDocument: + raise ValueError("domain constructor rejected parser metadata") + + monkeypatch.setattr( + ParsedDocument, + "rich_v3", + classmethod(reject_constructor), + ) + + outcome = compile_rich_markdown(b"# T\n", CONFIG) + + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE + assert outcome.position is not None + + +def test_direct_compiler_converts_section_constructor_rejection_to_typed_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def reject_section(*args: object, **kwargs: object) -> object: + raise ValueError("section constructor rejected parser metadata") + + monkeypatch.setattr( + "adapters.parsers.ragflow_markdown._section", + reject_section, + ) + + outcome = compile_rich_markdown(b"# T\n", CONFIG) + + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE + assert outcome.position is not None + + +def test_direct_compiler_converts_unexpected_section_exception_to_typed_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def reject_section(*args: object, **kwargs: object) -> object: + raise RuntimeError("unexpected section defect") + + monkeypatch.setattr( + "adapters.parsers.ragflow_markdown._section", + reject_section, + ) + + outcome = compile_rich_markdown(b"# T\n", CONFIG) + + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE + assert outcome.position is not None + + +def test_direct_compiler_converts_unexpected_domain_exception_to_typed_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def reject_constructor(*args: object, **kwargs: object) -> ParsedDocument: + raise RuntimeError("unexpected domain defect") + + monkeypatch.setattr( + ParsedDocument, + "rich_v3", + classmethod(reject_constructor), + ) + + outcome = compile_rich_markdown(b"# T\n", CONFIG) + + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE + assert outcome.position is not None + + +def test_direct_compiler_converts_parser_helper_rejection_to_typed_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def reject_parser_helper(*args: object, **kwargs: object) -> object: + raise ValueError("vendored parser helper rejected source") + + monkeypatch.setattr( + "third_party.ragflow.deepdoc.parser.markdown_parser." + "MarkdownElementExtractor._get_fence_marker", + reject_parser_helper, + ) + + outcome = compile_rich_markdown(b"Body\n", CONFIG) + + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE + assert outcome.position is not None diff --git a/tests/unit/test_compiler_runner_tables.py b/tests/unit/test_compiler_runner_tables.py index 58e18f9f..ac05346b 100644 --- a/tests/unit/test_compiler_runner_tables.py +++ b/tests/unit/test_compiler_runner_tables.py @@ -1,7 +1,15 @@ from __future__ import annotations +import pytest + from adapters.parsers.ragflow_markdown import compile_rich_markdown -from engine.supply import MarkdownCompilerConfig, ParsedDocument, SectionKind +from engine.supply import ( + CompilationFailure, + CompilationFailureCode, + MarkdownCompilerConfig, + ParsedDocument, + SectionKind, +) CONFIG = MarkdownCompilerConfig(version="markdown-config-v3") @@ -54,3 +62,68 @@ def test_table_spans_round_trip_against_original_crlf_bytes() -> None: table.source_text.encode("utf-8") ) assert b"\r\n" in table.source_text.encode("utf-8") + + +def test_table_with_trailing_whitespace_compiles_and_retains_exact_span() -> None: + source = ( + b"# Tables\n\n" + b"| Key | Value |\n" + b"| --- | --- |\n" + b"| alpha | ready | \n" + ) + + outcome = compile_rich_markdown(source, CONFIG) + + assert type(outcome) is ParsedDocument + table = next( + fragment + for fragment in outcome.fragments + if fragment.kind is SectionKind.TABLE + ) + span = table.position + assert source[span.start.byte_offset : span.end.byte_offset] == ( + table.source_text.encode("utf-8") + ) + assert table.source_text.endswith(" ") + + +@pytest.mark.parametrize( + "source", + ( + b"| A | B |\n| --- | --- |\n| x | y |\n| ragged |\n", + b"| A | B |\n| --- | --- |\n| x | y |\n| | |\n", + b"| A | B |\n| --- | --- | --- |\n| x | y |\n", + ), +) +def test_ragged_or_empty_row_keeps_table_as_one_exact_atomic_block( + source: bytes, +) -> None: + + outcome = compile_rich_markdown(source, CONFIG) + + assert type(outcome) is ParsedDocument + assert len(outcome.fragments) == 1 + fragment = outcome.fragments[0] + assert fragment.kind is SectionKind.PARAGRAPH + span = fragment.position + assert source[span.start.byte_offset : span.end.byte_offset] == source.rstrip(b"\n") + assert fragment.source_text.encode("utf-8") == source.rstrip(b"\n") + + +def test_oversize_ragged_table_refuses_instead_of_splitting_atomic_source() -> None: + source = ( + b"| A | B |\n" + b"| --- | --- |\n" + b"| one two three | four five six |\n" + b"| ragged seven eight |\n" + ) + bounded_config = MarkdownCompilerConfig( + version="markdown-config-v3", + token_ceiling=8, + ) + + outcome = compile_rich_markdown(source, bounded_config) + + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE + assert outcome.construct is None diff --git a/tests/unit/test_third_party_ragflow_registration.py b/tests/unit/test_third_party_ragflow_registration.py index effc9041..06b78b5f 100644 --- a/tests/unit/test_third_party_ragflow_registration.py +++ b/tests/unit/test_third_party_ragflow_registration.py @@ -5,8 +5,17 @@ import json import re import tomllib +from collections import Counter from pathlib import Path +import pytest + +from adapters.parsers.ragflow_markdown import compile_rich_markdown +from engine.supply import MarkdownCompilerConfig, ParsedDocument +from third_party.ragflow.deepdoc.parser.markdown_parser import ( + MarkdownElementExtractor, +) + REPOSITORY_ROOT = Path(__file__).parents[2] REGISTRATION_ROOT = REPOSITORY_ROOT / "third_party/ragflow" REGISTRATION_PATH = REGISTRATION_ROOT / "UPSTREAM.toml" @@ -155,7 +164,9 @@ def test_vendored_subtree_imports_only_approved_dependencies() -> None: assert "BSD 3-Clause" in modifications -def test_registered_parser_region_is_executed_by_the_ce_adapter() -> None: +def test_registered_parser_region_is_executed_by_the_ce_adapter( + monkeypatch: pytest.MonkeyPatch, +) -> None: adapter = REPOSITORY_ROOT / "adapters/parsers/ragflow_markdown.py" tree = ast.parse(adapter.read_bytes(), filename=str(adapter)) @@ -169,15 +180,38 @@ def test_registered_parser_region_is_executed_by_the_ce_adapter() -> None: assert [alias.name for alias in imports[0].names] == [ "MarkdownElementExtractor" ] - called_helpers = { - node.func.attr - for node in ast.walk(tree) - if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) - } - assert { + helper_names = ( "_get_fence_marker", "_is_closing_fence", "_is_table_row", "_is_table_separator_row", "_table_cells", - } <= called_helpers + ) + calls: Counter[str] = Counter() + for helper_name in helper_names: + original = getattr(MarkdownElementExtractor, helper_name) + + def recording_helper( + self: MarkdownElementExtractor, + *args: object, + _helper_name: str = helper_name, + _original: object = original, + ) -> object: + calls[_helper_name] += 1 + assert callable(_original) + return _original(self, *args) + + monkeypatch.setattr(MarkdownElementExtractor, helper_name, recording_helper) + + source = ( + b"# Executed\n\n" + b"```python\nprint('registered')\n```\n\n" + b"| Key | Value |\n| --- | --- |\n| parser | called |\n" + ) + outcome = compile_rich_markdown( + source, + MarkdownCompilerConfig("markdown-config-v3"), + ) + + assert type(outcome) is ParsedDocument + assert all(calls[name] > 0 for name in helper_names) From eaca41c061c2ff0a59da12a160acd08a682797f8 Mon Sep 17 00:00:00 2001 From: stone Date: Thu, 30 Jul 2026 01:44:29 +0800 Subject: [PATCH 06/16] docs: align rich compiler decision with evidence (#124) --- ...ompile-rich-markdown-in-an-owned-runner.md | 44 ++++++++++++++----- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md b/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md index 522088b2..c09f743b 100644 --- a/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md +++ b/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md @@ -38,15 +38,27 @@ index and executes under the exact parent WorkerLease binding. unordered lists, backtick or tilde fenced code blocks whose content may contain shorter fence runs, pipe tables, wikilinks, embeds, footnotes, HTML blocks, Obsidian - callouts, inline math, and ordinary paragraphs. CRLF, lone CR, and LF are - treated uniformly for grammar recognition and line/column calculation, but - exact decoded source bytes are retained so every byte span round-trips - against the original UTF-8 input and representation digests distinguish - distinct inputs. Frontmatter payload bytes are retained but not interpreted - or used as authority; semantic YAML validation, CommonMark compatibility, - and unrestricted HTML compatibility are not claimed. An optional leading - UTF-8 BOM is retained for representation identity and treated only as a - transport marker preceding the first Fragment. + callouts, ordinary blockquotes, inline math, emphasis/strong text, inline + code, inline or reference links/images and reference definitions, + strikethrough, angle-bracket literals, hard line breaks, thematic breaks, + and ordinary paragraphs. Pipe tables retain + ragged or empty rows as one exact atomic source block rather than silently + truncating the document; typed cell metadata remains best-effort within + that exact source block. + A bare unmatched fence-marker line plus following nonblank text is retained + as one exact literal paragraph block; an empty or language-bearing + unterminated fence remains a typed refusal. + These additional bounded constructs are explicit because the real-corpus + acceptance gate showed they are required to close the measured v1 gap; raw + syntax remains exact source text and is not re-rendered. CRLF, lone CR, and + LF are treated uniformly for grammar recognition and line/column + calculation, but exact decoded source bytes are retained so every byte span + round-trips against the original UTF-8 input and representation digests + distinguish distinct inputs. Frontmatter payload bytes are retained but not + interpreted or used as authority; semantic YAML validation, CommonMark + compatibility, and unrestricted HTML compatibility are not claimed. An + optional leading UTF-8 BOM is retained for representation identity and + treated only as a transport marker preceding the first Fragment. 3. **Existing output contracts.** The runner emits and deserializes only the existing `ParsedDocument`, `CompiledFragment`, `SourceSpan`, `StructuralPath`, `CompilationProvenance`, or typed `CompilationFailure` @@ -56,8 +68,13 @@ index and executes under the exact parent WorkerLease binding. line, and column span. Its `source_text` is exactly the original input byte slice at that span. Non-whitespace source bytes cannot be omitted. Table Fragments obey the same round-trip rule. Heading ancestry is derived during - compilation and copied into the same budget-visible Fragment. -5. **Hard splitting gate.** V3 uses a representation-bound token ceiling. The + compilation and copied into the same budget-visible Fragment. The v3 domain + constructor independently re-derives the closed source grammar, typed + section metadata, coordinates, paths, parent headings, stable Fragment + references, search phrases, contextual text, and the provenance-bound + ceiling; parser-supplied metadata cannot bypass those checks. +5. **Hard splitting gate.** V3 uses a representation-bound 2,048-token default + ceiling, recorded in both configuration and compilation provenance. The deterministic counter treats each non-whitespace run as one token. A block whose contextual text would exceed the ceiling is split at source token boundaries into ordered Fragments; every part retains the same preceding @@ -74,7 +91,10 @@ index and executes under the exact parent WorkerLease binding. classification, exact raw-byte position mapping, ancestry, typed output, and splitting because the upstream return shape cannot express those contracts. The process performs no network or database I/O and retains no - independent state, cache, index, or checkpoint. + independent state, cache, index, or checkpoint. Both the direct compiler + seam and subprocess envelope convert unexpected parser or domain-constructor + rejection into a typed all-or-nothing `CompilationFailure`; no partial + document or raw exception crosses the runner boundary. 7. **Activation remains deferred.** This decision proves the v3 pure transform and local acceptance reporting only. The active File import configuration, database publication functions, immutable Revision schemas, embeddings, From 8e9c72405d8c0df19f2787119609b6809e05da3e Mon Sep 17 00:00:00 2001 From: stone Date: Thu, 30 Jul 2026 02:30:11 +0800 Subject: [PATCH 07/16] fix: close compiler runner construction gaps (#124) --- adapters/parsers/ragflow_markdown.py | 51 +-- applications/compiler_runner.py | 37 +- engine/supply/markdown.py | 317 ++++++++++++++++++ tests/unit/test_compiler_runner_acceptance.py | 103 ++++++ tests/unit/test_compiler_runner_refusal.py | 51 +++ 5 files changed, 517 insertions(+), 42 deletions(-) diff --git a/adapters/parsers/ragflow_markdown.py b/adapters/parsers/ragflow_markdown.py index 2ada9904..bea6e662 100644 --- a/adapters/parsers/ragflow_markdown.py +++ b/adapters/parsers/ragflow_markdown.py @@ -686,27 +686,22 @@ def compile_rich_markdown( source_text = full_source[relative_start:relative_end] key = (parent_path, block.kind) counters[key] = counters.get(key, 0) + 1 - path = StructuralPath( - parent_path + (f"{block.kind.value}[{counters[key]}]",) - ) - position = _span( - normalized, - block.start + relative_start, - block.start + relative_end, - ) try: - section = _section(block, source_text, path, position) - except Exception: - return _failure( - CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE, + path = StructuralPath( + parent_path + (f"{block.kind.value}[{counters[key]}]",) + ) + position = _span( normalized, block.start + relative_start, + block.start + relative_end, + ) + section = _section(block, source_text, path, position) + ordinals[block.kind] = ordinals.get(block.kind, 0) + 1 + contextual = ( + f"{ancestry}\n\n{source_text}" if ancestry else source_text ) - ordinals[block.kind] = ordinals.get(block.kind, 0) + 1 - contextual = f"{ancestry}\n\n{source_text}" if ancestry else source_text - phrases = tuple(dict.fromkeys((source_text, section.text))) - fragments.append( - CompiledFragment( + phrases = tuple(dict.fromkeys((source_text, section.text))) + fragment = CompiledFragment( fragment_ref=( f"fragment:{block.kind.value}:{ordinals[block.kind]}" ), @@ -718,18 +713,24 @@ def compile_rich_markdown( parent_headings=parents, search_phrases=phrases, ) - ) + except Exception: + return _failure( + CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE, + normalized, + block.start + relative_start, + ) + fragments.append(fragment) sections.append(section) if block.kind is SectionKind.HEADING: headings.append(section) - provenance = CompilationProvenance( - compiler_version=MARKDOWN_COMPILER_V3_VERSION, - config_version=config.version, - canonicalization_profile=MARKDOWN_RICH_CANONICALIZATION_PROFILE, - compilation_digest_profile=MARKDOWN_RICH_COMPILATION_DIGEST_PROFILE, - token_ceiling=token_ceiling, - ) try: + provenance = CompilationProvenance( + compiler_version=MARKDOWN_COMPILER_V3_VERSION, + config_version=config.version, + canonicalization_profile=MARKDOWN_RICH_CANONICALIZATION_PROFILE, + compilation_digest_profile=MARKDOWN_RICH_COMPILATION_DIGEST_PROFILE, + token_ceiling=token_ceiling, + ) return ParsedDocument.rich_v3( canonical_text=normalized, sections=tuple(sections), diff --git a/applications/compiler_runner.py b/applications/compiler_runner.py index a2acd70c..882c2e26 100644 --- a/applications/compiler_runner.py +++ b/applications/compiler_runner.py @@ -89,21 +89,24 @@ def compile_in_local_compiler_runner( raise TypeError("compiler-runner config must be exact") if config.token_ceiling is None: raise ValueError("compiler-runner requires rich Markdown config") - completed = subprocess.run( - [ - sys.executable, - "-m", - _RUNNER_MODULE, - "--compile", - "--config", - config.version, - "--token-ceiling", - str(config.token_ceiling), - ], - input=source, - capture_output=True, - check=False, - ) + try: + completed = subprocess.run( + [ + sys.executable, + "-m", + _RUNNER_MODULE, + "--compile", + "--config", + config.version, + "--token-ceiling", + str(config.token_ceiling), + ], + input=source, + capture_output=True, + check=False, + ) + except Exception: + return _boundary_failure() if completed.returncode != 0: return _boundary_failure() try: @@ -121,12 +124,12 @@ def compile_in_local_compiler_runner( return deserialize_parsed_document( base64.b64decode(encoded, validate=True) ) - except (ValueError, TypeError): + except Exception: return _boundary_failure() if document.get("outcome") == "failure": try: return _failure_from_document(document.get("failure")) - except (KeyError, ValueError, TypeError): + except Exception: return _boundary_failure() return _boundary_failure() diff --git a/engine/supply/markdown.py b/engine/supply/markdown.py index 45106689..f01d84f6 100644 --- a/engine/supply/markdown.py +++ b/engine/supply/markdown.py @@ -923,6 +923,308 @@ def _rich_table_source_ranges(source: str) -> tuple[tuple[int, int], ...]: return tuple(ranges) +@dataclass(frozen=True, slots=True) +class _RichSourceBlock: + kind: SectionKind + start: int + end: int + indivisible: bool = False + heading_level: int | None = None + heading_text: str | None = None + + +def _rich_source_line_layout(source: str) -> tuple[list[str], list[int]]: + raw_lines = source.splitlines(keepends=True) or [source] + lines: list[str] = [] + starts: list[int] = [] + offset = 0 + for index, raw_line in enumerate(raw_lines): + content = raw_line.removesuffix("\n").removesuffix("\r") + bom_width = 1 if index == 0 and content.startswith("\ufeff") else 0 + lines.append(content[bom_width:]) + starts.append(offset + bom_width) + offset += len(raw_line) + return lines, starts + + +def _rich_source_line_end( + lines: list[str], starts: list[int], index: int +) -> int: + return starts[index] + len(lines[index]) + + +def _rich_table_starts_at(lines: list[str], index: int) -> bool: + if index + 1 >= len(lines) or "|" not in lines[index]: + return False + header = _rich_table_cells(lines[index]) + separator = _rich_table_cells(lines[index + 1]) + return ( + len(header) >= 2 + and any(header) + and len(separator) >= 2 + and all( + re.fullmatch(r":?-+:?", cell.replace(" ", "")) is not None + for cell in separator + ) + ) + + +def _rich_source_blocks(source: str) -> tuple[_RichSourceBlock, ...]: + """Independently derive v3 block boundaries from the canonical source.""" + + lines, starts = _rich_source_line_layout(source) + blocks: list[_RichSourceBlock] = [] + index = 0 + if lines and lines[0] == "---": + closing = next( + ( + candidate + for candidate in range(1, len(lines)) + if lines[candidate] == "---" + ), + None, + ) + if closing is None or closing == 1 or not any( + line.strip() for line in lines[1:closing] + ): + raise ValueError("rich frontmatter source must be complete and nonempty") + blocks.append( + _RichSourceBlock( + kind=SectionKind.PARAGRAPH, + start=starts[0], + end=_rich_source_line_end(lines, starts, closing), + ) + ) + index = closing + 1 + + while index < len(lines): + line = lines[index] + if not line.strip(): + index += 1 + continue + atx = _RICH_ATX_HEADING_PATTERN.fullmatch(line) + if atx is not None: + blocks.append( + _RichSourceBlock( + kind=SectionKind.HEADING, + start=starts[index], + end=_rich_source_line_end(lines, starts, index), + heading_level=len(atx.group(1)), + heading_text=atx.group(2).strip(), + ) + ) + index += 1 + continue + fence = _RICH_FENCE_PATTERN.match(line) + if fence is not None: + marker = fence.group("fence") + closing = index + 1 + while closing < len(lines) and re.fullmatch( + rf"[ \t]{{0,3}}{re.escape(marker[0])}{{{len(marker)},}}[ \t]*", + lines[closing], + ) is None: + closing += 1 + if closing >= len(lines): + language = line.lstrip()[len(marker) :].strip() + if not language and any( + candidate.strip() for candidate in lines[index + 1 :] + ): + blocks.append( + _RichSourceBlock( + kind=SectionKind.PARAGRAPH, + start=starts[index], + end=_rich_source_line_end(lines, starts, len(lines) - 1), + ) + ) + index = len(lines) + continue + raise ValueError("rich fenced source must match the closed grammar") + blocks.append( + _RichSourceBlock( + kind=SectionKind.FENCED_CODE, + start=starts[index], + end=_rich_source_line_end(lines, starts, closing), + indivisible=True, + ) + ) + index = closing + 1 + continue + if index + 1 < len(lines) and _RICH_SETEXT_PATTERN.fullmatch( + lines[index + 1] + ): + underline = lines[index + 1].lstrip() + blocks.append( + _RichSourceBlock( + kind=SectionKind.HEADING, + start=starts[index], + end=_rich_source_line_end(lines, starts, index + 1), + heading_level=1 if underline.startswith("=") else 2, + heading_text=line.strip(), + ) + ) + index += 2 + continue + if _rich_table_starts_at(lines, index): + end = index + 2 + while end < len(lines) and "|" in lines[end]: + end += 1 + header = _rich_table_cells(lines[index]) + separator = _rich_table_cells(lines[index + 1]) + rows = tuple( + _rich_table_cells(candidate) + for candidate in lines[index + 2 : end] + ) + valid = ( + bool(rows) + and len(header) >= 2 + and len(separator) == len(header) + and all( + len(row) == len(header) and all(row) + for row in (header, *rows) + ) + ) + blocks.append( + _RichSourceBlock( + kind=SectionKind.TABLE if valid else SectionKind.PARAGRAPH, + start=starts[index], + end=_rich_source_line_end(lines, starts, end - 1), + indivisible=not valid, + ) + ) + index = end + continue + if _RICH_LIST_ITEM_PATTERN.fullmatch(line) is not None: + end = index + 1 + while end < len(lines): + candidate = lines[end] + if _RICH_LIST_ITEM_PATTERN.fullmatch(candidate) is not None: + end += 1 + continue + if candidate.strip() and candidate.startswith((" ", "\t")): + end += 1 + continue + break + blocks.append( + _RichSourceBlock( + kind=SectionKind.LIST, + start=starts[index], + end=_rich_source_line_end(lines, starts, end - 1), + indivisible=True, + ) + ) + index = end + continue + if line.lstrip().startswith("<"): + if _RICH_ANGLE_LITERAL_PATTERN.fullmatch(line.strip()) is not None: + end = index + 1 + else: + end = index + 1 + while end < len(lines) and lines[end].strip(): + end += 1 + blocks.append( + _RichSourceBlock( + kind=SectionKind.PARAGRAPH, + start=starts[index], + end=_rich_source_line_end(lines, starts, end - 1), + ) + ) + index = end + continue + if line.lstrip().startswith(">"): + end = index + 1 + while end < len(lines) and lines[end].lstrip().startswith(">"): + end += 1 + blocks.append( + _RichSourceBlock( + kind=SectionKind.PARAGRAPH, + start=starts[index], + end=_rich_source_line_end(lines, starts, end - 1), + ) + ) + index = end + continue + if _THEMATIC_BREAK_PATTERN.fullmatch(line) is not None: + blocks.append( + _RichSourceBlock( + kind=SectionKind.PARAGRAPH, + start=starts[index], + end=_rich_source_line_end(lines, starts, index), + ) + ) + index += 1 + continue + end = index + 1 + while end < len(lines) and lines[end].strip(): + if ( + _RICH_ATX_HEADING_PATTERN.fullmatch(lines[end]) is not None + or _RICH_FENCE_PATTERN.match(lines[end]) is not None + or _RICH_LIST_ITEM_PATTERN.fullmatch(lines[end]) is not None + or lines[end].lstrip().startswith((">", "<")) + or _THEMATIC_BREAK_PATTERN.fullmatch(lines[end]) is not None + or _rich_table_starts_at(lines, end) + ): + break + end += 1 + blocks.append( + _RichSourceBlock( + kind=SectionKind.PARAGRAPH, + start=starts[index], + end=_rich_source_line_end(lines, starts, end - 1), + ) + ) + index = end + return tuple(blocks) + + +def _expected_rich_fragment_layout( + canonical_text: str, + token_ceiling: int, +) -> tuple[tuple[SectionKind, int, int], ...]: + expected: list[tuple[SectionKind, int, int]] = [] + headings: list[tuple[int, str]] = [] + for block in _rich_source_blocks(canonical_text): + if block.kind is SectionKind.HEADING: + assert block.heading_level is not None + headings = [ + heading for heading in headings if heading[0] < block.heading_level + ] + source = canonical_text[block.start : block.end] + ancestry = "\n\n".join( + f"{'#' * level} {text}" for level, text in headings + ) + capacity = token_ceiling - len(re.findall(r"\S+", ancestry)) + indivisible = block.indivisible or block.kind in { + SectionKind.HEADING, + SectionKind.LIST, + SectionKind.FENCED_CODE, + SectionKind.TABLE, + } + ranges: tuple[tuple[int, int], ...] + if indivisible: + if len(re.findall(r"\S+", source)) > capacity: + raise ValueError("rich indivisible source exceeds its ceiling") + ranges = ((0, len(source)),) + else: + tokens = tuple(re.finditer(r"\S+", source)) + if not tokens or capacity < 1: + raise ValueError("rich paragraph cannot fit its ceiling") + ranges = tuple( + ( + tokens[index].start(), + tokens[min(index + capacity, len(tokens)) - 1].end(), + ) + for index in range(0, len(tokens), capacity) + ) + for relative_start, relative_end in ranges: + start = len(canonical_text[: block.start + relative_start].encode("utf-8")) + end = len(canonical_text[: block.start + relative_end].encode("utf-8")) + expected.append((block.kind, start, end)) + if block.kind is SectionKind.HEADING: + assert block.heading_level is not None and block.heading_text is not None + headings.append((block.heading_level, block.heading_text)) + return tuple(expected) + + def _expected_rich_section_kind(source: str) -> SectionKind: lines = source.splitlines() if len(lines) >= 2 and lines[0] == "---" and lines[-1] == "---": @@ -1280,6 +1582,21 @@ def _validate_rich_content( or overlapping_sections[0].position.end.byte_offset != table_end ): raise ValueError("rich table source must remain atomic") + expected_layout = tuple( + (start, end) + for _, start, end in _expected_rich_fragment_layout( + canonical_text, token_ceiling + ) + ) + actual_layout = tuple( + ( + section.position.start.byte_offset, + section.position.end.byte_offset, + ) + for section in sections + ) + if actual_layout != expected_layout: + raise ValueError("rich block splitting must be exact") for section, fragment in zip(sections, fragments, strict=True): if ( fragment.kind is not section.kind diff --git a/tests/unit/test_compiler_runner_acceptance.py b/tests/unit/test_compiler_runner_acceptance.py index 7dba44e0..75acda04 100644 --- a/tests/unit/test_compiler_runner_acceptance.py +++ b/tests/unit/test_compiler_runner_acceptance.py @@ -226,6 +226,109 @@ def test_rich_constructor_rejects_forged_lineage_and_ancestry() -> None: ) +def _point_for(canonical_text: str, byte_offset: int) -> SourcePoint: + prefix = canonical_text.encode("utf-8")[:byte_offset].decode("utf-8") + logical = prefix.replace("\r\n", "\n").replace("\r", "\n") + return SourcePoint( + line=logical.count("\n") + 1, + column=len(logical.rsplit("\n", maxsplit=1)[-1]) + 1, + byte_offset=byte_offset, + ) + + +def _forged_sections( + canonical_text: str, + parts: tuple[tuple[SectionKind, str, tuple[str, ...]], ...], +) -> tuple[tuple[ParsedSection, ...], tuple[CompiledFragment, ...]]: + sections: list[ParsedSection] = [] + fragments: list[CompiledFragment] = [] + search_start = 0 + kind_ordinals: dict[SectionKind, int] = {} + for kind, source, list_items in parts: + start = canonical_text.index(source, search_start) + end = start + len(source.encode("utf-8")) + search_start = end + kind_ordinals[kind] = kind_ordinals.get(kind, 0) + 1 + ordinal = kind_ordinals[kind] + path = StructuralPath(("document", f"{kind.value}[{ordinal}]")) + position = SourceSpan( + start=_point_for(canonical_text, start), + end=_point_for(canonical_text, end), + ) + section = ParsedSection( + kind=kind, + text=source, + path=path, + position=position, + list_ordered=False if kind is SectionKind.LIST else None, + list_items=list_items, + ) + sections.append(section) + fragments.append( + CompiledFragment( + fragment_ref=f"fragment:{kind.value}:{ordinal}", + kind=kind, + path=path, + position=position, + source_text=source, + contextual_text=source, + parent_headings=(), + search_phrases=(source,), + ) + ) + return tuple(sections), tuple(fragments) + + +def test_rich_constructor_rejects_split_of_undersize_paragraph() -> None: + canonical_text = "one two three\n" + sections, fragments = _forged_sections( + canonical_text, + ( + (SectionKind.PARAGRAPH, "one", ()), + (SectionKind.PARAGRAPH, "two three", ()), + ), + ) + + with pytest.raises(ValueError, match="rich block splitting must be exact"): + ParsedDocument.rich_v3( + canonical_text=canonical_text, + sections=sections, + fragments=fragments, + provenance=CompilationProvenance( + compiler_version=MARKDOWN_COMPILER_V3_VERSION, + config_version="markdown-config-v3", + canonicalization_profile=MARKDOWN_RICH_CANONICALIZATION_PROFILE, + compilation_digest_profile=MARKDOWN_RICH_COMPILATION_DIGEST_PROFILE, + token_ceiling=2048, + ), + ) + + +def test_rich_constructor_rejects_split_of_one_contiguous_list() -> None: + canonical_text = "- one\n- two\n" + sections, fragments = _forged_sections( + canonical_text, + ( + (SectionKind.LIST, "- one", ("one",)), + (SectionKind.LIST, "- two", ("two",)), + ), + ) + + with pytest.raises(ValueError, match="rich block splitting must be exact"): + ParsedDocument.rich_v3( + canonical_text=canonical_text, + sections=sections, + fragments=fragments, + provenance=CompilationProvenance( + compiler_version=MARKDOWN_COMPILER_V3_VERSION, + config_version="markdown-config-v3", + canonicalization_profile=MARKDOWN_RICH_CANONICALIZATION_PROFILE, + compilation_digest_profile=MARKDOWN_RICH_COMPILATION_DIGEST_PROFILE, + token_ceiling=2048, + ), + ) + + def test_rich_constructor_rederives_section_kind_from_exact_source() -> None: compiled = compile_rich_markdown(b"# Root\n", CONFIG) assert type(compiled) is ParsedDocument diff --git a/tests/unit/test_compiler_runner_refusal.py b/tests/unit/test_compiler_runner_refusal.py index 9bc622f4..136fc792 100644 --- a/tests/unit/test_compiler_runner_refusal.py +++ b/tests/unit/test_compiler_runner_refusal.py @@ -11,6 +11,7 @@ from engine.supply import ( CompilationFailure, CompilationFailureCode, + CompiledFragment, MarkdownCompilerConfig, ParsedDocument, UnsupportedConstruct, @@ -206,6 +207,56 @@ def reject_constructor(*args: object, **kwargs: object) -> ParsedDocument: assert outcome.position is not None +def test_direct_compiler_converts_unexpected_fragment_exception_to_typed_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def reject_fragment(*args: object, **kwargs: object) -> CompiledFragment: + raise RuntimeError("unexpected Fragment defect") + + monkeypatch.setattr( + "adapters.parsers.ragflow_markdown.CompiledFragment", + reject_fragment, + ) + + outcome = compile_rich_markdown(b"# T\n", CONFIG) + + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE + assert outcome.position is not None + + +def test_runner_api_converts_unexpected_deserializer_exception_to_typed_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + subprocess, + "run", + lambda *args, **kwargs: subprocess.CompletedProcess( + args=args, + returncode=0, + stdout=json.dumps( + {"outcome": "parsed", "document": "e30="} + ).encode(), + stderr=b"", + ), + ) + + def reject_document(payload: bytes) -> ParsedDocument: + raise RuntimeError("unexpected deserializer defect") + + monkeypatch.setattr( + compiler_runner, + "deserialize_parsed_document", + reject_document, + ) + + outcome = compile_in_local_compiler_runner(b"# T\n", CONFIG) + + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE + assert outcome.position is None + + def test_direct_compiler_converts_parser_helper_rejection_to_typed_failure( monkeypatch: pytest.MonkeyPatch, ) -> None: From 66130fbe70051c831501753877cb6b6bc05d0fa2 Mon Sep 17 00:00:00 2001 From: stone Date: Thu, 30 Jul 2026 04:44:08 +0800 Subject: [PATCH 08/16] fix: close compiler runner evaluation gaps (#124) --- adapters/parsers/ragflow_markdown.py | 10 +- applications/compiler_runner.py | 2 + .../compiler-runner-acceptance-v1.schema.json | 4 + ...082-refine-rich-markdown-runner-closure.md | 63 +++++++ docs/decisions/README.md | 2 + engine/supply/markdown.py | 37 +++- eval/embedding_benchmark.py | 5 +- scripts/security_gate/runner.py | 28 +-- tests/catalog/test_m0_security_gate.py | 12 ++ tests/unit/test_compiler_runner_acceptance.py | 76 ++++++++ .../test_compiler_runner_acceptance_report.py | 169 +++++++++++++++++- .../unit/test_compiler_runner_determinism.py | 20 +++ ...est_compiler_runner_production_boundary.py | 50 ++++++ tests/unit/test_compiler_runner_refusal.py | 60 ++++++- ...test_embedding_benchmark_report_privacy.py | 4 +- 15 files changed, 508 insertions(+), 34 deletions(-) create mode 100644 docs/decisions/0082-refine-rich-markdown-runner-closure.md diff --git a/adapters/parsers/ragflow_markdown.py b/adapters/parsers/ragflow_markdown.py index bea6e662..4c391b32 100644 --- a/adapters/parsers/ragflow_markdown.py +++ b/adapters/parsers/ragflow_markdown.py @@ -24,6 +24,7 @@ SourceSpan, StructuralPath, UnsupportedConstruct, + is_markdown_control_character, unsupported_rich_markdown_inline, ) from third_party.ragflow.deepdoc.parser.markdown_parser import MarkdownElementExtractor @@ -120,14 +121,11 @@ def _normalize(source: bytes) -> str | CompilationFailure: safe_prefix, len(safe_prefix), ) - if any( - character == "\x00" or 0x7F <= ord(character) <= 0x9F - for character in decoded - ): + if any(is_markdown_control_character(character) for character in decoded): offset = next( index for index, character in enumerate(decoded) - if character == "\x00" or 0x7F <= ord(character) <= 0x9F + if is_markdown_control_character(character) ) return _failure( CompilationFailureCode.UNSUPPORTED_CONSTRUCT, @@ -176,7 +174,7 @@ def _frontmatter_end(lines: list[str]) -> int | None: if index == 1 or not any(line.strip() for line in lines[1:index]): return -1 return index - return -1 + return None def _blocks(text: str) -> tuple[_Block, ...] | CompilationFailure: diff --git a/applications/compiler_runner.py b/applications/compiler_runner.py index 882c2e26..6780bc76 100644 --- a/applications/compiler_runner.py +++ b/applications/compiler_runner.py @@ -27,6 +27,7 @@ ) _RUNNER_MODULE: Final = "applications.compiler_runner" +COMPILER_RUNNER_TIMEOUT_SECONDS: Final = 30.0 def _boundary_failure() -> CompilationFailure: @@ -104,6 +105,7 @@ def compile_in_local_compiler_runner( input=source, capture_output=True, check=False, + timeout=COMPILER_RUNNER_TIMEOUT_SECONDS, ) except Exception: return _boundary_failure() diff --git a/docs/contracts/compiler-runner-acceptance-v1.schema.json b/docs/contracts/compiler-runner-acceptance-v1.schema.json index 55a21550..7af71130 100644 --- a/docs/contracts/compiler-runner-acceptance-v1.schema.json +++ b/docs/contracts/compiler-runner-acceptance-v1.schema.json @@ -70,6 +70,10 @@ }, "refusalHistogram": { "type": "object", + "propertyNames": { + "type": "string", + "pattern": "^(?:invalid_utf8|unsupported_document_shape|unsupported_construct:(?:atx_closing_sequence|blockquote|code_block|control_character|emphasis|entity|escape|frontmatter_or_rule|hard_break|html|inline_code|link_or_image|list|nested_heading|strikethrough|table))$" + }, "additionalProperties": {"type": "integer", "minimum": 1} }, "maxFragmentTokenCount": {"type": "integer", "minimum": 0}, diff --git a/docs/decisions/0082-refine-rich-markdown-runner-closure.md b/docs/decisions/0082-refine-rich-markdown-runner-closure.md new file mode 100644 index 00000000..55800760 --- /dev/null +++ b/docs/decisions/0082-refine-rich-markdown-runner-closure.md @@ -0,0 +1,63 @@ +--- +name: adr-0082-refine-rich-markdown-runner-closure +version: "1.0.0" +description: > + Refine rich-Markdown frontmatter disambiguation, provenance identity, and + bounded runner termination without reopening the v3 representation. +--- + +# 0082. Refine rich Markdown runner closure + +- Status: accepted +- Date: 2026-07-30 +- Refines: ADR-0079 + +## Context + +ADR-0079 accepts both nonempty `---`-delimited frontmatter and thematic breaks, +but it did not define how an initial `---` without a valid closing delimiter is +classified. Treating every leading `---` as frontmatter made the accepted +thematic-break grammar depend on document position. Independent construction +testing also showed that v3 profile labels alone did not bind the exact +compiler/configuration identity, and a runner subprocess without a deadline +could fail to produce either an accepted document or a typed refusal. + +## Decision + +1. A leading `---` opens frontmatter only when a later `---` closes a nonempty + payload. Without that closing delimiter, the leading line is an accepted + thematic break. This delimiter-complete rule preserves both constructs in + the closed grammar and does not infer metadata from ordinary following prose. +2. Rich provenance binds the exact `context-engine-markdown-v3` compiler and + `markdown-config-v3` configuration identifiers as well as the v3 profiles. + The self-validating domain constructor rejects older or arbitrary identities. +3. The local compiler-runner call has a fixed positive timeout. Child timeout + or launch failure produces the same content-free typed boundary failure as + any other subprocess-boundary failure; an unbounded wait is not an outcome. +4. The constructor and parser share only the closed control-character + classifier: C0 controls other than tab and line endings, DEL, and C1 controls + are refused. The constructor invokes it independently over exact source, so + fenced-code metadata cannot bypass a parser-ingress check. + +## Rationale + +Delimiter completeness makes the two already accepted `---` meanings +deterministic without changing v1 or v2. Exact identity binding keeps stored +provenance interpretable, while a positive deadline completes ADR-0079's +all-or-nothing process contract even when a child wedges before emitting bytes. +Sharing the finite character predicate prevents drift without trusting any +parser-provided structure, span, or construct metadata. + +## Consequences + +- A document beginning with an unclosed `---` compiles that line as a thematic + break; complete nonempty frontmatter retains its exact raw span. +- Forged v3 documents cannot substitute an older or arbitrary compiler/config + identity while retaining rich profiles. +- Local compiler-runner callers receive a typed content-free failure after the + bounded deadline instead of waiting indefinitely. + +## Revisit trigger + +Revisit before changing leading delimiter disambiguation, v3 identity binding, +or the fixed subprocess timeout into a caller-controlled runtime parameter. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 5b6e46b1..0a967735 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -43,6 +43,7 @@ kernel, capability separation, and publication visibility model. | First Markdown compiler | [0036 — Deterministic narrow Markdown compilation](0036-compile-narrow-markdown-deterministically.md) | Exact bytes compile purely into one canonical heading-plus-paragraph ParsedDocument with normalized coordinates and versioned content/compilation identities | Path-coupled parsing, silent unsupported syntax, partial documents, unversioned derived identities, or parser-side I/O | | Structural Markdown compiler | [0038 — Structural Markdown units](0038-compile-and-publish-structural-markdown.md) | Explicit v2 compilation publishes one coherent Fragment per heading, paragraph, list, fenced code block, or table with exact provenance and same-Fragment heading ancestry | Silent v1 reinterpretation, item/cell fragmentation, post-authorization parent expansion, or unbudgeted context | | Rich Markdown compiler-runner | [0079 — Rich Markdown in an owned runner](0079-compile-rich-markdown-in-an-owned-runner.md) | Explicit v3 pure compilation accepts the closed rich grammar with exact source spans, same-Fragment ancestry, and a hard token ceiling while production activation stays deferred | Silent v1/v2 reinterpretation, runner-local state, unbounded Fragments, or activating publication without a separate decision | +| Rich Markdown runner closure | [0082 — Refine rich Markdown runner closure](0082-refine-rich-markdown-runner-closure.md) | Leading `---` is frontmatter only with a nonempty closing delimiter; exact v3 identities and a fixed subprocess timeout complete the typed runner contract | Position-dependent thematic-break refusal, relabeled v3 provenance, or an unbounded child wait | | Unchanged File acquisition | [0039 — File acquisition no-op](0039-deduplicate-unchanged-file-acquisitions.md) | Tenant/source/resource-scoped canonical identity and one PostgreSQL guard lock classify a complete active artifact before publication; each observation retains an immutable digest-only outcome | Cross-tenant/global deduplication, process-local locking, partial-artifact reuse, silent version reuse, or no-op by bypassing publication validation | | File Resource deletion | [0042 — Tombstone before cleanup](0042-tombstone-file-resources-before-cleanup.md) | One trusted Control transaction tombstones the active File Resource, advances its Organization Policy Epoch, and records immutable pending cleanup lineage before any physical deletion | Cleanup-defined visibility, caller-authored tenant/epoch/cleanup identity, index deletion as authorization, restore, or native watcher claims | | File Source progress | [0043 — Separate acquisition and publication progress](0043-separate-file-acquisition-progress-from-publication-progress.md), [0072 — Report File source status with closed refusals](0072-report-file-source-status-with-closed-refusals.md) | Append accepted changes separately from contiguous Runtime-visibility completion and expose content-free operational status through an Organization/Source-scoped Control read | One ambiguous checkpoint, skipped publication gaps, retained source/compiler diagnostics, Runtime authorization from status or watermarks, or false standard ProviderPort capability claims | @@ -186,3 +187,4 @@ touched: - [0079 — Compile rich Markdown in an owned runner](0079-compile-rich-markdown-in-an-owned-runner.md) - [0080 — Refuse authoritative evaluation without an executor](0080-refuse-authoritative-evaluation-without-an-executor.md) - [0081 — Seal candidate discovery behind a data-only session](0081-seal-candidate-discovery-behind-a-data-only-session.md) +- [0082 — Refine rich Markdown runner closure](0082-refine-rich-markdown-runner-closure.md) diff --git a/engine/supply/markdown.py b/engine/supply/markdown.py index f01d84f6..f26680ac 100644 --- a/engine/supply/markdown.py +++ b/engine/supply/markdown.py @@ -30,6 +30,17 @@ _MAX_VERSION_LENGTH: Final = 128 +def is_markdown_control_character(character: str) -> bool: + """Return whether one character is outside the closed Markdown grammar.""" + + if type(character) is not str or len(character) != 1: + raise TypeError("Markdown control classification requires one character") + codepoint = ord(character) + return (codepoint < 0x20 and character not in "\t\n\r") or ( + 0x7F <= codepoint <= 0x9F + ) + + def _require_version(value: object) -> str: if ( type(value) is not str @@ -311,6 +322,13 @@ def __post_init__(self) -> None: if self.content_hash_profile != MARKDOWN_CONTENT_HASH_PROFILE: raise ValueError("content hash profile must use the active version") if self.is_rich_v3: + if ( + self.compiler_version != MARKDOWN_COMPILER_V3_VERSION + or self.config_version != "markdown-config-v3" + ): + raise ValueError( + "rich provenance identity requires the v3 compiler and config" + ) if type(self.token_ceiling) is not int or self.token_ceiling < 1: raise ValueError("rich provenance requires a positive token ceiling") elif self.token_ceiling is not None: @@ -984,18 +1002,19 @@ def _rich_source_blocks(source: str) -> tuple[_RichSourceBlock, ...]: ), None, ) - if closing is None or closing == 1 or not any( + if closing == 1 or closing is not None and not any( line.strip() for line in lines[1:closing] ): raise ValueError("rich frontmatter source must be complete and nonempty") - blocks.append( - _RichSourceBlock( - kind=SectionKind.PARAGRAPH, - start=starts[0], - end=_rich_source_line_end(lines, starts, closing), + if closing is not None: + blocks.append( + _RichSourceBlock( + kind=SectionKind.PARAGRAPH, + start=starts[0], + end=_rich_source_line_end(lines, starts, closing), + ) ) - ) - index = closing + 1 + index = closing + 1 while index < len(lines): line = lines[index] @@ -1557,6 +1576,8 @@ def _validate_rich_content( ) -> None: if not sections or not fragments or len(sections) != len(fragments): raise ValueError("rich Markdown requires one Fragment per parsed section") + if any(is_markdown_control_character(character) for character in canonical_text): + raise ValueError("rich Markdown cannot contain a control character") canonical_bytes = canonical_text.encode("utf-8") prior_end = 0 refs: set[str] = set() diff --git a/eval/embedding_benchmark.py b/eval/embedding_benchmark.py index b349eff6..620b79c8 100644 --- a/eval/embedding_benchmark.py +++ b/eval/embedding_benchmark.py @@ -882,7 +882,7 @@ def _validate_report(document: object, schema: object) -> None: def validate_json_schema_document(value: object, schema: object) -> None: - """Validate documents against the tracked schema vocabulary used by eval.""" + """Validate documents against the tracked schema vocabulary used by reports.""" _validate_bounded_json_value(value) _validate_bounded_json_value(schema) @@ -928,7 +928,10 @@ def _validate_json_schema( raise BenchmarkUnavailable("benchmark report schema is unavailable") properties = _object(schema.get("properties", {})) additional = schema.get("additionalProperties", True) + property_names = schema.get("propertyNames") for key, item in document.items(): + if property_names is not None: + _validate_json_schema(key, _object(property_names), root) child_schema = properties.get(key) if child_schema is not None: _validate_json_schema(item, _object(child_schema), root) diff --git a/scripts/security_gate/runner.py b/scripts/security_gate/runner.py index 9d2803e5..eb1a43e7 100644 --- a/scripts/security_gate/runner.py +++ b/scripts/security_gate/runner.py @@ -260,6 +260,20 @@ def build_pytest_command( ) +def _report_execution_command( + selectors: Sequence[str], +) -> list[str]: + """Describe gate execution without retaining machine-local absolute paths.""" + + return list( + build_pytest_command( + selectors, + raw_path=Path(".context-engine/security-gate") / RAW_ARTIFACT_NAME, + python_executable="python", + ) + ) + + def _execute_pytest( command: Sequence[str], *, cwd: Path, env: Mapping[str, str] ) -> int: @@ -450,12 +464,7 @@ def _provenance( return { "runnerVersion": RUNNER_VERSION, **_git_state(paths.repository_root), - "executionCommand": list( - build_pytest_command( - selectors, - raw_path=paths.output_directory / RAW_ARTIFACT_NAME, - ) - ), + "executionCommand": _report_execution_command(selectors), "catalogDigest": _file_digest(paths.catalog), "fixtureDigest": canonical_digest(fixtures), "catalogSchemaDigest": _file_digest(paths.catalog_schema), @@ -536,12 +545,7 @@ def _best_effort_provenance( if live_database_revision is not None: provenance["liveDatabaseRevision"] = live_database_revision if selectors: - provenance["executionCommand"] = list( - build_pytest_command( - selectors, - raw_path=paths.output_directory / RAW_ARTIFACT_NAME, - ) - ) + provenance["executionCommand"] = _report_execution_command(selectors) configuration_fields = { field: provenance[field] for field in ( diff --git a/tests/catalog/test_m0_security_gate.py b/tests/catalog/test_m0_security_gate.py index a2b5ab46..42a94ac4 100644 --- a/tests/catalog/test_m0_security_gate.py +++ b/tests/catalog/test_m0_security_gate.py @@ -656,6 +656,18 @@ def test_provenance_has_exact_nonsecret_config_migration_and_fixture_digests( assert len(cast(str, provenance["fixtureDigest"])) == 64 assert len(cast(str, provenance["configurationDigest"])) == 64 assert "database.env" not in json.dumps(provenance) + assert provenance["executionCommand"] == [ + "python", + "-m", + "pytest", + "-p", + "scripts.security_gate.pytest_plugin", + "--security-gate-raw", + ".context-engine/security-gate/raw-evidence.json", + "--strict-markers", + "--strict-config", + "tests/test_gate_sample.py::test_fixture", + ] def test_git_state_hashes_staged_unstaged_and_untracked_nonignored_content( diff --git a/tests/unit/test_compiler_runner_acceptance.py b/tests/unit/test_compiler_runner_acceptance.py index 75acda04..eed9c959 100644 --- a/tests/unit/test_compiler_runner_acceptance.py +++ b/tests/unit/test_compiler_runner_acceptance.py @@ -185,6 +185,16 @@ def test_every_adr_listed_rich_construct_is_explicitly_accepted( assert outcome.fragments +@pytest.mark.parametrize("source", (b"---\n", b"---\n\ntext\n")) +def test_leading_unclosed_dash_rule_is_an_accepted_thematic_break( + source: bytes, +) -> None: + outcome = compile_rich_markdown(source, CONFIG) + + assert type(outcome) is ParsedDocument + assert outcome.fragments[0].source_text == "---" + + def test_rich_constructor_rejects_forged_lineage_and_ancestry() -> None: compiled = compile_rich_markdown(b"# Root\n\nBody\n", CONFIG) assert type(compiled) is ParsedDocument @@ -453,6 +463,72 @@ def test_rich_constructor_rejects_language_bearing_unmatched_fence() -> None: ) +@pytest.mark.parametrize( + "control_character", + ("\x00", "\x07", "\x1b", "\x1f", "\x7f", "\x85"), +) +def test_rich_constructor_rejects_every_control_character_forged_inside_fence( + control_character: str, +) -> None: + compiled = compile_rich_markdown(b"```text\nbody\n```\n", CONFIG) + assert type(compiled) is ParsedDocument + section = compiled.sections[0] + fragment = compiled.fragments[0] + forged_body = f"bo{control_character}y" + forged_source = fragment.source_text.replace("body", forged_body) + + with pytest.raises(ValueError, match="control character"): + ParsedDocument.rich_v3( + canonical_text=f"{forged_source}\n", + sections=( + replace( + section, + text=forged_source, + code_body=forged_body, + ), + ), + fragments=( + replace( + fragment, + source_text=forged_source, + contextual_text=forged_source, + search_phrases=(forged_source,), + ), + ), + provenance=compiled.provenance, + ) + + +@pytest.mark.parametrize( + ("compiler_version", "config_version"), + ( + ("context-engine-markdown-v1", "markdown-config-v1"), + ("arbitrary-compiler", "arbitrary-config"), + ), +) +def test_rich_constructor_rejects_forged_v3_provenance_identity( + compiler_version: str, + config_version: str, +) -> None: + compiled = compile_rich_markdown(b"Plain\n", CONFIG) + assert type(compiled) is ParsedDocument + + with pytest.raises(ValueError, match="rich provenance identity"): + provenance = CompilationProvenance( + compiler_version=compiler_version, + config_version=config_version, + canonicalization_profile=MARKDOWN_RICH_CANONICALIZATION_PROFILE, + compilation_digest_profile=MARKDOWN_RICH_COMPILATION_DIGEST_PROFILE, + token_ceiling=2048, + ) + ParsedDocument.rich_v3( + canonical_text=compiled.canonical_text, + sections=compiled.sections, + fragments=compiled.fragments, + provenance=provenance, + ) + + def test_rich_constructor_rejects_unlisted_construct_in_forged_document() -> None: compiled = compile_rich_markdown(b"Plain\n", CONFIG) assert type(compiled) is ParsedDocument diff --git a/tests/unit/test_compiler_runner_acceptance_report.py b/tests/unit/test_compiler_runner_acceptance_report.py index 9d07bf42..ffb08e38 100644 --- a/tests/unit/test_compiler_runner_acceptance_report.py +++ b/tests/unit/test_compiler_runner_acceptance_report.py @@ -1,10 +1,169 @@ from __future__ import annotations import json +import re import subprocess import sys +from copy import deepcopy +from hashlib import sha256 from pathlib import Path +import pytest + +from eval.embedding_benchmark import ( + BenchmarkUnavailable, + validate_json_schema_document, +) + +REPOSITORY_ROOT = Path(__file__).parents[2] +REPORT_SCHEMA = ( + REPOSITORY_ROOT / "docs/contracts/compiler-runner-acceptance-v1.schema.json" +) +_PERSONAL_ROOT_PATTERNS = ( + re.compile(r"/" + "Users" + r"/[^/\s]+/"), + re.compile(r"/" + "home" + r"/[^/\s]+/"), + re.compile(r"[A-Za-z]:\\(?:Users|Documents and Settings)\\", re.IGNORECASE), +) +_PRIVATE_ROOT_FRAGMENT_DIGESTS = frozenset( + {"c83ea566573fdfcacb79f350f86ea53935437f5672e1fe97703320cce4725394"} +) +_PRIVACY_BEARING_SCHEMA_WORDS = frozenset( + {"excerpt", "file", "filename", "path", "root", "source", "text", "title"} +) + + +def _schema_property_names(value: object) -> set[str]: + if type(value) is dict: + document = value + names: set[str] = set() + properties = document.get("properties") + if type(properties) is dict: + names.update(str(key).casefold() for key in properties) + names.update( + nested + for item in document.values() + for nested in _schema_property_names(item) + ) + return names + if type(value) is list: + return { + nested for item in value for nested in _schema_property_names(item) + } + return set() + + +def _schema_name_words(name: str) -> set[str]: + return { + word.casefold() + for word in re.findall( + r"[A-Z]+(?=[A-Z][a-z]|\b)|[A-Z]?[a-z]+|[0-9]+", + name, + ) + } + + +def _word_sequence_digests(value: str) -> set[str]: + words = tuple( + word.casefold() + for word in re.findall(r"[A-Z]+(?=[A-Z][a-z]|\b)|[A-Z]?[a-z]+", value) + ) + return { + sha256("".join(words[index:end]).encode("utf-8")).hexdigest() + for index in range(len(words)) + for end in range(index + 1, min(index + 4, len(words)) + 1) + } + + +def test_tracked_tree_and_acceptance_schema_cannot_carry_private_paths() -> None: + tracked = subprocess.run( + ["git", "ls-files", "-z"], + cwd=REPOSITORY_ROOT, + check=True, + capture_output=True, + ).stdout.split(b"\0") + leaks: list[str] = [] + for raw_path in tracked: + if not raw_path: + continue + path = REPOSITORY_ROOT / raw_path.decode("utf-8") + content = path.read_text(encoding="utf-8", errors="ignore") + if any(pattern.search(content) for pattern in _PERSONAL_ROOT_PATTERNS) or ( + _PRIVATE_ROOT_FRAGMENT_DIGESTS & _word_sequence_digests(content) + ): + leaks.append(path.relative_to(REPOSITORY_ROOT).as_posix()) + + schema_text = REPORT_SCHEMA.read_text(encoding="utf-8") + schema = json.loads(schema_text) + schema_names = _schema_property_names(schema) + assert leaks == [] + assert not any( + _PRIVACY_BEARING_SCHEMA_WORDS.intersection(_schema_name_words(name)) + for name in schema_names + ) + assert all( + pattern.search(schema_text) is None + for pattern in _PERSONAL_ROOT_PATTERNS + ) + + +def _count_only_report() -> dict[str, object]: + return { + "aggregateCompilationDigest": "a" * 64, + "compilerVersion": "context-engine-markdown-v3", + "configVersion": "markdown-config-v3", + "constructHistogram": { + "atxHeadings": 0, + "callouts": 0, + "embeds": 0, + "fencedCode": 0, + "footnotes": 0, + "frontmatter": 0, + "htmlBlocks": 0, + "inlineMath": 0, + "lists": 0, + "setextHeadings": 0, + "tables": 0, + "wikilinks": 0, + }, + "documents": { + "accepted": 1, + "acceptanceRate": "1.000000", + "refused": 0, + "total": 1, + }, + "maxFragmentTokenCount": 1, + "refusalHistogram": {}, + "schemaVersion": "compiler-runner-acceptance-v1", + "tokenCeiling": 2048, + } + + +def _private_path_shapes() -> tuple[str, ...]: + slash = chr(47) + backslash = chr(92) + return ( + slash.join(("", "Volumes", "private", "entry.md")), + slash.join(("", "var", "private", "entry.md")), + slash.join(("~", "private", "entry.md")), + "Z" + chr(58) + backslash + backslash.join(("private", "entry.md")), + backslash * 2 + backslash.join(("server", "share", "entry.md")), + "private-entry.md", + ) + + +@pytest.mark.parametrize("path_shaped_key", _private_path_shapes()) +def test_acceptance_schema_rejects_paths_and_filenames_in_histogram_keys( + path_shaped_key: str, +) -> None: + schema = json.loads(REPORT_SCHEMA.read_text(encoding="utf-8")) + report = deepcopy(_count_only_report()) + refusal_histogram = report["refusalHistogram"] + assert type(refusal_histogram) is dict + refusal_histogram[path_shaped_key] = 1 + + with pytest.raises(BenchmarkUnavailable, match="report schema"): + validate_json_schema_document(report, schema) + def test_acceptance_report_is_count_only_deterministic_and_written_under_ignore( tmp_path: Path, @@ -32,11 +191,17 @@ def test_acceptance_report_is_count_only_deterministic_and_written_under_ignore( str(output), ] - first = subprocess.run(command, check=True, capture_output=True, text=True) + first = subprocess.run( + command, check=True, capture_output=True, text=True, timeout=30 + ) first_bytes = output.read_bytes() - second = subprocess.run(command, check=True, capture_output=True, text=True) + second = subprocess.run( + command, check=True, capture_output=True, text=True, timeout=30 + ) report = json.loads(first_bytes) + schema = json.loads(REPORT_SCHEMA.read_text(encoding="utf-8")) + validate_json_schema_document(report, schema) assert report["schemaVersion"] == "compiler-runner-acceptance-v1" assert report["documents"] == { "accepted": 2, diff --git a/tests/unit/test_compiler_runner_determinism.py b/tests/unit/test_compiler_runner_determinism.py index 11e92a99..e9e50a15 100644 --- a/tests/unit/test_compiler_runner_determinism.py +++ b/tests/unit/test_compiler_runner_determinism.py @@ -8,6 +8,7 @@ import pytest +import applications.compiler_runner as compiler_runner from adapters.parsers.ragflow_markdown import compile_rich_markdown from engine.supply import ( MarkdownCompilerConfig, @@ -55,12 +56,14 @@ def test_rich_compilation_digest_is_stable_in_process_and_across_runner( input=source, capture_output=True, check=True, + timeout=30, ) second_process = subprocess.run( command, input=source, capture_output=True, check=True, + timeout=30, ) assert first_process.stdout == second_process.stdout envelope = json.loads(first_process.stdout) @@ -94,3 +97,20 @@ def test_representation_digest_distinguishes_exact_trailing_newline_bytes() -> N variants ) assert len({document.compilation_digest for document in documents}) == len(variants) + + +def test_local_runner_wrapper_cannot_be_substituted_with_a_direct_call( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def direct_call_must_not_run(*args: object, **kwargs: object) -> object: + raise AssertionError("wrapper bypassed its subprocess boundary") + + monkeypatch.setattr( + compiler_runner, + "compile_rich_markdown", + direct_call_must_not_run, + ) + + outcome = compiler_runner.compile_in_local_compiler_runner(b"# T\n", CONFIG) + + assert type(outcome) is ParsedDocument diff --git a/tests/unit/test_compiler_runner_production_boundary.py b/tests/unit/test_compiler_runner_production_boundary.py index 1c810d09..ad191bcd 100644 --- a/tests/unit/test_compiler_runner_production_boundary.py +++ b/tests/unit/test_compiler_runner_production_boundary.py @@ -7,6 +7,43 @@ REPOSITORY_ROOT = Path(__file__).parents[2] PRODUCTION_ROOTS = ("engine", "adapters", "applications") +_LOCAL_RUNNER = REPOSITORY_ROOT / "applications/compiler_runner.py" + + +def _qualified_production_calls() -> tuple[tuple[str, str], ...]: + calls: list[tuple[str, str]] = [] + for root_name in PRODUCTION_ROOTS: + for path in (REPOSITORY_ROOT / root_name).rglob("*.py"): + if path == _LOCAL_RUNNER: + continue + tree = ast.parse(path.read_bytes(), filename=str(path)) + imported_modules: dict[str, str] = {} + imported_symbols: dict[str, str] = {} + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + imported_modules[alias.asname or alias.name] = alias.name + elif isinstance(node, ast.ImportFrom) and node.module is not None: + for alias in node.names: + imported_symbols[alias.asname or alias.name] = ( + f"{node.module}.{alias.name}" + ) + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + if isinstance(node.func, ast.Name): + qualified = imported_symbols.get(node.func.id) + elif ( + isinstance(node.func, ast.Attribute) + and isinstance(node.func.value, ast.Name) + ): + module = imported_modules.get(node.func.value.id) + qualified = f"{module}.{node.func.attr}" if module else None + else: + qualified = None + if qualified is not None: + calls.append((str(path.relative_to(REPOSITORY_ROOT)), qualified)) + return tuple(calls) def test_unleased_subprocess_helper_is_explicitly_local_only() -> None: @@ -40,3 +77,16 @@ def test_no_production_module_calls_the_local_compiler_runner() -> None: if imported_from_runner or imported_runner: callers.append(str(path.relative_to(REPOSITORY_ROOT))) assert callers == [] + + +def test_no_production_module_calls_the_unleased_raw_compiler_entry_point() -> None: + forbidden_calls = { + "adapters.parsers.ragflow_markdown.compile_rich_markdown", + "applications.compiler_runner.compile_in_local_compiler_runner", + } + + assert tuple( + (path, qualified) + for path, qualified in _qualified_production_calls() + if qualified in forbidden_calls + ) == () diff --git a/tests/unit/test_compiler_runner_refusal.py b/tests/unit/test_compiler_runner_refusal.py index 136fc792..ae3f8506 100644 --- a/tests/unit/test_compiler_runner_refusal.py +++ b/tests/unit/test_compiler_runner_refusal.py @@ -2,6 +2,7 @@ import json import subprocess +from pathlib import Path import pytest @@ -27,15 +28,23 @@ b"# Heading\n\n```python\nprint('open')\n", CompilationFailureCode.UNSUPPORTED_CONSTRUCT, ), - ( - b"---\nkey: value\n# Missing close\n", - CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE, - ), (b"# Heading\n\n\xed\xa0\x80\n", CompilationFailureCode.INVALID_UTF8), ( b"# Heading\n\ncontains\x00nul\n", CompilationFailureCode.UNSUPPORTED_CONSTRUCT, ), + ( + b"```text\ncontains\x07bell\n```\n", + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + ), + ( + b"```text\ncontains\x1bescape\n```\n", + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + ), + ( + b"```text\ncontains\x1fseparator\n```\n", + CompilationFailureCode.UNSUPPORTED_CONSTRUCT, + ), (b"# Heading\n\ntruncated \xe4\xb8", CompilationFailureCode.INVALID_UTF8), ( b"# Heading\n\n\n", @@ -133,6 +142,49 @@ def test_runner_api_converts_child_process_failure_to_typed_failure( assert outcome.position is None +def test_runner_api_passes_a_bound_and_converts_timeout_to_typed_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + observed: list[float] = [] + + def wedge(*args: object, timeout: float, **kwargs: object) -> object: + observed.append(timeout) + raise subprocess.TimeoutExpired(cmd="wedged compiler", timeout=timeout) + + monkeypatch.setattr(subprocess, "run", wedge) + + outcome = compile_in_local_compiler_runner(b"# T\n", CONFIG) + + assert observed and observed[0] > 0 + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE + assert outcome.position is None + + +def test_runner_api_terminates_a_deliberately_wedged_child( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + executable = tmp_path / "wedged-compiler" + executable.write_text( + "#!/usr/bin/env python3\nimport time\ntime.sleep(60)\n", + encoding="utf-8", + ) + executable.chmod(0o700) + monkeypatch.setattr( + compiler_runner, + "sys", + type("Sys", (), {"executable": str(executable)}), + ) + monkeypatch.setattr(compiler_runner, "COMPILER_RUNNER_TIMEOUT_SECONDS", 0.05) + + outcome = compile_in_local_compiler_runner(b"# T\n", CONFIG) + + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE + assert outcome.position is None + + def test_direct_compiler_converts_domain_constructor_rejection_to_typed_failure( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/test_embedding_benchmark_report_privacy.py b/tests/unit/test_embedding_benchmark_report_privacy.py index 6def5f95..0d5290db 100644 --- a/tests/unit/test_embedding_benchmark_report_privacy.py +++ b/tests/unit/test_embedding_benchmark_report_privacy.py @@ -6,7 +6,9 @@ REPORT_PATH = Path("docs/evaluation/2026-07-29-embedding-benchmark.json") FORBIDDEN_KEYS = frozenset({"excerpt", "path", "query", "text", "title"}) -PERSONAL_PATH = re.compile(r"(?:/Users/|[A-Za-z]:\\|\.md(?:\b|$))") +PERSONAL_PATH = re.compile( + r"(?:/" + "Users" + r"/|[A-Za-z]:\\|\.md(?:\b|$))" +) def _keys(value: object) -> set[str]: From c121661847eb0836747a545fe7450549052726c6 Mon Sep 17 00:00:00 2001 From: stone Date: Thu, 30 Jul 2026 05:03:50 +0800 Subject: [PATCH 09/16] docs: fold runner closure into ADR-0079 --- ...ompile-rich-markdown-in-an-owned-runner.md | 37 ++++++++++- ...082-refine-rich-markdown-runner-closure.md | 63 ------------------- docs/decisions/README.md | 4 +- 3 files changed, 37 insertions(+), 67 deletions(-) delete mode 100644 docs/decisions/0082-refine-rich-markdown-runner-closure.md diff --git a/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md b/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md index c09f743b..5c17ffb2 100644 --- a/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md +++ b/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md @@ -21,6 +21,14 @@ replacement to retain version-explicit compilation, exact source provenance, all-or-nothing failure, atomic immutable publication, budget-visible context, and the sealed Runtime authorization order. +V3 accepts both nonempty `---`-delimited frontmatter and thematic breaks, so it +must define how an initial `---` without a valid closing delimiter is +classified. Treating every leading `---` as frontmatter would make the accepted +thematic-break grammar depend on document position. Independent construction +testing also showed that v3 profile labels alone do not bind the exact +compiler/configuration identity, and a runner subprocess without a deadline +could fail to produce either an accepted document or a typed refusal. + ADR-0074 permits a pinned, registered copy of RAGFlow's Apache-2.0 Markdown parser region after path-level license and nested-dependency verification. ADR-0075 permits Supply work in a ContextEngine-owned runner subprocess only @@ -45,6 +53,10 @@ index and executes under the exact parent WorkerLease binding. ragged or empty rows as one exact atomic source block rather than silently truncating the document; typed cell metadata remains best-effort within that exact source block. + A leading `---` opens frontmatter only when a later `---` closes a nonempty + payload. Without that closing delimiter, the leading line is an accepted + thematic break. This delimiter-complete rule preserves both constructs in + the closed grammar and does not infer metadata from ordinary following prose. A bare unmatched fence-marker line plus following nonblank text is retained as one exact literal paragraph block; an empty or language-bearing unterminated fence remains a typed refusal. @@ -73,6 +85,13 @@ index and executes under the exact parent WorkerLease binding. section metadata, coordinates, paths, parent headings, stable Fragment references, search phrases, contextual text, and the provenance-bound ceiling; parser-supplied metadata cannot bypass those checks. + Rich provenance binds the exact `context-engine-markdown-v3` compiler and + `markdown-config-v3` configuration identifiers as well as the v3 profiles. + The self-validating domain constructor rejects older or arbitrary identities. + The constructor and parser share only the closed control-character + classifier: C0 controls other than tab and line endings, DEL, and C1 controls + are refused. The constructor invokes it independently over exact source, so + fenced-code metadata cannot bypass a parser-ingress check. 5. **Hard splitting gate.** V3 uses a representation-bound 2,048-token default ceiling, recorded in both configuration and compilation provenance. The deterministic counter treats each non-whitespace run as one token. A block @@ -111,6 +130,20 @@ index and executes under the exact parent WorkerLease binding. ## Consequences +- Delimiter completeness makes the two already accepted `---` meanings + deterministic without changing v1 or v2. A document beginning with an + unclosed `---` compiles that line as a thematic break; complete nonempty + frontmatter retains its exact raw span. +- Exact identity binding keeps stored provenance interpretable. Forged v3 + documents cannot substitute an older or arbitrary compiler/config identity + while retaining rich profiles. Sharing the finite character predicate + prevents drift without trusting any parser-provided structure, span, or + construct metadata. +- The local compiler-runner call has a fixed positive timeout. Child timeout or + launch failure produces the same content-free typed boundary failure as any + other subprocess-boundary failure; an unbounded wait is not an outcome. A + positive deadline completes the all-or-nothing process contract even when a + child wedges before emitting bytes. - Rich-note and hard-bound behavior can be verified without creating durable content or adding another persistence truth. - Deterministic source spans and heading context remain part of the same @@ -126,4 +159,6 @@ index and executes under the exact parent WorkerLease binding. Revisit before accepting a construct outside the closed v3 grammar, changing the token counter or ceiling, splitting an indivisible construct differently, activating v3 publication, migrating an existing Revision, adding runner-local -state, or changing the same-Fragment ancestry rule. +state, changing the same-Fragment ancestry rule, changing leading delimiter +disambiguation or v3 identity binding, or making the fixed subprocess timeout a +caller-controlled runtime parameter. diff --git a/docs/decisions/0082-refine-rich-markdown-runner-closure.md b/docs/decisions/0082-refine-rich-markdown-runner-closure.md deleted file mode 100644 index 55800760..00000000 --- a/docs/decisions/0082-refine-rich-markdown-runner-closure.md +++ /dev/null @@ -1,63 +0,0 @@ ---- -name: adr-0082-refine-rich-markdown-runner-closure -version: "1.0.0" -description: > - Refine rich-Markdown frontmatter disambiguation, provenance identity, and - bounded runner termination without reopening the v3 representation. ---- - -# 0082. Refine rich Markdown runner closure - -- Status: accepted -- Date: 2026-07-30 -- Refines: ADR-0079 - -## Context - -ADR-0079 accepts both nonempty `---`-delimited frontmatter and thematic breaks, -but it did not define how an initial `---` without a valid closing delimiter is -classified. Treating every leading `---` as frontmatter made the accepted -thematic-break grammar depend on document position. Independent construction -testing also showed that v3 profile labels alone did not bind the exact -compiler/configuration identity, and a runner subprocess without a deadline -could fail to produce either an accepted document or a typed refusal. - -## Decision - -1. A leading `---` opens frontmatter only when a later `---` closes a nonempty - payload. Without that closing delimiter, the leading line is an accepted - thematic break. This delimiter-complete rule preserves both constructs in - the closed grammar and does not infer metadata from ordinary following prose. -2. Rich provenance binds the exact `context-engine-markdown-v3` compiler and - `markdown-config-v3` configuration identifiers as well as the v3 profiles. - The self-validating domain constructor rejects older or arbitrary identities. -3. The local compiler-runner call has a fixed positive timeout. Child timeout - or launch failure produces the same content-free typed boundary failure as - any other subprocess-boundary failure; an unbounded wait is not an outcome. -4. The constructor and parser share only the closed control-character - classifier: C0 controls other than tab and line endings, DEL, and C1 controls - are refused. The constructor invokes it independently over exact source, so - fenced-code metadata cannot bypass a parser-ingress check. - -## Rationale - -Delimiter completeness makes the two already accepted `---` meanings -deterministic without changing v1 or v2. Exact identity binding keeps stored -provenance interpretable, while a positive deadline completes ADR-0079's -all-or-nothing process contract even when a child wedges before emitting bytes. -Sharing the finite character predicate prevents drift without trusting any -parser-provided structure, span, or construct metadata. - -## Consequences - -- A document beginning with an unclosed `---` compiles that line as a thematic - break; complete nonempty frontmatter retains its exact raw span. -- Forged v3 documents cannot substitute an older or arbitrary compiler/config - identity while retaining rich profiles. -- Local compiler-runner callers receive a typed content-free failure after the - bounded deadline instead of waiting indefinitely. - -## Revisit trigger - -Revisit before changing leading delimiter disambiguation, v3 identity binding, -or the fixed subprocess timeout into a caller-controlled runtime parameter. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 0a967735..4a1fb07c 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -42,8 +42,7 @@ kernel, capability separation, and publication visibility model. | First File source registration | [0035 — Trusted File source registration](0035-register-file-sources-through-context-control.md) | One operation-bound trusted Control call atomically creates an Organization-owned source plus immutable active first version; all acquisition carriers remain unavailable | Caller-authored Organization/mode, host paths, registration-time File I/O, future capability claims, or cross-tenant idempotency | | First Markdown compiler | [0036 — Deterministic narrow Markdown compilation](0036-compile-narrow-markdown-deterministically.md) | Exact bytes compile purely into one canonical heading-plus-paragraph ParsedDocument with normalized coordinates and versioned content/compilation identities | Path-coupled parsing, silent unsupported syntax, partial documents, unversioned derived identities, or parser-side I/O | | Structural Markdown compiler | [0038 — Structural Markdown units](0038-compile-and-publish-structural-markdown.md) | Explicit v2 compilation publishes one coherent Fragment per heading, paragraph, list, fenced code block, or table with exact provenance and same-Fragment heading ancestry | Silent v1 reinterpretation, item/cell fragmentation, post-authorization parent expansion, or unbudgeted context | -| Rich Markdown compiler-runner | [0079 — Rich Markdown in an owned runner](0079-compile-rich-markdown-in-an-owned-runner.md) | Explicit v3 pure compilation accepts the closed rich grammar with exact source spans, same-Fragment ancestry, and a hard token ceiling while production activation stays deferred | Silent v1/v2 reinterpretation, runner-local state, unbounded Fragments, or activating publication without a separate decision | -| Rich Markdown runner closure | [0082 — Refine rich Markdown runner closure](0082-refine-rich-markdown-runner-closure.md) | Leading `---` is frontmatter only with a nonempty closing delimiter; exact v3 identities and a fixed subprocess timeout complete the typed runner contract | Position-dependent thematic-break refusal, relabeled v3 provenance, or an unbounded child wait | +| Rich Markdown compiler-runner | [0079 — Rich Markdown in an owned runner](0079-compile-rich-markdown-in-an-owned-runner.md) | Explicit v3 pure compilation accepts the delimiter-complete closed rich grammar with exact source spans, exact provenance identity, same-Fragment ancestry, a hard token ceiling, and bounded typed runner termination while production activation stays deferred | Silent v1/v2 reinterpretation, position-dependent thematic-break refusal, relabeled v3 provenance, runner-local state, unbounded waits or Fragments, or activating publication without a separate decision | | Unchanged File acquisition | [0039 — File acquisition no-op](0039-deduplicate-unchanged-file-acquisitions.md) | Tenant/source/resource-scoped canonical identity and one PostgreSQL guard lock classify a complete active artifact before publication; each observation retains an immutable digest-only outcome | Cross-tenant/global deduplication, process-local locking, partial-artifact reuse, silent version reuse, or no-op by bypassing publication validation | | File Resource deletion | [0042 — Tombstone before cleanup](0042-tombstone-file-resources-before-cleanup.md) | One trusted Control transaction tombstones the active File Resource, advances its Organization Policy Epoch, and records immutable pending cleanup lineage before any physical deletion | Cleanup-defined visibility, caller-authored tenant/epoch/cleanup identity, index deletion as authorization, restore, or native watcher claims | | File Source progress | [0043 — Separate acquisition and publication progress](0043-separate-file-acquisition-progress-from-publication-progress.md), [0072 — Report File source status with closed refusals](0072-report-file-source-status-with-closed-refusals.md) | Append accepted changes separately from contiguous Runtime-visibility completion and expose content-free operational status through an Organization/Source-scoped Control read | One ambiguous checkpoint, skipped publication gaps, retained source/compiler diagnostics, Runtime authorization from status or watermarks, or false standard ProviderPort capability claims | @@ -187,4 +186,3 @@ touched: - [0079 — Compile rich Markdown in an owned runner](0079-compile-rich-markdown-in-an-owned-runner.md) - [0080 — Refuse authoritative evaluation without an executor](0080-refuse-authoritative-evaluation-without-an-executor.md) - [0081 — Seal candidate discovery behind a data-only session](0081-seal-candidate-discovery-behind-a-data-only-session.md) -- [0082 — Refine rich Markdown runner closure](0082-refine-rich-markdown-runner-closure.md) From 5cde236e15d3a1fd3bc3ec85955cb7693a6e17ad Mon Sep 17 00:00:00 2001 From: stone Date: Thu, 30 Jul 2026 06:51:52 +0800 Subject: [PATCH 10/16] fix: close compiler runner residual gates (#124) --- adapters/parsers/ragflow_markdown.py | 38 ++- ...ompile-rich-markdown-in-an-owned-runner.md | 15 +- engine/supply/markdown.py | 26 +- scripts/run_m0_security_gate.py | 12 +- scripts/security_gate/runner.py | 9 +- .../test_zzz_security_gate_cli_privacy.py | 55 ++++ tests/unit/test_compiler_runner_acceptance.py | 36 +++ ...est_compiler_runner_production_boundary.py | 238 +++++++++++++++++- tests/unit/test_security_gate_cli_privacy.py | 69 +++++ 9 files changed, 446 insertions(+), 52 deletions(-) create mode 100644 tests/integration/test_zzz_security_gate_cli_privacy.py create mode 100644 tests/unit/test_security_gate_cli_privacy.py diff --git a/adapters/parsers/ragflow_markdown.py b/adapters/parsers/ragflow_markdown.py index 4c391b32..3e0418a0 100644 --- a/adapters/parsers/ragflow_markdown.py +++ b/adapters/parsers/ragflow_markdown.py @@ -171,9 +171,7 @@ def _frontmatter_end(lines: list[str]) -> int | None: return None for index in range(1, len(lines)): if lines[index] == "---": - if index == 1 or not any(line.strip() for line in lines[1:index]): - return -1 - return index + return index if any(line.strip() for line in lines[1:index]) else None return None @@ -186,12 +184,6 @@ def _blocks(text: str) -> tuple[_Block, ...] | CompilationFailure: blocks: list[_Block] = [] index = 0 frontmatter_end = _frontmatter_end(lines) - if frontmatter_end == -1: - return _failure( - CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE, - text, - 0, - ) if frontmatter_end is not None: blocks.append( _Block( @@ -287,6 +279,20 @@ def _blocks(text: str) -> tuple[_Block, ...] | CompilationFailure: index = closing + 1 continue + if re.fullmatch( + r" {0,3}(?:(?:\*[ \t]*){3,}|(?:_[ \t]*){3,}|(?:-[ \t]*){3,})", + line, + ): + blocks.append( + _Block( + kind=SectionKind.PARAGRAPH, + start=starts[index], + end=_line_end(lines, starts, index), + ) + ) + index += 1 + continue + if index + 1 < len(lines) and _SETEXT.fullmatch(lines[index + 1]): construct = unsupported_rich_markdown_inline(line.strip()) if construct is not None: @@ -494,20 +500,6 @@ def _blocks(text: str) -> tuple[_Block, ...] | CompilationFailure: UnsupportedConstruct.NESTED_HEADING, ) - if re.fullmatch( - r" {0,3}(?:(?:\*[ \t]*){3,}|(?:_[ \t]*){3,}|(?:-[ \t]*){3,})", - line, - ): - blocks.append( - _Block( - kind=SectionKind.PARAGRAPH, - start=starts[index], - end=_line_end(lines, starts, index), - ) - ) - index += 1 - continue - end = index + 1 while end < len(lines) and lines[end].strip(): if ( diff --git a/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md b/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md index 5c17ffb2..e8043091 100644 --- a/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md +++ b/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md @@ -53,10 +53,17 @@ index and executes under the exact parent WorkerLease binding. ragged or empty rows as one exact atomic source block rather than silently truncating the document; typed cell metadata remains best-effort within that exact source block. - A leading `---` opens frontmatter only when a later `---` closes a nonempty - payload. Without that closing delimiter, the leading line is an accepted - thematic break. This delimiter-complete rule preserves both constructs in - the closed grammar and does not infer metadata from ordinary following prose. + A leading `---` opens frontmatter only when the first later `---` closes a + nonempty payload. The complete delimiter matrix is closed as follows: a bare + `---` is one thematic break; adjacent delimiters are two thematic breaks; + delimiters separated only by blank lines remain thematic breaks; a nonempty + key-value payload with a closing delimiter is frontmatter; the same payload + without a closing delimiter is ordinary content after the leading thematic + break; and adjacent delimiters followed by text remain two thematic breaks + followed by ordinary content. CRLF and LF forms have identical grammar, and + an optional leading BOM changes none of these classifications. This + delimiter-complete rule preserves both constructs and does not infer + metadata from empty payloads or ordinary following prose. A bare unmatched fence-marker line plus following nonblank text is retained as one exact literal paragraph block; an empty or language-bearing unterminated fence remains a typed refusal. diff --git a/engine/supply/markdown.py b/engine/supply/markdown.py index f26680ac..7274a5f6 100644 --- a/engine/supply/markdown.py +++ b/engine/supply/markdown.py @@ -1002,11 +1002,9 @@ def _rich_source_blocks(source: str) -> tuple[_RichSourceBlock, ...]: ), None, ) - if closing == 1 or closing is not None and not any( + if closing is not None and any( line.strip() for line in lines[1:closing] ): - raise ValueError("rich frontmatter source must be complete and nonempty") - if closing is not None: blocks.append( _RichSourceBlock( kind=SectionKind.PARAGRAPH, @@ -1068,6 +1066,16 @@ def _rich_source_blocks(source: str) -> tuple[_RichSourceBlock, ...]: ) index = closing + 1 continue + if _THEMATIC_BREAK_PATTERN.fullmatch(line) is not None: + blocks.append( + _RichSourceBlock( + kind=SectionKind.PARAGRAPH, + start=starts[index], + end=_rich_source_line_end(lines, starts, index), + ) + ) + index += 1 + continue if index + 1 < len(lines) and _RICH_SETEXT_PATTERN.fullmatch( lines[index + 1] ): @@ -1162,16 +1170,6 @@ def _rich_source_blocks(source: str) -> tuple[_RichSourceBlock, ...]: ) index = end continue - if _THEMATIC_BREAK_PATTERN.fullmatch(line) is not None: - blocks.append( - _RichSourceBlock( - kind=SectionKind.PARAGRAPH, - start=starts[index], - end=_rich_source_line_end(lines, starts, index), - ) - ) - index += 1 - continue end = index + 1 while end < len(lines) and lines[end].strip(): if ( @@ -1248,6 +1246,8 @@ def _expected_rich_section_kind(source: str) -> SectionKind: lines = source.splitlines() if len(lines) >= 2 and lines[0] == "---" and lines[-1] == "---": return SectionKind.PARAGRAPH + if len(lines) == 1 and _THEMATIC_BREAK_PATTERN.fullmatch(lines[0]) is not None: + return SectionKind.PARAGRAPH if ( _RICH_ATX_HEADING_PATTERN.fullmatch(source) is not None or len(lines) == 2 diff --git a/scripts/run_m0_security_gate.py b/scripts/run_m0_security_gate.py index 06fddbbd..43bc10c5 100644 --- a/scripts/run_m0_security_gate.py +++ b/scripts/run_m0_security_gate.py @@ -4,8 +4,11 @@ from __future__ import annotations import argparse +import io +import logging import sys from collections.abc import Sequence +from contextlib import redirect_stderr, redirect_stdout from pathlib import Path from scripts.security_gate.runner import GatePaths, run_gate @@ -25,15 +28,20 @@ def build_parser() -> argparse.ArgumentParser: def main(argv: Sequence[str] | None = None) -> int: arguments = build_parser().parse_args(argv) paths = GatePaths.defaults(arguments.output_dir.resolve()) + prior_logging_disable = logging.root.manager.disable try: - report = run_gate(paths) + logging.disable(logging.CRITICAL) + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + report = run_gate(paths) except Exception as error: print(f"M0 security gate failed: {type(error).__name__}", file=sys.stderr) return 1 + finally: + logging.disable(prior_logging_disable) if report.get("m0SecurityDecision") != "pass": print("M0 SECURITY FAIL", file=sys.stderr) return 1 - print(f"M0 SECURITY PASS ({paths.output_directory})") + print("M0 SECURITY PASS") return 0 diff --git a/scripts/security_gate/runner.py b/scripts/security_gate/runner.py index eb1a43e7..b52b2fda 100644 --- a/scripts/security_gate/runner.py +++ b/scripts/security_gate/runner.py @@ -277,7 +277,14 @@ def _report_execution_command( def _execute_pytest( command: Sequence[str], *, cwd: Path, env: Mapping[str, str] ) -> int: - return subprocess.run(command, cwd=cwd, env=env, check=False).returncode + return subprocess.run( + command, + cwd=cwd, + env=env, + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ).returncode def _audit_rls( diff --git a/tests/integration/test_zzz_security_gate_cli_privacy.py b/tests/integration/test_zzz_security_gate_cli_privacy.py new file mode 100644 index 00000000..d00a88fb --- /dev/null +++ b/tests/integration/test_zzz_security_gate_cli_privacy.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import re +import subprocess +import sys +from hashlib import sha256 +from pathlib import Path + +import pytest + +REPOSITORY_ROOT = Path(__file__).parents[2] +_ABSOLUTE_PATHS = ( + re.compile(r"(? set[str]: + words = tuple( + word.casefold() + for word in re.findall(r"[A-Z]+(?=[A-Z][a-z]|\b)|[A-Z]?[a-z]+", value) + ) + return { + sha256("".join(words[index:end]).encode("utf-8")).hexdigest() + for index in range(len(words)) + for end in range(index + 1, min(index + 4, len(words)) + 1) + } + + +@pytest.mark.integration +def test_real_security_gate_cli_output_is_private_and_portable( + tmp_path: Path, +) -> None: + completed = subprocess.run( + ( + sys.executable, + "scripts/run_m0_security_gate.py", + "--output-dir", + str(tmp_path / "gate-evidence"), + ), + cwd=REPOSITORY_ROOT, + check=False, + capture_output=True, + text=True, + timeout=300, + ) + output = completed.stdout + completed.stderr + + assert completed.returncode == 0 + assert output == "M0 SECURITY PASS\n" + assert all(pattern.search(output) is None for pattern in _ABSOLUTE_PATHS) + assert not (_PRIVATE_ROOT_FRAGMENT_DIGESTS & _word_sequence_digests(output)) diff --git a/tests/unit/test_compiler_runner_acceptance.py b/tests/unit/test_compiler_runner_acceptance.py index eed9c959..e32b5b81 100644 --- a/tests/unit/test_compiler_runner_acceptance.py +++ b/tests/unit/test_compiler_runner_acceptance.py @@ -195,6 +195,42 @@ def test_leading_unclosed_dash_rule_is_an_accepted_thematic_break( assert outcome.fragments[0].source_text == "---" +@pytest.mark.parametrize("newline", (b"\n", b"\r\n")) +@pytest.mark.parametrize("bom", (b"", b"\xef\xbb\xbf")) +@pytest.mark.parametrize( + ("lines", "expected_fragments"), + ( + ((b"---",), ("---",)), + ((b"---", b"---"), ("---", "---")), + ((b"---", b"", b"---"), ("---", "---")), + ((b"---", b"key: value", b"---"), ("frontmatter",)), + ((b"---", b"key: value"), ("---", "key: value")), + ((b"---", b"---", b"text"), ("---", "---", "text")), + ), +) +def test_leading_dash_delimiter_matrix_is_closed_and_exact( + newline: bytes, + bom: bytes, + lines: tuple[bytes, ...], + expected_fragments: tuple[str, ...], +) -> None: + source = bom + newline.join(lines) + newline + + outcome = compile_rich_markdown(source, CONFIG) + + assert type(outcome) is ParsedDocument + expected = tuple( + (newline.join(lines).decode("utf-8") if value == "frontmatter" else value) + for value in expected_fragments + ) + assert tuple(fragment.source_text for fragment in outcome.fragments) == expected + for fragment in outcome.fragments: + span = fragment.position + assert source[span.start.byte_offset : span.end.byte_offset] == ( + fragment.source_text.encode("utf-8") + ) + + def test_rich_constructor_rejects_forged_lineage_and_ancestry() -> None: compiled = compile_rich_markdown(b"# Root\n\nBody\n", CONFIG) assert type(compiled) is ParsedDocument diff --git a/tests/unit/test_compiler_runner_production_boundary.py b/tests/unit/test_compiler_runner_production_boundary.py index ad191bcd..7c39f97c 100644 --- a/tests/unit/test_compiler_runner_production_boundary.py +++ b/tests/unit/test_compiler_runner_production_boundary.py @@ -1,6 +1,7 @@ from __future__ import annotations import ast +from collections.abc import Iterable from pathlib import Path import applications.compiler_runner as compiler_runner @@ -8,6 +9,18 @@ REPOSITORY_ROOT = Path(__file__).parents[2] PRODUCTION_ROOTS = ("engine", "adapters", "applications") _LOCAL_RUNNER = REPOSITORY_ROOT / "applications/compiler_runner.py" +_FORBIDDEN_PRODUCTION_CALLS = frozenset( + { + "adapters.parsers.ragflow_markdown.compile_rich_markdown", + "applications.compiler_runner.compile_in_local_compiler_runner", + } +) +_IGNORED_ROOT_MODULES = frozenset( + { + "adapters.parsers.ragflow_markdown", + "applications.compiler_runner", + } +) def _qualified_production_calls() -> tuple[tuple[str, str], ...]: @@ -46,6 +59,172 @@ def _qualified_production_calls() -> tuple[tuple[str, str], ...]: return tuple(calls) +def _module_name(repository_root: Path, path: Path) -> str: + relative = path.relative_to(repository_root) + parts = relative.with_suffix("").parts + return ".".join(parts[:-1] if parts[-1] == "__init__" else parts) + + +def _function_definitions( + module: str, tree: ast.Module +) -> dict[str, ast.FunctionDef | ast.AsyncFunctionDef]: + definitions: dict[str, ast.FunctionDef | ast.AsyncFunctionDef] = {} + for node in tree.body: + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): + definitions[f"{module}.{node.name}"] = node + elif isinstance(node, ast.ClassDef): + for child in node.body: + if isinstance(child, ast.FunctionDef | ast.AsyncFunctionDef): + definitions[f"{module}.{node.name}.{child.name}"] = child + return definitions + + +def _module_imports(tree: ast.Module) -> dict[str, str]: + imports: dict[str, str] = {} + for node in tree.body: + if isinstance(node, ast.Import): + for alias in node.names: + imports[alias.asname or alias.name] = alias.name + elif isinstance(node, ast.ImportFrom) and node.module is not None: + for alias in node.names: + imports[alias.asname or alias.name] = f"{node.module}.{alias.name}" + return imports + + +def _scoped_nodes( + node: ast.FunctionDef | ast.AsyncFunctionDef, +) -> Iterable[ast.AST]: + pending: list[ast.AST] = list(node.body) + while pending: + current = pending.pop() + yield current + if isinstance( + current, + ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef | ast.Lambda, + ): + continue + pending.extend(ast.iter_child_nodes(current)) + + +def _resolve_expression( + expression: ast.expr, + *, + names: dict[str, str], + module: str, + module_functions: frozenset[str], +) -> str | None: + if isinstance(expression, ast.Name): + imported = names.get(expression.id) + local = f"{module}.{expression.id}" + return imported if imported is not None else ( + local if local in module_functions else None + ) + if isinstance(expression, ast.Attribute): + base = _resolve_expression( + expression.value, + names=names, + module=module, + module_functions=module_functions, + ) + return f"{base}.{expression.attr}" if base is not None else None + return None + + +def _function_calls( + node: ast.FunctionDef | ast.AsyncFunctionDef, + *, + imports: dict[str, str], + module: str, + module_functions: frozenset[str], +) -> frozenset[str]: + names = dict(imports) + scoped = tuple(_scoped_nodes(node)) + changed = True + while changed: + changed = False + for child in scoped: + if not isinstance(child, ast.Assign): + continue + resolved = _resolve_expression( + child.value, + names=names, + module=module, + module_functions=module_functions, + ) + if resolved is None: + continue + for target in child.targets: + if isinstance(target, ast.Name) and names.get(target.id) != resolved: + names[target.id] = resolved + changed = True + return frozenset( + resolved + for child in scoped + if isinstance(child, ast.Call) + for resolved in ( + _resolve_expression( + child.func, + names=names, + module=module, + module_functions=module_functions, + ), + ) + if resolved is not None + ) + + +def _production_paths_to_forbidden( + repository_root: Path, + *, + production_roots: tuple[str, ...], + ignored_root_modules: frozenset[str], + forbidden: frozenset[str], +) -> tuple[tuple[str, ...], ...]: + parsed: dict[str, ast.Module] = {} + definitions: dict[str, ast.FunctionDef | ast.AsyncFunctionDef] = {} + for root_name in production_roots: + root = repository_root / root_name + if not root.exists(): + continue + for path in root.rglob("*.py"): + module = _module_name(repository_root, path) + tree = ast.parse(path.read_bytes(), filename=str(path)) + parsed[module] = tree + definitions.update(_function_definitions(module, tree)) + function_names = frozenset(definitions) + graph: dict[str, frozenset[str]] = {} + for qualified, definition in definitions.items(): + module = qualified.rsplit(".", maxsplit=1)[0] + while module not in parsed: + module = module.rsplit(".", maxsplit=1)[0] + graph[qualified] = _function_calls( + definition, + imports=_module_imports(parsed[module]), + module=module, + module_functions=frozenset( + name for name in function_names if name.startswith(f"{module}.") + ), + ) + + paths: set[tuple[str, ...]] = set() + + def walk(current: str, path: tuple[str, ...]) -> None: + for called in graph.get(current, frozenset()): + next_path = (*path, called) + if called in forbidden: + paths.add(next_path) + elif called in graph and called not in path: + walk(called, next_path) + + for function in graph: + if not any( + function == module or function.startswith(f"{module}.") + for module in ignored_root_modules + ): + walk(function, (function,)) + return tuple(sorted(paths)) + + def test_unleased_subprocess_helper_is_explicitly_local_only() -> None: assert not hasattr(compiler_runner, "compile_in_compiler_runner") assert hasattr(compiler_runner, "compile_in_local_compiler_runner") @@ -80,13 +259,54 @@ def test_no_production_module_calls_the_local_compiler_runner() -> None: def test_no_production_module_calls_the_unleased_raw_compiler_entry_point() -> None: - forbidden_calls = { - "adapters.parsers.ragflow_markdown.compile_rich_markdown", - "applications.compiler_runner.compile_in_local_compiler_runner", - } - - assert tuple( - (path, qualified) - for path, qualified in _qualified_production_calls() - if qualified in forbidden_calls + assert _production_paths_to_forbidden( + REPOSITORY_ROOT, + production_roots=PRODUCTION_ROOTS, + ignored_root_modules=_IGNORED_ROOT_MODULES, + forbidden=_FORBIDDEN_PRODUCTION_CALLS, ) == () + + +def test_production_reachability_analysis_detects_transitive_indirect_calls( + tmp_path: Path, +) -> None: + (tmp_path / "applications").mkdir() + (tmp_path / "adapters/parsers").mkdir(parents=True) + (tmp_path / "applications/bridge.py").write_text( + """ +from adapters.parsers.ragflow_markdown import compile_rich_markdown as rich_compile + +def hidden(raw, config): + compiler = rich_compile + return compiler(raw, config) +""".lstrip(), + encoding="utf-8", + ) + (tmp_path / "applications/entry.py").write_text( + """ +from applications.bridge import hidden + +def production_entry(raw, config): + return hidden(raw, config) +""".lstrip(), + encoding="utf-8", + ) + (tmp_path / "adapters/parsers/ragflow_markdown.py").write_text( + "def compile_rich_markdown(raw, config):\n return raw, config\n", + encoding="utf-8", + ) + + paths = _production_paths_to_forbidden( + tmp_path, + production_roots=("engine", "adapters", "applications"), + ignored_root_modules=frozenset(), + forbidden=frozenset( + {"adapters.parsers.ragflow_markdown.compile_rich_markdown"} + ), + ) + + assert ( + "applications.entry.production_entry", + "applications.bridge.hidden", + "adapters.parsers.ragflow_markdown.compile_rich_markdown", + ) in paths diff --git a/tests/unit/test_security_gate_cli_privacy.py b/tests/unit/test_security_gate_cli_privacy.py new file mode 100644 index 00000000..cba41c73 --- /dev/null +++ b/tests/unit/test_security_gate_cli_privacy.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import logging +import os +import re +import sys +from pathlib import Path + +import pytest + +from scripts.run_m0_security_gate import main as security_gate_main +from scripts.security_gate.runner import _execute_pytest + +_ABSOLUTE_PATHS = ( + re.compile(r"(? None: + assert _SYNTHETIC_PRIVATE_ROOT not in output + assert all(pattern.search(output) is None for pattern in _ABSOLUTE_PATHS) + + +def test_gate_cli_scrubs_complete_output_across_all_channels( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + absolute_root = f"/private-machine/{_SYNTHETIC_PRIVATE_ROOT}" + + def noisy_gate(_paths: object) -> dict[str, str]: + print(f"stdout evidence at {absolute_root}/stdout.json") + print(f"stderr evidence at {absolute_root}/stderr.json", file=sys.stderr) + logger = logging.getLogger("synthetic-security-gate-output") + logger.addHandler(logging.StreamHandler(sys.stderr)) + logger.warning("log evidence at %s/log.json", absolute_root) + return { + "m0SecurityDecision": "pass", + "releaseDecision": "not-evaluated", + } + + monkeypatch.setattr("scripts.run_m0_security_gate.run_gate", noisy_gate) + + assert security_gate_main(["--output-dir", absolute_root]) == 0 + captured = capsys.readouterr() + _assert_private_safe_output(captured.out + captured.err) + + +def test_gate_child_process_output_is_scrubbed_before_forwarding( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + absolute_root = f"/private-machine/{_SYNTHETIC_PRIVATE_ROOT}" + program = ( + "import sys; " + f"print({str(absolute_root + '/stdout.json')!r}); " + f"print({str(absolute_root + '/stderr.json')!r}, file=sys.stderr)" + ) + + exit_code = _execute_pytest( + (sys.executable, "-c", program), + cwd=tmp_path, + env=os.environ, + ) + + assert exit_code == 0 + captured = capsys.readouterr() + _assert_private_safe_output(captured.out + captured.err) From a2a05069807353ab0e5edfac4b638ab3031bc483 Mon Sep 17 00:00:00 2001 From: stone Date: Thu, 30 Jul 2026 07:20:44 +0800 Subject: [PATCH 11/16] fix: close compiler runner final residuals (#124) --- adapters/parsers/ragflow_markdown.py | 19 ++- ...ompile-rich-markdown-in-an-owned-runner.md | 21 ++- engine/supply/markdown.py | 22 ++- .../test_zzz_security_gate_cli_privacy.py | 19 --- tests/unit/test_compiler_runner_acceptance.py | 9 +- .../test_compiler_runner_acceptance_report.py | 40 +++-- ...est_compiler_runner_production_boundary.py | 143 ++++++++++++++++-- 7 files changed, 212 insertions(+), 61 deletions(-) diff --git a/adapters/parsers/ragflow_markdown.py b/adapters/parsers/ragflow_markdown.py index 3e0418a0..15b358fc 100644 --- a/adapters/parsers/ragflow_markdown.py +++ b/adapters/parsers/ragflow_markdown.py @@ -41,6 +41,11 @@ re.IGNORECASE, ) _ANGLE_LITERAL: Final = re.compile(r"<[^<>\r\n]+>") +_FRONTMATTER_MAPPING: Final = re.compile( + r"^[ \t]*(?:[A-Za-z0-9_.-]+|\"[^\"\r\n]+\"|'[^'\r\n]+')" + r"[ \t]*:[ \t]*(?:.*)?$" +) +_FRONTMATTER_SEQUENCE: Final = re.compile(r"^[ \t]*-[ \t]+\S.*$") @dataclass(frozen=True, slots=True) @@ -171,7 +176,19 @@ def _frontmatter_end(lines: list[str]) -> int | None: return None for index in range(1, len(lines)): if lines[index] == "---": - return index if any(line.strip() for line in lines[1:index]) else None + first_payload_line = next( + (line for line in lines[1:index] if line.strip()), + None, + ) + return ( + index + if first_payload_line is not None + and ( + _FRONTMATTER_MAPPING.fullmatch(first_payload_line) is not None + or _FRONTMATTER_SEQUENCE.fullmatch(first_payload_line) is not None + ) + else None + ) return None diff --git a/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md b/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md index e8043091..319f0fc2 100644 --- a/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md +++ b/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md @@ -54,16 +54,21 @@ index and executes under the exact parent WorkerLease binding. truncating the document; typed cell metadata remains best-effort within that exact source block. A leading `---` opens frontmatter only when the first later `---` closes a - nonempty payload. The complete delimiter matrix is closed as follows: a bare + nonempty YAML-shaped mapping or sequence payload. The complete delimiter + matrix is closed as follows: a bare `---` is one thematic break; adjacent delimiters are two thematic breaks; delimiters separated only by blank lines remain thematic breaks; a nonempty - key-value payload with a closing delimiter is frontmatter; the same payload - without a closing delimiter is ordinary content after the leading thematic - break; and adjacent delimiters followed by text remain two thematic breaks - followed by ordinary content. CRLF and LF forms have identical grammar, and - an optional leading BOM changes none of these classifications. This - delimiter-complete rule preserves both constructs and does not infer - metadata from empty payloads or ordinary following prose. + YAML-shaped mapping or sequence payload with a closing delimiter is + frontmatter; the same payload without a closing delimiter is ordinary + content after the leading thematic break; ordinary prose between delimiters + is a leading thematic break followed by ordinary Markdown (so a closing + `---` may serve as its setext underline); and adjacent delimiters followed + by text remain two thematic breaks followed by ordinary content. CRLF and LF + forms have identical grammar, and an optional leading BOM changes none of + these classifications. The mapping/sequence check is lexical rather than + semantic YAML interpretation. This delimiter-complete rule preserves both + constructs and does not infer metadata from empty payloads or ordinary + following prose. A bare unmatched fence-marker line plus following nonblank text is retained as one exact literal paragraph block; an empty or language-bearing unterminated fence remains a typed refusal. diff --git a/engine/supply/markdown.py b/engine/supply/markdown.py index 7274a5f6..a1573534 100644 --- a/engine/supply/markdown.py +++ b/engine/supply/markdown.py @@ -1002,9 +1002,25 @@ def _rich_source_blocks(source: str) -> tuple[_RichSourceBlock, ...]: ), None, ) - if closing is not None and any( - line.strip() for line in lines[1:closing] - ): + first_payload_line = next( + (line for line in lines[1:closing] if line.strip()), + None, + ) if closing is not None else None + mapping_payload = ( + first_payload_line is not None + and re.fullmatch( + r"^[ \t]*(?:[A-Za-z0-9_.-]+|\"[^\"\r\n]+\"|'[^'\r\n]+')" + r"[ \t]*:[ \t]*(?:.*)?$", + first_payload_line, + ) + is not None + ) + sequence_payload = ( + first_payload_line is not None + and re.fullmatch(r"^[ \t]*-[ \t]+\S.*$", first_payload_line) + is not None + ) + if closing is not None and (mapping_payload or sequence_payload): blocks.append( _RichSourceBlock( kind=SectionKind.PARAGRAPH, diff --git a/tests/integration/test_zzz_security_gate_cli_privacy.py b/tests/integration/test_zzz_security_gate_cli_privacy.py index d00a88fb..7a0a970b 100644 --- a/tests/integration/test_zzz_security_gate_cli_privacy.py +++ b/tests/integration/test_zzz_security_gate_cli_privacy.py @@ -3,7 +3,6 @@ import re import subprocess import sys -from hashlib import sha256 from pathlib import Path import pytest @@ -13,23 +12,6 @@ re.compile(r"(? set[str]: - words = tuple( - word.casefold() - for word in re.findall(r"[A-Z]+(?=[A-Z][a-z]|\b)|[A-Z]?[a-z]+", value) - ) - return { - sha256("".join(words[index:end]).encode("utf-8")).hexdigest() - for index in range(len(words)) - for end in range(index + 1, min(index + 4, len(words)) + 1) - } - - @pytest.mark.integration def test_real_security_gate_cli_output_is_private_and_portable( tmp_path: Path, @@ -52,4 +34,3 @@ def test_real_security_gate_cli_output_is_private_and_portable( assert completed.returncode == 0 assert output == "M0 SECURITY PASS\n" assert all(pattern.search(output) is None for pattern in _ABSOLUTE_PATHS) - assert not (_PRIVATE_ROOT_FRAGMENT_DIGESTS & _word_sequence_digests(output)) diff --git a/tests/unit/test_compiler_runner_acceptance.py b/tests/unit/test_compiler_runner_acceptance.py index e32b5b81..3797efe6 100644 --- a/tests/unit/test_compiler_runner_acceptance.py +++ b/tests/unit/test_compiler_runner_acceptance.py @@ -204,6 +204,7 @@ def test_leading_unclosed_dash_rule_is_an_accepted_thematic_break( ((b"---", b"---"), ("---", "---")), ((b"---", b"", b"---"), ("---", "---")), ((b"---", b"key: value", b"---"), ("frontmatter",)), + ((b"---", b"ordinary prose", b"---"), ("---", "ordinary-prose")), ((b"---", b"key: value"), ("---", "key: value")), ((b"---", b"---", b"text"), ("---", "---", "text")), ), @@ -220,7 +221,13 @@ def test_leading_dash_delimiter_matrix_is_closed_and_exact( assert type(outcome) is ParsedDocument expected = tuple( - (newline.join(lines).decode("utf-8") if value == "frontmatter" else value) + ( + newline.join(lines).decode("utf-8") + if value == "frontmatter" + else newline.join(lines[1:]).decode("utf-8") + if value == "ordinary-prose" + else value + ) for value in expected_fragments ) assert tuple(fragment.source_text for fragment in outcome.fragments) == expected diff --git a/tests/unit/test_compiler_runner_acceptance_report.py b/tests/unit/test_compiler_runner_acceptance_report.py index ffb08e38..357e6b64 100644 --- a/tests/unit/test_compiler_runner_acceptance_report.py +++ b/tests/unit/test_compiler_runner_acceptance_report.py @@ -5,7 +5,6 @@ import subprocess import sys from copy import deepcopy -from hashlib import sha256 from pathlib import Path import pytest @@ -24,12 +23,14 @@ re.compile(r"/" + "home" + r"/[^/\s]+/"), re.compile(r"[A-Za-z]:\\(?:Users|Documents and Settings)\\", re.IGNORECASE), ) -_PRIVATE_ROOT_FRAGMENT_DIGESTS = frozenset( - {"c83ea566573fdfcacb79f350f86ea53935437f5672e1fe97703320cce4725394"} -) +_SYNTHETIC_PRIVATE_ROOT_FRAGMENT = "-".join(("synthetic", "corpus", "canary")) _PRIVACY_BEARING_SCHEMA_WORDS = frozenset( {"excerpt", "file", "filename", "path", "root", "source", "text", "title"} ) +_PRIVACY_GUARD_PATHS = ( + Path(__file__), + REPOSITORY_ROOT / "tests/integration/test_zzz_security_gate_cli_privacy.py", +) def _schema_property_names(value: object) -> set[str]: @@ -62,16 +63,25 @@ def _schema_name_words(name: str) -> set[str]: } -def _word_sequence_digests(value: str) -> set[str]: - words = tuple( - word.casefold() - for word in re.findall(r"[A-Z]+(?=[A-Z][a-z]|\b)|[A-Z]?[a-z]+", value) +def _contains_private_location(value: str) -> bool: + return any(pattern.search(value) for pattern in _PERSONAL_ROOT_PATTERNS) or ( + _SYNTHETIC_PRIVATE_ROOT_FRAGMENT in value ) - return { - sha256("".join(words[index:end]).encode("utf-8")).hexdigest() - for index in range(len(words)) - for end in range(index + 1, min(index + 4, len(words)) + 1) - } + + +def test_privacy_guards_do_not_embed_identifier_fingerprints() -> None: + fingerprint = re.compile(r"(? None: + assert _contains_private_location("/" + "Users" + "/person/corpus") + assert _contains_private_location(_SYNTHETIC_PRIVATE_ROOT_FRAGMENT) + assert not _contains_private_location("aggregate-counts-only") def test_tracked_tree_and_acceptance_schema_cannot_carry_private_paths() -> None: @@ -87,9 +97,7 @@ def test_tracked_tree_and_acceptance_schema_cannot_carry_private_paths() -> None continue path = REPOSITORY_ROOT / raw_path.decode("utf-8") content = path.read_text(encoding="utf-8", errors="ignore") - if any(pattern.search(content) for pattern in _PERSONAL_ROOT_PATTERNS) or ( - _PRIVATE_ROOT_FRAGMENT_DIGESTS & _word_sequence_digests(content) - ): + if _contains_private_location(content): leaks.append(path.relative_to(REPOSITORY_ROOT).as_posix()) schema_text = REPORT_SCHEMA.read_text(encoding="utf-8") diff --git a/tests/unit/test_compiler_runner_production_boundary.py b/tests/unit/test_compiler_runner_production_boundary.py index 7c39f97c..3978b47d 100644 --- a/tests/unit/test_compiler_runner_production_boundary.py +++ b/tests/unit/test_compiler_runner_production_boundary.py @@ -69,28 +69,55 @@ def _function_definitions( module: str, tree: ast.Module ) -> dict[str, ast.FunctionDef | ast.AsyncFunctionDef]: definitions: dict[str, ast.FunctionDef | ast.AsyncFunctionDef] = {} - for node in tree.body: - if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): - definitions[f"{module}.{node.name}"] = node - elif isinstance(node, ast.ClassDef): - for child in node.body: - if isinstance(child, ast.FunctionDef | ast.AsyncFunctionDef): - definitions[f"{module}.{node.name}.{child.name}"] = child + + def collect(nodes: list[ast.stmt], prefix: str) -> None: + for node in nodes: + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): + qualified = f"{prefix}.{node.name}" + definitions[qualified] = node + collect(node.body, qualified) + elif isinstance(node, ast.ClassDef): + collect(node.body, f"{prefix}.{node.name}") + + collect(tree.body, module) return definitions -def _module_imports(tree: ast.Module) -> dict[str, str]: +def _resolve_import_module(module: str, imported: str | None, level: int) -> str: + if level == 0: + return imported or "" + package = module.rsplit(".", maxsplit=1)[0] + parts = package.split(".") if package else [] + retained = parts[: max(0, len(parts) - level + 1)] + return ".".join((*retained, *((imported or "").split(".")))) + + +def _imports(nodes: Iterable[ast.AST], *, module: str) -> dict[str, str]: imports: dict[str, str] = {} - for node in tree.body: + for node in nodes: if isinstance(node, ast.Import): for alias in node.names: - imports[alias.asname or alias.name] = alias.name - elif isinstance(node, ast.ImportFrom) and node.module is not None: + bound = alias.asname or alias.name.split(".", maxsplit=1)[0] + imports[bound] = alias.name if alias.asname else bound + elif isinstance(node, ast.ImportFrom): + imported_module = _resolve_import_module( + module, + node.module, + node.level, + ) for alias in node.names: - imports[alias.asname or alias.name] = f"{node.module}.{alias.name}" + imports[alias.asname or alias.name] = ( + f"{imported_module}.{alias.name}" + if imported_module + else alias.name + ) return imports +def _module_imports(tree: ast.Module, *, module: str) -> dict[str, str]: + return _imports(tree.body, module=module) + + def _scoped_nodes( node: ast.FunctionDef | ast.AsyncFunctionDef, ) -> Iterable[ast.AST]: @@ -133,12 +160,21 @@ def _resolve_expression( def _function_calls( node: ast.FunctionDef | ast.AsyncFunctionDef, *, + qualified: str, imports: dict[str, str], module: str, module_functions: frozenset[str], ) -> frozenset[str]: names = dict(imports) scoped = tuple(_scoped_nodes(node)) + names.update(_imports(scoped, module=module)) + names.update( + { + child.name: f"{qualified}.{child.name}" + for child in node.body + if isinstance(child, ast.FunctionDef | ast.AsyncFunctionDef) + } + ) changed = True while changed: changed = False @@ -199,7 +235,8 @@ def _production_paths_to_forbidden( module = module.rsplit(".", maxsplit=1)[0] graph[qualified] = _function_calls( definition, - imports=_module_imports(parsed[module]), + qualified=qualified, + imports=_module_imports(parsed[module], module=module), module=module, module_functions=frozenset( name for name in function_names if name.startswith(f"{module}.") @@ -310,3 +347,83 @@ def production_entry(raw, config): "applications.bridge.hidden", "adapters.parsers.ragflow_markdown.compile_rich_markdown", ) in paths + + +def test_production_reachability_detects_scoped_and_relative_import_aliases( + tmp_path: Path, +) -> None: + (tmp_path / "applications").mkdir() + (tmp_path / "adapters/parsers").mkdir(parents=True) + (tmp_path / "applications/bridge.py").write_text( + """ +def hidden(raw, config): + from adapters.parsers.ragflow_markdown import compile_rich_markdown as rich + compiler = rich + return compiler(raw, config) +""".lstrip(), + encoding="utf-8", + ) + (tmp_path / "applications/entry.py").write_text( + """ +from .bridge import hidden as delegated + +def production_entry(raw, config): + return delegated(raw, config) +""".lstrip(), + encoding="utf-8", + ) + (tmp_path / "adapters/parsers/ragflow_markdown.py").write_text( + "def compile_rich_markdown(raw, config):\n return raw, config\n", + encoding="utf-8", + ) + + paths = _production_paths_to_forbidden( + tmp_path, + production_roots=("engine", "adapters", "applications"), + ignored_root_modules=frozenset(), + forbidden=frozenset( + {"adapters.parsers.ragflow_markdown.compile_rich_markdown"} + ), + ) + + assert ( + "applications.entry.production_entry", + "applications.bridge.hidden", + "adapters.parsers.ragflow_markdown.compile_rich_markdown", + ) in paths + + +def test_production_reachability_detects_reachable_nested_closure( + tmp_path: Path, +) -> None: + (tmp_path / "applications").mkdir() + (tmp_path / "adapters/parsers").mkdir(parents=True) + (tmp_path / "applications/entry.py").write_text( + """ +def production_entry(raw, config): + def hidden(): + from adapters.parsers.ragflow_markdown import compile_rich_markdown + return compile_rich_markdown(raw, config) + return hidden() +""".lstrip(), + encoding="utf-8", + ) + (tmp_path / "adapters/parsers/ragflow_markdown.py").write_text( + "def compile_rich_markdown(raw, config):\n return raw, config\n", + encoding="utf-8", + ) + + paths = _production_paths_to_forbidden( + tmp_path, + production_roots=("engine", "adapters", "applications"), + ignored_root_modules=frozenset(), + forbidden=frozenset( + {"adapters.parsers.ragflow_markdown.compile_rich_markdown"} + ), + ) + + assert ( + "applications.entry.production_entry", + "applications.entry.production_entry.hidden", + "adapters.parsers.ragflow_markdown.compile_rich_markdown", + ) in paths From d02f81be3b56231bbf0ae39e988074d45839c89f Mon Sep 17 00:00:00 2001 From: stone Date: Thu, 30 Jul 2026 07:33:29 +0800 Subject: [PATCH 12/16] test: close compiler reachability graph (#124) --- ...est_compiler_runner_production_boundary.py | 96 ++++++++++++++++++- 1 file changed, 94 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_compiler_runner_production_boundary.py b/tests/unit/test_compiler_runner_production_boundary.py index 3978b47d..b3085d22 100644 --- a/tests/unit/test_compiler_runner_production_boundary.py +++ b/tests/unit/test_compiler_runner_production_boundary.py @@ -78,6 +78,15 @@ def collect(nodes: list[ast.stmt], prefix: str) -> None: collect(node.body, qualified) elif isinstance(node, ast.ClassDef): collect(node.body, f"{prefix}.{node.name}") + else: + collect( + [ + child + for child in ast.iter_child_nodes(node) + if isinstance(child, ast.stmt) + ], + prefix, + ) collect(tree.body, module) return definitions @@ -89,7 +98,8 @@ def _resolve_import_module(module: str, imported: str | None, level: int) -> str package = module.rsplit(".", maxsplit=1)[0] parts = package.split(".") if package else [] retained = parts[: max(0, len(parts) - level + 1)] - return ".".join((*retained, *((imported or "").split(".")))) + imported_parts = imported.split(".") if imported else [] + return ".".join((*retained, *imported_parts)) def _imports(nodes: Iterable[ast.AST], *, module: str) -> dict[str, str]: @@ -171,7 +181,7 @@ def _function_calls( names.update( { child.name: f"{qualified}.{child.name}" - for child in node.body + for child in scoped if isinstance(child, ast.FunctionDef | ast.AsyncFunctionDef) } ) @@ -427,3 +437,85 @@ def hidden(): "applications.entry.production_entry.hidden", "adapters.parsers.ragflow_markdown.compile_rich_markdown", ) in paths + + +def test_production_reachability_detects_conditionally_declared_closure( + tmp_path: Path, +) -> None: + (tmp_path / "applications").mkdir() + (tmp_path / "adapters/parsers").mkdir(parents=True) + (tmp_path / "applications/entry.py").write_text( + """ +def production_entry(raw, config, enabled=True): + if enabled: + def hidden(): + from adapters.parsers.ragflow_markdown import compile_rich_markdown + return compile_rich_markdown(raw, config) + return hidden() + return None +""".lstrip(), + encoding="utf-8", + ) + (tmp_path / "adapters/parsers/ragflow_markdown.py").write_text( + "def compile_rich_markdown(raw, config):\n return raw, config\n", + encoding="utf-8", + ) + + paths = _production_paths_to_forbidden( + tmp_path, + production_roots=("engine", "adapters", "applications"), + ignored_root_modules=frozenset(), + forbidden=frozenset( + {"adapters.parsers.ragflow_markdown.compile_rich_markdown"} + ), + ) + + assert ( + "applications.entry.production_entry", + "applications.entry.production_entry.hidden", + "adapters.parsers.ragflow_markdown.compile_rich_markdown", + ) in paths + + +def test_production_reachability_detects_relative_module_import( + tmp_path: Path, +) -> None: + (tmp_path / "applications").mkdir() + (tmp_path / "adapters/parsers").mkdir(parents=True) + (tmp_path / "applications/bridge.py").write_text( + """ +from adapters.parsers.ragflow_markdown import compile_rich_markdown + +def hidden(raw, config): + return compile_rich_markdown(raw, config) +""".lstrip(), + encoding="utf-8", + ) + (tmp_path / "applications/entry.py").write_text( + """ +from . import bridge + +def production_entry(raw, config): + return bridge.hidden(raw, config) +""".lstrip(), + encoding="utf-8", + ) + (tmp_path / "adapters/parsers/ragflow_markdown.py").write_text( + "def compile_rich_markdown(raw, config):\n return raw, config\n", + encoding="utf-8", + ) + + paths = _production_paths_to_forbidden( + tmp_path, + production_roots=("engine", "adapters", "applications"), + ignored_root_modules=frozenset(), + forbidden=frozenset( + {"adapters.parsers.ragflow_markdown.compile_rich_markdown"} + ), + ) + + assert ( + "applications.entry.production_entry", + "applications.bridge.hidden", + "adapters.parsers.ragflow_markdown.compile_rich_markdown", + ) in paths From 03109bde9ec9c8bbd3b9f381f7777326dfcfcb32 Mon Sep 17 00:00:00 2001 From: stone Date: Thu, 30 Jul 2026 07:36:40 +0800 Subject: [PATCH 13/16] test: harden compiler production veto (#124) --- ...est_compiler_runner_production_boundary.py | 179 +++++++++++++++++- 1 file changed, 170 insertions(+), 9 deletions(-) diff --git a/tests/unit/test_compiler_runner_production_boundary.py b/tests/unit/test_compiler_runner_production_boundary.py index b3085d22..ae2e0d20 100644 --- a/tests/unit/test_compiler_runner_production_boundary.py +++ b/tests/unit/test_compiler_runner_production_boundary.py @@ -79,14 +79,13 @@ def collect(nodes: list[ast.stmt], prefix: str) -> None: elif isinstance(node, ast.ClassDef): collect(node.body, f"{prefix}.{node.name}") else: - collect( - [ - child - for child in ast.iter_child_nodes(node) - if isinstance(child, ast.stmt) - ], - prefix, - ) + pending = list(ast.iter_child_nodes(node)) + while pending: + child = pending.pop() + if isinstance(child, ast.stmt): + collect([child], prefix) + else: + pending.extend(ast.iter_child_nodes(child)) collect(tree.body, module) return definitions @@ -219,6 +218,53 @@ def _function_calls( ) +def _module_calls( + tree: ast.Module, + *, + module: str, + module_functions: frozenset[str], +) -> frozenset[str]: + body = tuple( + node + for node in tree.body + if not isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef) + ) + names = _module_imports(tree, module=module) + changed = True + while changed: + changed = False + for node in body: + if not isinstance(node, ast.Assign): + continue + resolved = _resolve_expression( + node.value, + names=names, + module=module, + module_functions=module_functions, + ) + if resolved is None: + continue + for target in node.targets: + if isinstance(target, ast.Name) and names.get(target.id) != resolved: + names[target.id] = resolved + changed = True + return frozenset( + resolved + for root in body + for node in ast.walk(root) + if isinstance(node, ast.Call) + for resolved in ( + _resolve_expression( + node.func, + names=names, + module=module, + module_functions=module_functions, + ), + ) + if resolved is not None + ) + + def _production_paths_to_forbidden( repository_root: Path, *, @@ -238,7 +284,16 @@ def _production_paths_to_forbidden( parsed[module] = tree definitions.update(_function_definitions(module, tree)) function_names = frozenset(definitions) - graph: dict[str, frozenset[str]] = {} + graph = { + module: _module_calls( + tree, + module=module, + module_functions=frozenset( + name for name in function_names if name.startswith(f"{module}.") + ), + ) + for module, tree in parsed.items() + } for qualified, definition in definitions.items(): module = qualified.rsplit(".", maxsplit=1)[0] while module not in parsed: @@ -314,6 +369,37 @@ def test_no_production_module_calls_the_unleased_raw_compiler_entry_point() -> N ) == () +def test_production_reachability_detects_module_scope_calls(tmp_path: Path) -> None: + (tmp_path / "applications").mkdir() + (tmp_path / "adapters/parsers").mkdir(parents=True) + (tmp_path / "applications/entry.py").write_text( + """ +from adapters.parsers.ragflow_markdown import compile_rich_markdown + +RESULT = compile_rich_markdown(b"raw", object()) +""".lstrip(), + encoding="utf-8", + ) + (tmp_path / "adapters/parsers/ragflow_markdown.py").write_text( + "def compile_rich_markdown(raw, config):\n return raw, config\n", + encoding="utf-8", + ) + + paths = _production_paths_to_forbidden( + tmp_path, + production_roots=("engine", "adapters", "applications"), + ignored_root_modules=frozenset(), + forbidden=frozenset( + {"adapters.parsers.ragflow_markdown.compile_rich_markdown"} + ), + ) + + assert ( + "applications.entry", + "adapters.parsers.ragflow_markdown.compile_rich_markdown", + ) in paths + + def test_production_reachability_analysis_detects_transitive_indirect_calls( tmp_path: Path, ) -> None: @@ -519,3 +605,78 @@ def production_entry(raw, config): "applications.bridge.hidden", "adapters.parsers.ragflow_markdown.compile_rich_markdown", ) in paths + + +def test_production_reachability_detects_except_handler_closure( + tmp_path: Path, +) -> None: + (tmp_path / "applications").mkdir() + (tmp_path / "adapters/parsers").mkdir(parents=True) + (tmp_path / "applications/entry.py").write_text( + """ +def production_entry(raw, config): + try: + raise RuntimeError + except RuntimeError: + def hidden(): + from adapters.parsers.ragflow_markdown import compile_rich_markdown + return compile_rich_markdown(raw, config) + return hidden() +""".lstrip(), + encoding="utf-8", + ) + (tmp_path / "adapters/parsers/ragflow_markdown.py").write_text( + "def compile_rich_markdown(raw, config):\n return raw, config\n", + encoding="utf-8", + ) + + paths = _production_paths_to_forbidden( + tmp_path, + production_roots=("engine", "adapters", "applications"), + ignored_root_modules=frozenset(), + forbidden=frozenset( + {"adapters.parsers.ragflow_markdown.compile_rich_markdown"} + ), + ) + + assert ( + "applications.entry.production_entry", + "applications.entry.production_entry.hidden", + "adapters.parsers.ragflow_markdown.compile_rich_markdown", + ) in paths + + +def test_production_reachability_detects_module_initializer_call( + tmp_path: Path, +) -> None: + (tmp_path / "applications").mkdir() + (tmp_path / "adapters/parsers").mkdir(parents=True) + (tmp_path / "applications/entry.py").write_text( + """ +from adapters.parsers.ragflow_markdown import compile_rich_markdown + +raw = b"# heading" +config = object() +compiler = compile_rich_markdown +compiler(raw, config) +""".lstrip(), + encoding="utf-8", + ) + (tmp_path / "adapters/parsers/ragflow_markdown.py").write_text( + "def compile_rich_markdown(raw, config):\n return raw, config\n", + encoding="utf-8", + ) + + paths = _production_paths_to_forbidden( + tmp_path, + production_roots=("engine", "adapters", "applications"), + ignored_root_modules=frozenset(), + forbidden=frozenset( + {"adapters.parsers.ragflow_markdown.compile_rich_markdown"} + ), + ) + + assert ( + "applications.entry", + "adapters.parsers.ragflow_markdown.compile_rich_markdown", + ) in paths From 67d35047fa070d6bc721a436a8bd238948e39654 Mon Sep 17 00:00:00 2001 From: stone Date: Thu, 30 Jul 2026 07:51:22 +0800 Subject: [PATCH 14/16] fix: enforce compiler acceptance capability (#124) --- applications/compiler_runner.py | 84 ++- ...ompile-rich-markdown-in-an-owned-runner.md | 20 +- eval/_compiler_acceptance.py | 31 + .../unit/test_compiler_runner_determinism.py | 7 +- ...est_compiler_runner_production_boundary.py | 643 ++---------------- tests/unit/test_compiler_runner_refusal.py | 76 ++- 6 files changed, 258 insertions(+), 603 deletions(-) create mode 100644 eval/_compiler_acceptance.py diff --git a/applications/compiler_runner.py b/applications/compiler_runner.py index 6780bc76..3ba1565d 100644 --- a/applications/compiler_runner.py +++ b/applications/compiler_runner.py @@ -10,7 +10,7 @@ import subprocess import sys from pathlib import Path, PurePath -from typing import Final, cast +from typing import Final, Protocol, cast from adapters.parsers.ragflow_markdown import compile_rich_markdown, rich_token_count from engine.supply import ( @@ -25,11 +25,26 @@ canonicalize_parsed_document, deserialize_parsed_document, ) +from eval._compiler_acceptance import ( + _AcceptanceContext, + acceptance_context, + is_acceptance_context, +) _RUNNER_MODULE: Final = "applications.compiler_runner" COMPILER_RUNNER_TIMEOUT_SECONDS: Final = 30.0 +class _AcceptanceEntryPoint(Protocol): + def __call__( + self, + source: bytes, + config: MarkdownCompilerConfig, + *, + acceptance_context: _AcceptanceContext, + ) -> CompilationOutcome: ... + + def _boundary_failure() -> CompilationFailure: return CompilationFailure( code=CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE, @@ -78,12 +93,36 @@ def _failure_from_document(value: object) -> CompilationFailure: ) +def _require_acceptance_context( + entry_point: _AcceptanceEntryPoint, +) -> _AcceptanceEntryPoint: + def guarded( + source: bytes, + config: MarkdownCompilerConfig, + *, + acceptance_context: _AcceptanceContext | None = None, + ) -> CompilationOutcome: + if is_acceptance_context(acceptance_context): + return entry_point( + source, + config, + acceptance_context=cast(_AcceptanceContext, acceptance_context), + ) + return _boundary_failure() + + return cast(_AcceptanceEntryPoint, guarded) + + +@_require_acceptance_context def compile_in_local_compiler_runner( source: bytes, config: MarkdownCompilerConfig, + *, + acceptance_context: _AcceptanceContext, ) -> CompilationOutcome: """Compile in an unleased local process that production must never call.""" + assert is_acceptance_context(acceptance_context) if type(source) is not bytes: raise TypeError("compiler-runner source must be exact bytes") if type(config) is not MarkdownCompilerConfig: @@ -136,11 +175,19 @@ def compile_in_local_compiler_runner( return _boundary_failure() -def _emit(source: bytes, config: MarkdownCompilerConfig) -> None: - try: - outcome = compile_rich_markdown(source, config) - except Exception: - outcome = _boundary_failure() +def _emit( + source: bytes, + config: MarkdownCompilerConfig, + *, + acceptance_context: _AcceptanceContext | None = None, +) -> None: + if not is_acceptance_context(acceptance_context): + outcome: CompilationOutcome = _boundary_failure() + else: + try: + outcome = compile_rich_markdown(source, config) + except Exception: + outcome = _boundary_failure() if type(outcome) is ParsedDocument: envelope: dict[str, object] = { "outcome": "parsed", @@ -200,7 +247,14 @@ def _safe_markdown_files(root: Path) -> tuple[Path, ...]: ) -def _acceptance_report(root: Path, token_ceiling: int) -> dict[str, object]: +def _acceptance_report( + root: Path, + token_ceiling: int, + *, + acceptance_context: _AcceptanceContext, +) -> dict[str, object]: + if not is_acceptance_context(acceptance_context): + raise ValueError("acceptance report requires its private context") accepted = 0 refused = 0 digests: list[str] = [] @@ -263,10 +317,20 @@ def _acceptance_report(root: Path, token_ceiling: int) -> dict[str, object]: } -def _write_acceptance_report(root: Path, output: Path, token_ceiling: int) -> None: +def _write_acceptance_report( + root: Path, + output: Path, + token_ceiling: int, + *, + acceptance_context: _AcceptanceContext, +) -> None: if ".context-engine" not in output.parts: raise ValueError("acceptance reports must be written under .context-engine") - report = _acceptance_report(root, token_ceiling) + report = _acceptance_report( + root, + token_ceiling, + acceptance_context=acceptance_context, + ) serialized = json.dumps(report, sort_keys=True, separators=(",", ":")) + "\n" output.parent.mkdir(parents=True, exist_ok=True) output.write_text(serialized, encoding="utf-8") @@ -282,6 +346,7 @@ def main() -> None: args.config, token_ceiling=cast(int, args.token_ceiling), ), + acceptance_context=acceptance_context(), ) return if args.acceptance_report: @@ -291,6 +356,7 @@ def main() -> None: cast(Path, args.root), cast(Path, args.output), cast(int, args.token_ceiling), + acceptance_context=acceptance_context(), ) return raise SystemExit("one runner operation is required") diff --git a/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md b/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md index 319f0fc2..fb0267eb 100644 --- a/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md +++ b/docs/decisions/0079-compile-rich-markdown-in-an-owned-runner.md @@ -121,11 +121,17 @@ index and executes under the exact parent WorkerLease binding. recognition methods directly. ContextEngine rewrites rich construct classification, exact raw-byte position mapping, ancestry, typed output, and splitting because the upstream return shape cannot express those - contracts. The process performs no network or database I/O and retains no - independent state, cache, index, or checkpoint. Both the direct compiler - seam and subprocess envelope convert unexpected parser or domain-constructor - rejection into a typed all-or-nothing `CompilationFailure`; no partial - document or raw exception crosses the runner boundary. + contracts. The unleased entry point requires an exact process-local + acceptance context from a module-private capability that is not exported, + configurable, or environment-derived; omitting or forging that context is a + typed refusal. A sound static gate additionally forbids every other + production module from directly importing the raw compiler, local runner, + or private capability. The process performs no network or database I/O and + retains no independent state, cache, index, or checkpoint. Both the direct + compiler seam and subprocess envelope convert unexpected parser or + domain-constructor rejection into a typed all-or-nothing + `CompilationFailure`; no partial document or raw exception crosses the + runner boundary. 7. **Activation remains deferred.** This decision proves the v3 pure transform and local acceptance reporting only. The active File import configuration, database publication functions, immutable Revision schemas, embeddings, @@ -133,8 +139,8 @@ index and executes under the exact parent WorkerLease binding. production publication requires a separate decision and complete atomic publication and re-embedding evidence. The Supply execution bridge in issue #125 owns the future production invocation and must bind it to the exact - parent WorkerLease. No production module may call this issue's unleased - local subprocess helper. + parent WorkerLease. Production activation cannot reuse or manufacture this + issue's acceptance context. 8. **Runtime remains sealed.** Compilation changes no Runtime composition. After any future v3 activation, each published Fragment is still only a candidate until it crosses the exact diff --git a/eval/_compiler_acceptance.py b/eval/_compiler_acceptance.py new file mode 100644 index 00000000..ad499950 --- /dev/null +++ b/eval/_compiler_acceptance.py @@ -0,0 +1,31 @@ +"""Private capability for local compiler acceptance and tests only.""" + +from __future__ import annotations + +from typing import Final + +_CONSTRUCTION_TOKEN: Final = object() + + +class _AcceptanceContext: + __slots__ = () + + def __new__(cls, token: object) -> _AcceptanceContext: + if cls is not _AcceptanceContext or token is not _CONSTRUCTION_TOKEN: + raise TypeError("compiler acceptance context cannot be constructed") + return super().__new__(cls) + + +_CONTEXT: Final = _AcceptanceContext(_CONSTRUCTION_TOKEN) + + +def acceptance_context() -> _AcceptanceContext: + """Return the process-local capability for explicit acceptance work.""" + + return _CONTEXT + + +def is_acceptance_context(value: object) -> bool: + """Return whether the exact private process-local capability was supplied.""" + + return value is _CONTEXT diff --git a/tests/unit/test_compiler_runner_determinism.py b/tests/unit/test_compiler_runner_determinism.py index e9e50a15..883c9a1a 100644 --- a/tests/unit/test_compiler_runner_determinism.py +++ b/tests/unit/test_compiler_runner_determinism.py @@ -15,6 +15,7 @@ ParsedDocument, deserialize_parsed_document, ) +from eval._compiler_acceptance import acceptance_context FIXTURES = Path(__file__).parents[1] / "fixtures/markdown" CONFIG = MarkdownCompilerConfig(version="markdown-config-v3") @@ -111,6 +112,10 @@ def direct_call_must_not_run(*args: object, **kwargs: object) -> object: direct_call_must_not_run, ) - outcome = compiler_runner.compile_in_local_compiler_runner(b"# T\n", CONFIG) + outcome = compiler_runner.compile_in_local_compiler_runner( + b"# T\n", + CONFIG, + acceptance_context=acceptance_context(), + ) assert type(outcome) is ParsedDocument diff --git a/tests/unit/test_compiler_runner_production_boundary.py b/tests/unit/test_compiler_runner_production_boundary.py index ae2e0d20..979ebb4f 100644 --- a/tests/unit/test_compiler_runner_production_boundary.py +++ b/tests/unit/test_compiler_runner_production_boundary.py @@ -1,96 +1,33 @@ from __future__ import annotations import ast -from collections.abc import Iterable from pathlib import Path import applications.compiler_runner as compiler_runner REPOSITORY_ROOT = Path(__file__).parents[2] PRODUCTION_ROOTS = ("engine", "adapters", "applications") -_LOCAL_RUNNER = REPOSITORY_ROOT / "applications/compiler_runner.py" -_FORBIDDEN_PRODUCTION_CALLS = frozenset( +_IGNORED_MODULES = frozenset( { - "adapters.parsers.ragflow_markdown.compile_rich_markdown", - "applications.compiler_runner.compile_in_local_compiler_runner", + "adapters.parsers.ragflow_markdown", + "applications.compiler_runner", } ) -_IGNORED_ROOT_MODULES = frozenset( +_FORBIDDEN_MODULES = frozenset( { "adapters.parsers.ragflow_markdown", "applications.compiler_runner", + "eval._compiler_acceptance", } ) -def _qualified_production_calls() -> tuple[tuple[str, str], ...]: - calls: list[tuple[str, str]] = [] - for root_name in PRODUCTION_ROOTS: - for path in (REPOSITORY_ROOT / root_name).rglob("*.py"): - if path == _LOCAL_RUNNER: - continue - tree = ast.parse(path.read_bytes(), filename=str(path)) - imported_modules: dict[str, str] = {} - imported_symbols: dict[str, str] = {} - for node in ast.walk(tree): - if isinstance(node, ast.Import): - for alias in node.names: - imported_modules[alias.asname or alias.name] = alias.name - elif isinstance(node, ast.ImportFrom) and node.module is not None: - for alias in node.names: - imported_symbols[alias.asname or alias.name] = ( - f"{node.module}.{alias.name}" - ) - for node in ast.walk(tree): - if not isinstance(node, ast.Call): - continue - if isinstance(node.func, ast.Name): - qualified = imported_symbols.get(node.func.id) - elif ( - isinstance(node.func, ast.Attribute) - and isinstance(node.func.value, ast.Name) - ): - module = imported_modules.get(node.func.value.id) - qualified = f"{module}.{node.func.attr}" if module else None - else: - qualified = None - if qualified is not None: - calls.append((str(path.relative_to(REPOSITORY_ROOT)), qualified)) - return tuple(calls) - - def _module_name(repository_root: Path, path: Path) -> str: relative = path.relative_to(repository_root) parts = relative.with_suffix("").parts return ".".join(parts[:-1] if parts[-1] == "__init__" else parts) -def _function_definitions( - module: str, tree: ast.Module -) -> dict[str, ast.FunctionDef | ast.AsyncFunctionDef]: - definitions: dict[str, ast.FunctionDef | ast.AsyncFunctionDef] = {} - - def collect(nodes: list[ast.stmt], prefix: str) -> None: - for node in nodes: - if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): - qualified = f"{prefix}.{node.name}" - definitions[qualified] = node - collect(node.body, qualified) - elif isinstance(node, ast.ClassDef): - collect(node.body, f"{prefix}.{node.name}") - else: - pending = list(ast.iter_child_nodes(node)) - while pending: - child = pending.pop() - if isinstance(child, ast.stmt): - collect([child], prefix) - else: - pending.extend(ast.iter_child_nodes(child)) - - collect(tree.body, module) - return definitions - - def _resolve_import_module(module: str, imported: str | None, level: int) -> str: if level == 0: return imported or "" @@ -101,13 +38,17 @@ def _resolve_import_module(module: str, imported: str | None, level: int) -> str return ".".join((*retained, *imported_parts)) -def _imports(nodes: Iterable[ast.AST], *, module: str) -> dict[str, str]: - imports: dict[str, str] = {} - for node in nodes: +def _forbidden_imports(module: str, tree: ast.Module) -> frozenset[str]: + violations: set[str] = set() + for node in ast.walk(tree): if isinstance(node, ast.Import): for alias in node.names: - bound = alias.asname or alias.name.split(".", maxsplit=1)[0] - imports[bound] = alias.name if alias.asname else bound + if any( + alias.name == forbidden + or alias.name.startswith(f"{forbidden}.") + for forbidden in _FORBIDDEN_MODULES + ): + violations.add(alias.name) elif isinstance(node, ast.ImportFrom): imported_module = _resolve_import_module( module, @@ -115,216 +56,44 @@ def _imports(nodes: Iterable[ast.AST], *, module: str) -> dict[str, str]: node.level, ) for alias in node.names: - imports[alias.asname or alias.name] = ( + qualified = ( f"{imported_module}.{alias.name}" if imported_module else alias.name ) - return imports - - -def _module_imports(tree: ast.Module, *, module: str) -> dict[str, str]: - return _imports(tree.body, module=module) - - -def _scoped_nodes( - node: ast.FunctionDef | ast.AsyncFunctionDef, -) -> Iterable[ast.AST]: - pending: list[ast.AST] = list(node.body) - while pending: - current = pending.pop() - yield current - if isinstance( - current, - ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef | ast.Lambda, - ): - continue - pending.extend(ast.iter_child_nodes(current)) - - -def _resolve_expression( - expression: ast.expr, - *, - names: dict[str, str], - module: str, - module_functions: frozenset[str], -) -> str | None: - if isinstance(expression, ast.Name): - imported = names.get(expression.id) - local = f"{module}.{expression.id}" - return imported if imported is not None else ( - local if local in module_functions else None - ) - if isinstance(expression, ast.Attribute): - base = _resolve_expression( - expression.value, - names=names, - module=module, - module_functions=module_functions, - ) - return f"{base}.{expression.attr}" if base is not None else None - return None - - -def _function_calls( - node: ast.FunctionDef | ast.AsyncFunctionDef, - *, - qualified: str, - imports: dict[str, str], - module: str, - module_functions: frozenset[str], -) -> frozenset[str]: - names = dict(imports) - scoped = tuple(_scoped_nodes(node)) - names.update(_imports(scoped, module=module)) - names.update( - { - child.name: f"{qualified}.{child.name}" - for child in scoped - if isinstance(child, ast.FunctionDef | ast.AsyncFunctionDef) - } - ) - changed = True - while changed: - changed = False - for child in scoped: - if not isinstance(child, ast.Assign): - continue - resolved = _resolve_expression( - child.value, - names=names, - module=module, - module_functions=module_functions, - ) - if resolved is None: - continue - for target in child.targets: - if isinstance(target, ast.Name) and names.get(target.id) != resolved: - names[target.id] = resolved - changed = True - return frozenset( - resolved - for child in scoped - if isinstance(child, ast.Call) - for resolved in ( - _resolve_expression( - child.func, - names=names, - module=module, - module_functions=module_functions, - ), - ) - if resolved is not None - ) - - -def _module_calls( - tree: ast.Module, - *, - module: str, - module_functions: frozenset[str], -) -> frozenset[str]: - body = tuple( - node - for node in tree.body - if not isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef) - ) - names = _module_imports(tree, module=module) - changed = True - while changed: - changed = False - for node in body: - if not isinstance(node, ast.Assign): - continue - resolved = _resolve_expression( - node.value, - names=names, - module=module, - module_functions=module_functions, - ) - if resolved is None: - continue - for target in node.targets: - if isinstance(target, ast.Name) and names.get(target.id) != resolved: - names[target.id] = resolved - changed = True - return frozenset( - resolved - for root in body - for node in ast.walk(root) - if isinstance(node, ast.Call) - for resolved in ( - _resolve_expression( - node.func, - names=names, - module=module, - module_functions=module_functions, - ), - ) - if resolved is not None - ) + if any( + imported_module == forbidden + or imported_module.startswith(f"{forbidden}.") + or qualified == forbidden + or qualified.startswith(f"{forbidden}.") + for forbidden in _FORBIDDEN_MODULES + ): + violations.add(qualified) + return frozenset(violations) -def _production_paths_to_forbidden( +def _production_import_violations( repository_root: Path, *, production_roots: tuple[str, ...], - ignored_root_modules: frozenset[str], - forbidden: frozenset[str], -) -> tuple[tuple[str, ...], ...]: - parsed: dict[str, ast.Module] = {} - definitions: dict[str, ast.FunctionDef | ast.AsyncFunctionDef] = {} + ignored_modules: frozenset[str], +) -> tuple[tuple[str, str], ...]: + violations: set[tuple[str, str]] = set() for root_name in production_roots: root = repository_root / root_name if not root.exists(): continue for path in root.rglob("*.py"): module = _module_name(repository_root, path) + if module in ignored_modules: + continue tree = ast.parse(path.read_bytes(), filename=str(path)) - parsed[module] = tree - definitions.update(_function_definitions(module, tree)) - function_names = frozenset(definitions) - graph = { - module: _module_calls( - tree, - module=module, - module_functions=frozenset( - name for name in function_names if name.startswith(f"{module}.") - ), - ) - for module, tree in parsed.items() - } - for qualified, definition in definitions.items(): - module = qualified.rsplit(".", maxsplit=1)[0] - while module not in parsed: - module = module.rsplit(".", maxsplit=1)[0] - graph[qualified] = _function_calls( - definition, - qualified=qualified, - imports=_module_imports(parsed[module], module=module), - module=module, - module_functions=frozenset( - name for name in function_names if name.startswith(f"{module}.") - ), - ) - - paths: set[tuple[str, ...]] = set() - - def walk(current: str, path: tuple[str, ...]) -> None: - for called in graph.get(current, frozenset()): - next_path = (*path, called) - if called in forbidden: - paths.add(next_path) - elif called in graph and called not in path: - walk(called, next_path) - - for function in graph: - if not any( - function == module or function.startswith(f"{module}.") - for module in ignored_root_modules - ): - walk(function, (function,)) - return tuple(sorted(paths)) + relative = path.relative_to(repository_root).as_posix() + violations.update( + (relative, imported) + for imported in _forbidden_imports(module, tree) + ) + return tuple(sorted(violations)) def test_unleased_subprocess_helper_is_explicitly_local_only() -> None: @@ -332,351 +101,67 @@ def test_unleased_subprocess_helper_is_explicitly_local_only() -> None: assert hasattr(compiler_runner, "compile_in_local_compiler_runner") -def test_no_production_module_calls_the_local_compiler_runner() -> None: - forbidden_module = "applications.compiler_runner" - forbidden_symbol = "compile_in_local_compiler_runner" - callers: list[str] = [] - for root_name in PRODUCTION_ROOTS: - for path in (REPOSITORY_ROOT / root_name).rglob("*.py"): - if path == REPOSITORY_ROOT / "applications/compiler_runner.py": - continue - tree = ast.parse(path.read_bytes(), filename=str(path)) - for node in ast.walk(tree): - imported_from_runner = isinstance(node, ast.ImportFrom) and ( - node.module == forbidden_module - or any(alias.name == forbidden_symbol for alias in node.names) - or ( - node.module == "applications" - and any( - alias.name == "compiler_runner" for alias in node.names - ) - ) - ) - imported_runner = isinstance(node, ast.Import) and any( - alias.name == forbidden_module for alias in node.names - ) - if imported_from_runner or imported_runner: - callers.append(str(path.relative_to(REPOSITORY_ROOT))) - assert callers == [] - - -def test_no_production_module_calls_the_unleased_raw_compiler_entry_point() -> None: - assert _production_paths_to_forbidden( +def test_no_production_module_imports_an_unleased_compiler_surface() -> None: + assert _production_import_violations( REPOSITORY_ROOT, production_roots=PRODUCTION_ROOTS, - ignored_root_modules=_IGNORED_ROOT_MODULES, - forbidden=_FORBIDDEN_PRODUCTION_CALLS, + ignored_modules=_IGNORED_MODULES, ) == () -def test_production_reachability_detects_module_scope_calls(tmp_path: Path) -> None: - (tmp_path / "applications").mkdir() - (tmp_path / "adapters/parsers").mkdir(parents=True) - (tmp_path / "applications/entry.py").write_text( - """ -from adapters.parsers.ragflow_markdown import compile_rich_markdown - -RESULT = compile_rich_markdown(b"raw", object()) -""".lstrip(), - encoding="utf-8", - ) - (tmp_path / "adapters/parsers/ragflow_markdown.py").write_text( - "def compile_rich_markdown(raw, config):\n return raw, config\n", - encoding="utf-8", - ) - - paths = _production_paths_to_forbidden( - tmp_path, - production_roots=("engine", "adapters", "applications"), - ignored_root_modules=frozenset(), - forbidden=frozenset( - {"adapters.parsers.ragflow_markdown.compile_rich_markdown"} - ), - ) - - assert ( - "applications.entry", - "adapters.parsers.ragflow_markdown.compile_rich_markdown", - ) in paths - - -def test_production_reachability_analysis_detects_transitive_indirect_calls( +def test_direct_production_import_gate_rejects_the_unleased_entry_point( tmp_path: Path, ) -> None: (tmp_path / "applications").mkdir() - (tmp_path / "adapters/parsers").mkdir(parents=True) - (tmp_path / "applications/bridge.py").write_text( - """ -from adapters.parsers.ragflow_markdown import compile_rich_markdown as rich_compile - -def hidden(raw, config): - compiler = rich_compile - return compiler(raw, config) -""".lstrip(), - encoding="utf-8", - ) (tmp_path / "applications/entry.py").write_text( - """ -from applications.bridge import hidden - -def production_entry(raw, config): - return hidden(raw, config) -""".lstrip(), - encoding="utf-8", - ) - (tmp_path / "adapters/parsers/ragflow_markdown.py").write_text( - "def compile_rich_markdown(raw, config):\n return raw, config\n", + "from applications.compiler_runner import " + "compile_in_local_compiler_runner\n", encoding="utf-8", ) - paths = _production_paths_to_forbidden( + assert _production_import_violations( tmp_path, - production_roots=("engine", "adapters", "applications"), - ignored_root_modules=frozenset(), - forbidden=frozenset( - {"adapters.parsers.ragflow_markdown.compile_rich_markdown"} + production_roots=("applications",), + ignored_modules=frozenset(), + ) == ( + ( + "applications/entry.py", + "applications.compiler_runner.compile_in_local_compiler_runner", ), ) - assert ( - "applications.entry.production_entry", - "applications.bridge.hidden", - "adapters.parsers.ragflow_markdown.compile_rich_markdown", - ) in paths - -def test_production_reachability_detects_scoped_and_relative_import_aliases( +def test_production_import_gate_rejects_private_capability_imports( tmp_path: Path, ) -> None: (tmp_path / "applications").mkdir() - (tmp_path / "adapters/parsers").mkdir(parents=True) - (tmp_path / "applications/bridge.py").write_text( - """ -def hidden(raw, config): - from adapters.parsers.ragflow_markdown import compile_rich_markdown as rich - compiler = rich - return compiler(raw, config) -""".lstrip(), - encoding="utf-8", - ) (tmp_path / "applications/entry.py").write_text( - """ -from .bridge import hidden as delegated - -def production_entry(raw, config): - return delegated(raw, config) -""".lstrip(), - encoding="utf-8", - ) - (tmp_path / "adapters/parsers/ragflow_markdown.py").write_text( - "def compile_rich_markdown(raw, config):\n return raw, config\n", + "from eval import _compiler_acceptance\n", encoding="utf-8", ) - paths = _production_paths_to_forbidden( + assert _production_import_violations( tmp_path, - production_roots=("engine", "adapters", "applications"), - ignored_root_modules=frozenset(), - forbidden=frozenset( - {"adapters.parsers.ragflow_markdown.compile_rich_markdown"} - ), - ) + production_roots=("applications",), + ignored_modules=frozenset(), + ) == (("applications/entry.py", "eval._compiler_acceptance"),) - assert ( - "applications.entry.production_entry", - "applications.bridge.hidden", - "adapters.parsers.ragflow_markdown.compile_rich_markdown", - ) in paths - -def test_production_reachability_detects_reachable_nested_closure( +def test_production_import_gate_rejects_module_and_submodule_spellings( tmp_path: Path, ) -> None: (tmp_path / "applications").mkdir() - (tmp_path / "adapters/parsers").mkdir(parents=True) (tmp_path / "applications/entry.py").write_text( - """ -def production_entry(raw, config): - def hidden(): - from adapters.parsers.ragflow_markdown import compile_rich_markdown - return compile_rich_markdown(raw, config) - return hidden() -""".lstrip(), - encoding="utf-8", - ) - (tmp_path / "adapters/parsers/ragflow_markdown.py").write_text( - "def compile_rich_markdown(raw, config):\n return raw, config\n", + "import applications.compiler_runner\n" + "import adapters.parsers.ragflow_markdown.helpers\n", encoding="utf-8", ) - paths = _production_paths_to_forbidden( + assert _production_import_violations( tmp_path, - production_roots=("engine", "adapters", "applications"), - ignored_root_modules=frozenset(), - forbidden=frozenset( - {"adapters.parsers.ragflow_markdown.compile_rich_markdown"} - ), - ) - - assert ( - "applications.entry.production_entry", - "applications.entry.production_entry.hidden", - "adapters.parsers.ragflow_markdown.compile_rich_markdown", - ) in paths - - -def test_production_reachability_detects_conditionally_declared_closure( - tmp_path: Path, -) -> None: - (tmp_path / "applications").mkdir() - (tmp_path / "adapters/parsers").mkdir(parents=True) - (tmp_path / "applications/entry.py").write_text( - """ -def production_entry(raw, config, enabled=True): - if enabled: - def hidden(): - from adapters.parsers.ragflow_markdown import compile_rich_markdown - return compile_rich_markdown(raw, config) - return hidden() - return None -""".lstrip(), - encoding="utf-8", - ) - (tmp_path / "adapters/parsers/ragflow_markdown.py").write_text( - "def compile_rich_markdown(raw, config):\n return raw, config\n", - encoding="utf-8", - ) - - paths = _production_paths_to_forbidden( - tmp_path, - production_roots=("engine", "adapters", "applications"), - ignored_root_modules=frozenset(), - forbidden=frozenset( - {"adapters.parsers.ragflow_markdown.compile_rich_markdown"} - ), - ) - - assert ( - "applications.entry.production_entry", - "applications.entry.production_entry.hidden", - "adapters.parsers.ragflow_markdown.compile_rich_markdown", - ) in paths - - -def test_production_reachability_detects_relative_module_import( - tmp_path: Path, -) -> None: - (tmp_path / "applications").mkdir() - (tmp_path / "adapters/parsers").mkdir(parents=True) - (tmp_path / "applications/bridge.py").write_text( - """ -from adapters.parsers.ragflow_markdown import compile_rich_markdown - -def hidden(raw, config): - return compile_rich_markdown(raw, config) -""".lstrip(), - encoding="utf-8", - ) - (tmp_path / "applications/entry.py").write_text( - """ -from . import bridge - -def production_entry(raw, config): - return bridge.hidden(raw, config) -""".lstrip(), - encoding="utf-8", - ) - (tmp_path / "adapters/parsers/ragflow_markdown.py").write_text( - "def compile_rich_markdown(raw, config):\n return raw, config\n", - encoding="utf-8", + production_roots=("applications",), + ignored_modules=frozenset(), + ) == ( + ("applications/entry.py", "adapters.parsers.ragflow_markdown.helpers"), + ("applications/entry.py", "applications.compiler_runner"), ) - - paths = _production_paths_to_forbidden( - tmp_path, - production_roots=("engine", "adapters", "applications"), - ignored_root_modules=frozenset(), - forbidden=frozenset( - {"adapters.parsers.ragflow_markdown.compile_rich_markdown"} - ), - ) - - assert ( - "applications.entry.production_entry", - "applications.bridge.hidden", - "adapters.parsers.ragflow_markdown.compile_rich_markdown", - ) in paths - - -def test_production_reachability_detects_except_handler_closure( - tmp_path: Path, -) -> None: - (tmp_path / "applications").mkdir() - (tmp_path / "adapters/parsers").mkdir(parents=True) - (tmp_path / "applications/entry.py").write_text( - """ -def production_entry(raw, config): - try: - raise RuntimeError - except RuntimeError: - def hidden(): - from adapters.parsers.ragflow_markdown import compile_rich_markdown - return compile_rich_markdown(raw, config) - return hidden() -""".lstrip(), - encoding="utf-8", - ) - (tmp_path / "adapters/parsers/ragflow_markdown.py").write_text( - "def compile_rich_markdown(raw, config):\n return raw, config\n", - encoding="utf-8", - ) - - paths = _production_paths_to_forbidden( - tmp_path, - production_roots=("engine", "adapters", "applications"), - ignored_root_modules=frozenset(), - forbidden=frozenset( - {"adapters.parsers.ragflow_markdown.compile_rich_markdown"} - ), - ) - - assert ( - "applications.entry.production_entry", - "applications.entry.production_entry.hidden", - "adapters.parsers.ragflow_markdown.compile_rich_markdown", - ) in paths - - -def test_production_reachability_detects_module_initializer_call( - tmp_path: Path, -) -> None: - (tmp_path / "applications").mkdir() - (tmp_path / "adapters/parsers").mkdir(parents=True) - (tmp_path / "applications/entry.py").write_text( - """ -from adapters.parsers.ragflow_markdown import compile_rich_markdown - -raw = b"# heading" -config = object() -compiler = compile_rich_markdown -compiler(raw, config) -""".lstrip(), - encoding="utf-8", - ) - (tmp_path / "adapters/parsers/ragflow_markdown.py").write_text( - "def compile_rich_markdown(raw, config):\n return raw, config\n", - encoding="utf-8", - ) - - paths = _production_paths_to_forbidden( - tmp_path, - production_roots=("engine", "adapters", "applications"), - ignored_root_modules=frozenset(), - forbidden=frozenset( - {"adapters.parsers.ragflow_markdown.compile_rich_markdown"} - ), - ) - - assert ( - "applications.entry", - "adapters.parsers.ragflow_markdown.compile_rich_markdown", - ) in paths diff --git a/tests/unit/test_compiler_runner_refusal.py b/tests/unit/test_compiler_runner_refusal.py index ae3f8506..77671b12 100644 --- a/tests/unit/test_compiler_runner_refusal.py +++ b/tests/unit/test_compiler_runner_refusal.py @@ -3,6 +3,7 @@ import json import subprocess from pathlib import Path +from typing import Any, cast import pytest @@ -17,10 +18,43 @@ ParsedDocument, UnsupportedConstruct, ) +from eval._compiler_acceptance import _AcceptanceContext, acceptance_context CONFIG = MarkdownCompilerConfig(version="markdown-config-v3") +def test_unleased_runner_without_acceptance_context_is_a_typed_refusal() -> None: + unchecked_runner = cast(Any, compile_in_local_compiler_runner) + outcome = unchecked_runner(b"# T\n", CONFIG) + + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE + assert outcome.position is None + + +def test_explicit_acceptance_context_allows_the_local_runner() -> None: + outcome = compile_in_local_compiler_runner( + b"# T\n", + CONFIG, + acceptance_context=acceptance_context(), + ) + + assert type(outcome) is ParsedDocument + + +def test_acceptance_context_cannot_be_forged() -> None: + with pytest.raises(TypeError, match="cannot be constructed"): + _AcceptanceContext(object()) + + outcome = compile_in_local_compiler_runner( + b"# T\n", + CONFIG, + acceptance_context=cast(Any, object()), + ) + assert type(outcome) is CompilationFailure + assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE + + @pytest.mark.parametrize( ("source", "expected_code"), [ @@ -56,7 +90,11 @@ def test_malformed_source_returns_typed_failure_across_runner_boundary( source: bytes, expected_code: CompilationFailureCode, ) -> None: - outcome = compile_in_local_compiler_runner(source, CONFIG) + outcome = compile_in_local_compiler_runner( + source, + CONFIG, + acceptance_context=acceptance_context(), + ) assert type(outcome) is CompilationFailure assert outcome.code is expected_code @@ -77,7 +115,11 @@ def test_unlisted_inline_constructs_are_typed_refusals( source: bytes, construct: UnsupportedConstruct, ) -> None: - outcome = compile_in_local_compiler_runner(source, CONFIG) + outcome = compile_in_local_compiler_runner( + source, + CONFIG, + acceptance_context=acceptance_context(), + ) assert type(outcome) is CompilationFailure assert outcome.code is CompilationFailureCode.UNSUPPORTED_CONSTRUCT @@ -111,7 +153,11 @@ def raise_unexpected( monkeypatch.setattr(compiler_runner, "compile_rich_markdown", raise_unexpected) - compiler_runner._emit(b"# T\n", CONFIG) + compiler_runner._emit( + b"# T\n", + CONFIG, + acceptance_context=acceptance_context(), + ) outcome = compiler_runner._failure_from_document( json.loads(capsys.readouterr().out)["failure"] @@ -135,7 +181,11 @@ def test_runner_api_converts_child_process_failure_to_typed_failure( ), ) - outcome = compile_in_local_compiler_runner(b"# T\n", CONFIG) + outcome = compile_in_local_compiler_runner( + b"# T\n", + CONFIG, + acceptance_context=acceptance_context(), + ) assert type(outcome) is CompilationFailure assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE @@ -153,7 +203,11 @@ def wedge(*args: object, timeout: float, **kwargs: object) -> object: monkeypatch.setattr(subprocess, "run", wedge) - outcome = compile_in_local_compiler_runner(b"# T\n", CONFIG) + outcome = compile_in_local_compiler_runner( + b"# T\n", + CONFIG, + acceptance_context=acceptance_context(), + ) assert observed and observed[0] > 0 assert type(outcome) is CompilationFailure @@ -178,7 +232,11 @@ def test_runner_api_terminates_a_deliberately_wedged_child( ) monkeypatch.setattr(compiler_runner, "COMPILER_RUNNER_TIMEOUT_SECONDS", 0.05) - outcome = compile_in_local_compiler_runner(b"# T\n", CONFIG) + outcome = compile_in_local_compiler_runner( + b"# T\n", + CONFIG, + acceptance_context=acceptance_context(), + ) assert type(outcome) is CompilationFailure assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE @@ -302,7 +360,11 @@ def reject_document(payload: bytes) -> ParsedDocument: reject_document, ) - outcome = compile_in_local_compiler_runner(b"# T\n", CONFIG) + outcome = compile_in_local_compiler_runner( + b"# T\n", + CONFIG, + acceptance_context=acceptance_context(), + ) assert type(outcome) is CompilationFailure assert outcome.code is CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE From d8b9f794893a798f761cf24c07f4b48c250b3a0e Mon Sep 17 00:00:00 2001 From: stone Date: Thu, 30 Jul 2026 11:19:04 +0800 Subject: [PATCH 15/16] test: close relative compiler import veto (#124) --- ...est_compiler_runner_production_boundary.py | 111 +++++++++++++++++- 1 file changed, 107 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_compiler_runner_production_boundary.py b/tests/unit/test_compiler_runner_production_boundary.py index 979ebb4f..73c9a069 100644 --- a/tests/unit/test_compiler_runner_production_boundary.py +++ b/tests/unit/test_compiler_runner_production_boundary.py @@ -1,6 +1,8 @@ from __future__ import annotations import ast +import subprocess +import sys from pathlib import Path import applications.compiler_runner as compiler_runner @@ -28,17 +30,28 @@ def _module_name(repository_root: Path, path: Path) -> str: return ".".join(parts[:-1] if parts[-1] == "__init__" else parts) -def _resolve_import_module(module: str, imported: str | None, level: int) -> str: +def _resolve_import_module( + module: str, + imported: str | None, + level: int, + *, + is_package: bool, +) -> str: if level == 0: return imported or "" - package = module.rsplit(".", maxsplit=1)[0] + package = module if is_package else module.rsplit(".", maxsplit=1)[0] parts = package.split(".") if package else [] retained = parts[: max(0, len(parts) - level + 1)] imported_parts = imported.split(".") if imported else [] return ".".join((*retained, *imported_parts)) -def _forbidden_imports(module: str, tree: ast.Module) -> frozenset[str]: +def _forbidden_imports( + module: str, + tree: ast.Module, + *, + is_package: bool, +) -> frozenset[str]: violations: set[str] = set() for node in ast.walk(tree): if isinstance(node, ast.Import): @@ -54,6 +67,7 @@ def _forbidden_imports(module: str, tree: ast.Module) -> frozenset[str]: module, node.module, node.level, + is_package=is_package, ) for alias in node.names: qualified = ( @@ -91,7 +105,11 @@ def _production_import_violations( relative = path.relative_to(repository_root).as_posix() violations.update( (relative, imported) - for imported in _forbidden_imports(module, tree) + for imported in _forbidden_imports( + module, + tree, + is_package=path.name == "__init__.py", + ) ) return tuple(sorted(violations)) @@ -165,3 +183,88 @@ def test_production_import_gate_rejects_module_and_submodule_spellings( ("applications/entry.py", "adapters.parsers.ragflow_markdown.helpers"), ("applications/entry.py", "applications.compiler_runner"), ) + + +def test_production_import_gate_rejects_live_relative_call_from_package_init( + tmp_path: Path, +) -> None: + package = tmp_path / "adapters/parsers" + package.mkdir(parents=True) + (tmp_path / "adapters/__init__.py").write_text("", encoding="utf-8") + (package / "ragflow_markdown.py").write_text( + "def compile_rich_markdown(source: bytes):\n" + " return source + b' compiled'\n", + encoding="utf-8", + ) + (package / "__init__.py").write_text( + "from .ragflow_markdown import compile_rich_markdown\n" + "\n" + "def production_rich_compile(source: bytes):\n" + " return compile_rich_markdown(source)\n", + encoding="utf-8", + ) + + completed = subprocess.run( + [ + sys.executable, + "-c", + "from adapters.parsers import production_rich_compile; " + "assert production_rich_compile(b'source') == b'source compiled'", + ], + cwd=tmp_path, + check=False, + capture_output=True, + timeout=10, + ) + + assert completed.returncode == 0 + assert _production_import_violations( + tmp_path, + production_roots=("adapters",), + ignored_modules=frozenset(), + ) == ( + ( + "adapters/parsers/__init__.py", + "adapters.parsers.ragflow_markdown.compile_rich_markdown", + ), + ) + + +def test_production_import_gate_resolves_every_relative_package_level( + tmp_path: Path, +) -> None: + parsers = tmp_path / "adapters/parsers" + nested = parsers / "nested" + deeper = nested / "deeper" + deeper.mkdir(parents=True) + (parsers / "__init__.py").write_text( + "from . import ragflow_markdown\n", + encoding="utf-8", + ) + (nested / "__init__.py").write_text( + "from ..ragflow_markdown import compile_rich_markdown\n", + encoding="utf-8", + ) + (deeper / "__init__.py").write_text( + "from ... import ragflow_markdown\n", + encoding="utf-8", + ) + + assert _production_import_violations( + tmp_path, + production_roots=("adapters",), + ignored_modules=frozenset(), + ) == ( + ( + "adapters/parsers/__init__.py", + "adapters.parsers.ragflow_markdown", + ), + ( + "adapters/parsers/nested/__init__.py", + "adapters.parsers.ragflow_markdown.compile_rich_markdown", + ), + ( + "adapters/parsers/nested/deeper/__init__.py", + "adapters.parsers.ragflow_markdown", + ), + ) From de8c95fa6f0668cb156515f938504b40d68135c4 Mon Sep 17 00:00:00 2001 From: stone Date: Thu, 30 Jul 2026 11:19:04 +0800 Subject: [PATCH 16/16] fix: redact compiler acceptance read failures (#124) --- applications/compiler_runner.py | 8 ++- .../test_compiler_runner_acceptance_report.py | 67 +++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/applications/compiler_runner.py b/applications/compiler_runner.py index 3ba1565d..7fa35652 100644 --- a/applications/compiler_runner.py +++ b/applications/compiler_runner.py @@ -262,7 +262,13 @@ def _acceptance_report( histogram = {name: 0 for name in _CONSTRUCT_PATTERNS} refusal_histogram: dict[str, int] = {} for path in _safe_markdown_files(root): - source = path.read_bytes() + try: + source = path.read_bytes() + except OSError: + refused += 1 + category = CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE.value + refusal_histogram[category] = refusal_histogram.get(category, 0) + 1 + continue try: inspected = source.removeprefix(b"\xef\xbb\xbf").decode("utf-8") except UnicodeDecodeError: diff --git a/tests/unit/test_compiler_runner_acceptance_report.py b/tests/unit/test_compiler_runner_acceptance_report.py index 357e6b64..d7010849 100644 --- a/tests/unit/test_compiler_runner_acceptance_report.py +++ b/tests/unit/test_compiler_runner_acceptance_report.py @@ -9,6 +9,7 @@ import pytest +import applications.compiler_runner as compiler_runner from eval.embedding_benchmark import ( BenchmarkUnavailable, validate_json_schema_document, @@ -228,3 +229,69 @@ def test_acceptance_report_is_count_only_deterministic_and_written_under_ignore( assert "refused.md" not in first_bytes.decode("utf-8") assert first_bytes == output.read_bytes() assert first.stdout == second.stdout + + +@pytest.mark.parametrize( + "failure_kind", + ("io", "permission", "vanished", "directory"), +) +def test_acceptance_cli_error_paths_emit_only_counted_private_safe_outcomes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + failure_kind: str, +) -> None: + corpus = tmp_path / _SYNTHETIC_PRIVATE_ROOT_FRAGMENT + corpus.mkdir() + note = corpus / "private-note-canary.md" + if failure_kind == "directory": + note.mkdir() + elif failure_kind != "vanished": + note.write_text("# Synthetic\n", encoding="utf-8") + output = tmp_path / ".context-engine/compiler-runner-acceptance.json" + monkeypatch.setattr( + compiler_runner, + "_safe_markdown_files", + lambda root: (note,), + ) + original_read_bytes = Path.read_bytes + + def read_bytes(path: Path) -> bytes: + if path != note: + return original_read_bytes(path) + if failure_kind == "io": + raise OSError(5, "synthetic I/O failure", str(path)) + if failure_kind == "permission": + raise PermissionError(13, "synthetic permission failure", str(path)) + return original_read_bytes(path) + + monkeypatch.setattr(Path, "read_bytes", read_bytes) + monkeypatch.setattr( + sys, + "argv", + [ + "compiler-runner", + "--acceptance-report", + "--root", + str(corpus), + "--output", + str(output), + ], + ) + + compiler_runner.main() + + captured = capsys.readouterr() + emitted = captured.out + captured.err + report = json.loads(output.read_text(encoding="utf-8")) + assert report["documents"] == { + "accepted": 0, + "acceptanceRate": "0.000000", + "refused": 1, + "total": 1, + } + assert report["refusalHistogram"] == {"unsupported_document_shape": 1} + assert captured.err == "" + assert not _contains_private_location(emitted) + assert str(tmp_path) not in emitted + assert note.name not in emitted