Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions adapters/parsers/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
"""Parser adapters that implement Supply compiler contracts."""

from adapters.parsers.markdown import compile_markdown

__all__ = ["compile_markdown"]
139 changes: 139 additions & 0 deletions adapters/parsers/markdown.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
"""Pure adapter for the first deliberately narrow Markdown grammar."""

from __future__ import annotations

import re
from typing import Final

from engine.supply.markdown import (
MARKDOWN_COMPILER_VERSION,
CompilationFailure,
CompilationFailureCode,
CompilationOutcome,
CompilationProvenance,
MarkdownCompilerConfig,
ParsedDocument,
ParsedSection,
SectionKind,
SourcePoint,
SourceSpan,
StructuralPath,
unsupported_markdown_construct,
)

_UTF8_BOM: Final = b"\xef\xbb\xbf"
_HEADING_PATTERN: Final = re.compile(r"^# (\S(?:.*\S)?)$")


def _point(line: int, column: int, byte_offset: int) -> SourcePoint:
return SourcePoint(line=line, column=column, byte_offset=byte_offset)


def _normalized_text(source: bytes) -> str | CompilationFailure:
raw = source.removeprefix(_UTF8_BOM)
try:
decoded = raw.decode("utf-8", errors="strict")
except UnicodeDecodeError as error:
prefix = raw[: error.start].decode("utf-8", errors="strict")
prefix = prefix.replace("\r\n", "\n").replace("\r", "\n")
last_newline = prefix.rfind("\n")
line = prefix.count("\n") + 1
column = len(prefix[last_newline + 1 :]) + 1
return CompilationFailure(
code=CompilationFailureCode.INVALID_UTF8,
position=_point(line, column, len(prefix.encode("utf-8"))),
)
normalized = decoded.replace("\r\n", "\n").replace("\r", "\n")
return normalized.rstrip("\n") + "\n"


def _failure_point(lines: list[str], line_index: int) -> SourcePoint:
prior = "\n".join(lines[:line_index])
byte_offset = len(prior.encode("utf-8")) + (1 if line_index else 0)
return _point(line_index + 1, 1, byte_offset)


def _sections(
normalized: str,
heading_text: str,
) -> tuple[ParsedSection, ParsedSection]:
heading_line, _, paragraph_line = normalized.removesuffix("\n").split("\n")
heading_end = len(heading_line.encode("utf-8"))
paragraph_start = heading_end + 2
paragraph_end = paragraph_start + len(paragraph_line.encode("utf-8"))
return (
ParsedSection(
kind=SectionKind.HEADING,
text=heading_text,
path=StructuralPath(("document", "heading[1]")),
position=SourceSpan(
start=_point(1, 1, 0),
end=_point(1, len(heading_line) + 1, heading_end),
),
level=1,
),
ParsedSection(
kind=SectionKind.PARAGRAPH,
text=paragraph_line,
path=StructuralPath(
("document", "heading[1]", "paragraph[1]")
),
position=SourceSpan(
start=_point(3, 1, paragraph_start),
end=_point(3, len(paragraph_line) + 1, paragraph_end),
),
),
)


def compile_markdown(
source: bytes,
config: MarkdownCompilerConfig,
) -> CompilationOutcome:
"""Compile exact bytes into the one supported heading-plus-paragraph shape."""

if type(source) is not bytes:
raise TypeError("Markdown compiler source must be exact bytes")
if type(config) is not MarkdownCompilerConfig:
raise TypeError("Markdown compiler config must be MarkdownCompilerConfig")
normalized = _normalized_text(source)
if isinstance(normalized, CompilationFailure):
return normalized

lines = normalized.removesuffix("\n").split("\n")
for line_index, line in enumerate(lines):
construct = unsupported_markdown_construct(
line,
supported_heading=line_index == 0,
)
if construct is not None:
return CompilationFailure(
code=CompilationFailureCode.UNSUPPORTED_CONSTRUCT,
position=_failure_point(lines, line_index),
construct=construct,
)
if (
len(lines) != 3
or lines[1] != ""
or not lines[2]
or lines[2] != lines[2].strip()
):
return CompilationFailure(
code=CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE,
position=_point(1, 1, 0),
)
heading_match = _HEADING_PATTERN.fullmatch(lines[0])
if heading_match is None:
return CompilationFailure(
code=CompilationFailureCode.UNSUPPORTED_DOCUMENT_SHAPE,
position=_point(1, 1, 0),
)
provenance = CompilationProvenance(
compiler_version=MARKDOWN_COMPILER_VERSION,
config_version=config.version,
)
return ParsedDocument.issue_22(
canonical_text=normalized,
sections=_sections(normalized, heading_match.group(1)),
provenance=provenance,
)
92 changes: 92 additions & 0 deletions docs/decisions/0036-compile-narrow-markdown-deterministically.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
---
name: adr-0036-deterministic-narrow-markdown-compiler
version: "1.0.0"
description: >
Fix the first pure Markdown compiler to one heading-plus-paragraph shape,
canonical normalized text, end-exclusive positions, and versioned digests.
---

# 0036. Compile the first Markdown shape from canonical bytes

- Status: accepted
- Date: 2026-07-22
- Refines: ADR-0016, ADR-0018

## Context

The first File tracer bullet needs a reproducible compiler result before any
`ContextRevision`, `ContextFragment`, persistence, indexing, or publication
behavior is activated. Passing a path would couple compilation to acquisition
and host state. Treating unsupported Markdown syntax as ordinary paragraph text
would make later parser upgrades silently change meaning. A content-only hash
also cannot identify which compiler/configuration semantics produced the typed
structure.

Source positions need one stable coordinate system after BOM and newline
normalization. Original byte offsets cannot remain comparable when CRLF becomes
LF, while character offsets alone are insufficient for exact serialized
provenance.

## Decision

Supply owns the typed `ParsedDocument`, failure, provenance, serialization, and
digest contracts. The pure `adapters/parsers` implementation exposes
`compile_markdown(bytes, MarkdownCompilerConfig)` and imports those inward-facing
contracts; the shared domain never imports parser implementation. The adapter
accepts exact UTF-8 bytes, removes at most one leading UTF-8 BOM, normalizes CRLF
and CR to LF, and canonicalizes trailing newlines to exactly one LF. The only
successful grammar is one level-one ATX heading, one blank line, and one plain
single-line paragraph. All source positions refer to the canonical normalized
UTF-8 text: line and column are one-based, byte offset is zero-based, and spans
are end-exclusive.

Invalid UTF-8 and every recognized out-of-scope Markdown construct return a
typed all-or-nothing `CompilationFailure`. They never return a partial
`ParsedDocument`. The current compiler emits an exact empty warning tuple; it
does not claim any lossy warning behavior.

The closed-grammar classifier is a Supply-domain invariant shared by the
adapter and `ParsedDocument` self-validation. Parser ingress is therefore not
the only enforcement point: direct typed construction cannot manufacture a
valid digest for content that the active narrow grammar rejects.

`ParsedDocument` contains canonical text, source-ordered typed sections,
structural paths, canonical source spans, compiler/configuration provenance,
and two distinct SHA-256 identities:

1. `content_hash` hashes canonical normalized UTF-8 only, so BOM/newline
transport variants share content identity; and
2. `compilation_digest` hashes an RFC 8785 canonical document under a domain
separator and includes structure, positions, warnings, compiler version,
configuration version, profiles, and content hash.

The public canonical serialization adds the verified compilation digest to the
same document. Frozen bytes and output plus two fresh interpreter processes with
different hash seeds prove reproducibility.

## Rationale

A bytes-only pure function creates the smallest stable seam between future File
acquisition and immutable publication. Canonical coordinates make normalized
inputs comparable. Separating content identity from compilation identity allows
safe unchanged-byte reasoning without treating a compiler/configuration change
as the same derived artifact.

Failing closed on syntax outside the deliberately narrow grammar prevents the
first compiler from advertising CommonMark coverage it does not implement.

## Consequences

Issue #22 creates no source discovery, filesystem reads, database access,
network calls, model calls, `ContextRevision`, `ContextFragment`, or publication
state. Later parser expansion must version compiler/configuration provenance and
add frozen fixtures before accepting another construct. Acquisition owns the
mapping from original source bytes to this compiler input; it cannot redefine
the canonical output contract.

## Revisit trigger

Revisit when a later issue adds a Markdown construct, source-map requirement,
or representation-affecting configuration. Any replacement must preserve pure
bytes input, typed all-or-nothing failure, version-sensitive compilation
identity, frozen canonical serialization, and cross-process determinism.
2 changes: 2 additions & 0 deletions docs/decisions/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ kernel, capability separation, and publication visibility model.
| Release security catalog | [0019 — Security catalog normalization](0019-security-catalog-normalization.md) | One machine catalog contains exactly fifteen stable release IDs; overlapping labels and derived scenarios keep their safeguards without inflating the count | Parallel prose catalogs, renumbering, or treating inactive cache behavior as a canonical release family |
| Executable M0 security veto | [0034 — Registered executable security evidence](0034-execute-the-m0-security-veto-from-registered-evidence.md) | Exact current tests, explicit hard-oracle observations, and live all-table RLS facts produce provenance-bearing independent gate artifacts | Planned IDs presented as executed proof, skip/retry-to-green, manifest-only RLS claims, or aggregate scoring |
| First File source registration | [0035 — Trusted File source registration](0035-register-file-sources-through-context-control.md) | One operation-bound trusted Control call atomically creates an Organization-owned source plus immutable active first version; all acquisition carriers remain unavailable | Caller-authored Organization/mode, host paths, registration-time File I/O, future capability claims, or cross-tenant idempotency |
| First Markdown compiler | [0036 — Deterministic narrow Markdown compilation](0036-compile-narrow-markdown-deterministically.md) | Exact bytes compile purely into one canonical heading-plus-paragraph ParsedDocument with normalized coordinates and versioned content/compilation identities | Path-coupled parsing, silent unsupported syntax, partial documents, unversioned derived identities, or parser-side I/O |

Each baseline ADR is `accepted` and contains Context, Decision, Rationale,
Consequences, and Revisit trigger sections. A revisit trigger permits review; it
Expand Down Expand Up @@ -119,3 +120,4 @@ touched:
- [0033 — Organization release promotion owner](0033-promote-organization-releases-through-one-learning-owner.md)
- [0034 — Registered executable security evidence](0034-execute-the-m0-security-veto-from-registered-evidence.md)
- [0035 — Trusted File source registration](0035-register-file-sources-through-context-control.md)
- [0036 — Deterministic narrow Markdown compilation](0036-compile-narrow-markdown-deterministically.md)
40 changes: 40 additions & 0 deletions engine/supply/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,49 @@
worker_lease_digest,
worker_lease_nonce_digest,
)
from engine.supply.markdown import (
MARKDOWN_CANONICALIZATION_PROFILE,
MARKDOWN_COMPILATION_DIGEST_PROFILE,
MARKDOWN_COMPILER_VERSION,
MARKDOWN_CONTENT_HASH_PROFILE,
CompilationFailure,
CompilationFailureCode,
CompilationOutcome,
CompilationProvenance,
CompilationWarning,
CompilationWarningCode,
MarkdownCompilerConfig,
ParsedDocument,
ParsedSection,
SectionKind,
SourcePoint,
SourceSpan,
StructuralPath,
UnsupportedConstruct,
canonicalize_parsed_document,
)

__all__ = [
"MARKDOWN_CANONICALIZATION_PROFILE",
"MARKDOWN_COMPILATION_DIGEST_PROFILE",
"MARKDOWN_COMPILER_VERSION",
"MARKDOWN_CONTENT_HASH_PROFILE",
"WORKER_LEASE_ACTOR_KIND",
"WORKER_LEASE_OPERATION",
"CompilationFailure",
"CompilationFailureCode",
"CompilationOutcome",
"CompilationProvenance",
"CompilationWarning",
"CompilationWarningCode",
"MarkdownCompilerConfig",
"ParsedDocument",
"ParsedSection",
"SectionKind",
"SourcePoint",
"SourceSpan",
"StructuralPath",
"UnsupportedConstruct",
"WorkNotAvailable",
"WorkerLeaseClaims",
"WorkerLeaseCodec",
Expand All @@ -26,6 +65,7 @@
"WorkerLeaseRejectionCategory",
"WorkerLeaseToken",
"generate_worker_lease_nonce",
"canonicalize_parsed_document",
"worker_lease_digest",
"worker_lease_nonce_digest",
]
Loading
Loading