RFC 008 slice 0: validation contracts as executable artifacts

pyproject.toml

@@ -66,6 +66,7 @@ include-package-data = true
 [tool.setuptools.package-data]
 "openenv.cli" = ["templates/**/*"]
 "openenv.cli.importers" = ["templates/**/*"]
+"openenv.validation" = ["policies/*.json", "schemas/*.json"]
 
 [tool.setuptools.exclude-package-data]
 "*" = ["*.pyc", "*.pyo", "__pycache__/*"]

scripts/sync_validation_schemas.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""Sync committed validation JSON Schemas with their pydantic models.
+
+The pydantic models in src/openenv/validation/ are the source of truth;
+src/openenv/validation/schemas/*.schema.json are committed exports. This script
+keeps them in sync (same pattern as scripts/sync_env_docs.py).
+
+Modes:
+  --check : Exit non-zero if out of sync (for CI)
+  --fix   : Regenerate the committed schema files from the models
+
+Run with: PYTHONPATH=src python scripts/sync_validation_schemas.py --check
+"""
+
+import argparse
+import json
+import os
+import sys
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, os.path.join(ROOT, "src"))
+
+SCHEMAS_DIR = os.path.join(ROOT, "src", "openenv", "validation", "schemas")
+
+
+def rendered_schemas():
+    """Return {filename: rendered JSON text} for every exported schema."""
+    from openenv.validation.manifest import NormalizedManifest
+    from openenv.validation.report import ValidationReport
+
+    exports = {
+        "manifest.schema.json": NormalizedManifest,
+        "report.schema.json": ValidationReport,
+    }
+    return {
+        fname: json.dumps(model.model_json_schema(), indent=2, sort_keys=True) + "\n"
+        for fname, model in exports.items()
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    mode = parser.add_mutually_exclusive_group(required=True)
+    mode.add_argument(
+        "--check", action="store_true", help="exit non-zero if out of sync"
+    )
+    mode.add_argument("--fix", action="store_true", help="regenerate committed schemas")
+    args = parser.parse_args()
+
+    os.makedirs(SCHEMAS_DIR, exist_ok=True)
+    stale = []
+    for fname, rendered in rendered_schemas().items():
+        path = os.path.join(SCHEMAS_DIR, fname)
+        existing = None
+        if os.path.exists(path):
+            with open(path) as f:
+                existing = f.read()
+        if existing == rendered:
+            continue
+        if args.fix:
+            with open(path, "w") as f:
+                f.write(rendered)
+            print(f"wrote {os.path.relpath(path, ROOT)}")
+        else:
+            stale.append(fname)
+
+    if stale:
+        print("committed schemas are out of sync with the pydantic models:")
+        for fname in stale:
+            print(f"  {fname}")
+        print("run: PYTHONPATH=src python scripts/sync_validation_schemas.py --fix")
+        return 1
+    if args.check:
+        print("validation schemas in sync")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

src/openenv/validation/__init__.py

@@ -0,0 +1,88 @@
+"""Environment validation (RFC 008): contracts and the local `openenv validate` pipeline.
+
+Ships local validation plus the contracts that let any operator build a hub —
+signature detection, parsers, one normalized manifest, a grader registry, a report
+schema, and a versioned severity policy. Graders read the manifest, never the
+signature.
+"""
+
+from .graders import ENTRY_POINT_GROUP, Grader, GraderRegistry, Subject
+from .manifest import (
+    CapabilitiesSpec,
+    JudgePin,
+    NetworkPolicy,
+    NormalizedManifest,
+    OracleDeclaration,
+    ResourceDeclaration,
+    RewardDeclaration,
+    TaskDistributionPin,
+    TypeSpec,
+    VerifierBinding,
+)
+from .parsers import Parser, ParserRegistry
+from .policy import (
+    apply_policy,
+    DeclarationBounds,
+    load_policy,
+    PolicyEntry,
+    PolicyError,
+    SeverityPolicy,
+)
+from .providers import ExecResult, RunningSubject, ValidationProvider
+from .report import CheckResult, ValidationReport
+from .signature import (
+    SignatureError,
+    UNSUPPORTED_CATEGORIES,
+    UnsupportedPackageError,
+    WELL_KNOWN_FILES,
+)
+from .types import (
+    CheckStatus,
+    Lane,
+    Level,
+    ProviderCapability,
+    Severity,
+    SignatureKind,
+    Verdict,
+)
+
+__all__ = [
+    "ENTRY_POINT_GROUP",
+    "UNSUPPORTED_CATEGORIES",
+    "WELL_KNOWN_FILES",
+    "CapabilitiesSpec",
+    "CheckResult",
+    "CheckStatus",
+    "DeclarationBounds",
+    "ExecResult",
+    "Grader",
+    "GraderRegistry",
+    "JudgePin",
+    "Lane",
+    "Level",
+    "NetworkPolicy",
+    "NormalizedManifest",
+    "OracleDeclaration",
+    "Parser",
+    "ParserRegistry",
+    "PolicyEntry",
+    "PolicyError",
+    "ProviderCapability",
+    "ResourceDeclaration",
+    "RewardDeclaration",
+    "RunningSubject",
+    "Severity",
+    "SeverityPolicy",
+    "SignatureError",
+    "SignatureKind",
+    "Subject",
+    "TaskDistributionPin",
+    "TypeSpec",
+    "UnsupportedPackageError",
+    "ValidationProvider",
+    "ValidationReport",
+    "Verdict",
+    "VerifierBinding",
+    "apply_policy",
+    "load_policy",
+]

src/openenv/validation/graders/__init__.py

@@ -0,0 +1,147 @@
+"""Grader protocol and registry.
+
+Selection reads the manifest only — the signature never reaches
+[`~openenv.validation.graders.GraderRegistry.select`]. Write a grader once and it works
+with every format whose parser supplies the manifest fields.
+"""
+
+from dataclasses import dataclass
+from importlib.metadata import entry_points
+from pathlib import Path
+from typing import Protocol, runtime_checkable
+
+from ..manifest import NormalizedManifest
+from ..providers import RunningSubject
+from ..report import CheckResult
+from ..types import Level, ProviderCapability
+
+ENTRY_POINT_GROUP = "openenv.validation.graders"
+"""Third-party graders register under this entry-point group — zero core changes."""
+
+
+@dataclass(frozen=True)
+class Subject:
+    """
+    The one argument every grader receives.
+
+    Attributes:
+        root (`Path`):
+            Package source tree.
+        manifest ([`~openenv.validation.manifest.NormalizedManifest`]):
+            The parsed, normalized manifest.
+        image_ref (`str`, *optional*):
+            Built image reference; `None` under `--skip-build`.
+        running ([`~openenv.validation.providers.RunningSubject`], *optional*):
+            The started sandbox; `None` at the static level.
+        outputs_dir (`Path`):
+            Where trajectory records and replay artifacts are written.
+    """
+
+    root: Path
+    manifest: NormalizedManifest
+    image_ref: str | None
+    running: RunningSubject | None
+    outputs_dir: Path
+
+
+@runtime_checkable
+class Grader(Protocol):
+    """
+    One check. Emits a [`~openenv.validation.report.CheckResult`] with a
+    [`~openenv.validation.types.CheckStatus`]; never a severity.
+
+    Attributes:
+        check_id (`str`):
+            Stable, policy-addressed id, e.g. `"semantic.oracle_max"`.
+        level ([`~openenv.validation.types.Level`]):
+            The validation level this check belongs to.
+        requires_capabilities (`frozenset[str]`):
+            `CapabilitiesSpec` field names that must be truthy for this grader to apply.
+        requires_provider (`frozenset` of [`~openenv.validation.types.ProviderCapability`]):
+            Provider capabilities this grader needs; missing ones SKIP the check.
+        depends_on (`tuple[str, ...]`):
+            Check ids; any non-PASS dependency SKIPs this check with the dependency
+            named (this encodes "L2 determinism is a precondition of L3 semantics").
+    """
+
+    check_id: str
+    level: Level
+    requires_capabilities: frozenset[str]
+    requires_provider: frozenset[ProviderCapability]
+    depends_on: tuple[str, ...]
+
+    def applies_to(self, manifest: NormalizedManifest) -> bool: ...
+
+    def run(self, subject: Subject) -> CheckResult: ...
+
+
+class GraderRegistry:
+    """
+    Holds all known graders and selects the applicable subset per run.
+
+    Core graders are registered directly; third parties register via the
+    `"openenv.validation.graders"` entry-point group.
+    """
+
+    def __init__(self) -> None:
+        self._graders: dict[str, Grader] = {}
+
+    def register(self, grader: Grader) -> None:
+        """
+        Register a grader by its check id.
+
+        Args:
+            grader ([`~openenv.validation.graders.Grader`]):
+                The grader to register.
+        """
+        if grader.check_id in self._graders:
+            raise ValueError(
+                f"grader already registered for check id {grader.check_id!r}"
+            )
+        self._graders[grader.check_id] = grader
+
+    def load_entry_points(self) -> int:
+        """
+        Register third-party graders from the entry-point group.
+
+        Returns:
+            `int`: the number of graders loaded.
+        """
+        loaded = 0
+        for ep in entry_points(group=ENTRY_POINT_GROUP):
+            grader = ep.load()
+            if callable(grader) and not isinstance(grader, Grader):
+                grader = grader()
+            self.register(grader)
+            loaded += 1
+        return loaded
+
+    def select(self, manifest: NormalizedManifest, max_level: Level) -> list[Grader]:
+        """
+        Select the graders that apply to a manifest, up to a level ceiling.
+
+        Selection reads the manifest only: capabilities select contract graders, type
+        tags select domain graders. The signature never reaches this method's inputs.
+
+        Args:
+            manifest ([`~openenv.validation.manifest.NormalizedManifest`]):
+                The parsed manifest.
+            max_level ([`~openenv.validation.types.Level`]):
+                Ceiling; graders above it are not selected.
+
+        Returns:
+            `list` of [`~openenv.validation.graders.Grader`], ordered by (level, check id).
+        """
+        selected = []
+        for grader in self._graders.values():
+            if grader.level > max_level:
+                continue
+            if not all(
+                getattr(manifest.capabilities, field)
+                for field in grader.requires_capabilities
+            ):
+                continue
+            if not grader.applies_to(manifest):
+                continue
+            selected.append(grader)
+        return sorted(selected, key=lambda g: (g.level, g.check_id))