diff --git a/envs/echo_env/openenv.yaml b/envs/echo_env/openenv.yaml index 6f178906d..34968ac1d 100644 --- a/envs/echo_env/openenv.yaml +++ b/envs/echo_env/openenv.yaml @@ -1,6 +1,23 @@ spec_version: 1 name: echo_env +version: 0.1.0 type: space runtime: fastapi app: server.app:app port: 8000 +validation: + reward: + range: [0.0, 1.0] + oracle_tolerance: 0.0 + floor_margin: 0.5 + resources: + cpu: 1.0 + memory_mb: 1024 + disk_mb: 512 + episode_timeout_s: 60.0 + capabilities: + verifier: + kind: reward_channel + declared_tools: [echo_message, echo_with_length] + types: + tags: [demo] diff --git a/src/openenv/cli/commands/validate.py b/src/openenv/cli/commands/validate.py index b3926a769..f8813428c 100644 --- a/src/openenv/cli/commands/validate.py +++ b/src/openenv/cli/commands/validate.py @@ -3,8 +3,9 @@ """ OpenEnv validate command. -This module provides the 'openenv validate' command to check if environments -are properly configured for multi-mode deployment. +Local packages run through the RFC 008 validation pipeline (signature -> parse -> +grade -> severity policy -> report). Running servers are probed via --url (this +legacy probe folds into the pipeline's runtime level in a later slice). """ import json @@ -12,13 +13,29 @@ from typing import Annotated import typer -from openenv.cli._validation import ( - build_local_validation_json_report, - format_validation_report, - get_deployment_modes, - validate_multi_mode_deployment, - validate_running_environment, +from openenv.cli._validation import validate_running_environment +from openenv.validation import ( + Level, + load_policy, + PolicyError, + SignatureError, + UnsupportedPackageError, + ValidationReport, + Verdict, ) +from openenv.validation.runner import run_validation + +# Exit-code contract (RFC 008 §8): +EXIT_PASS = 0 # verdict PASS or WARN +EXIT_FAIL = 1 # verdict FAIL +EXIT_UNSUPPORTED = 2 # SignatureError / UnsupportedPackageError +EXIT_INTERNAL = 3 # internal error + +_LEVELS = { + "static": Level.STATIC, + "runtime": Level.RUNTIME, + "semantic": Level.SEMANTIC, +} def _looks_like_url(value: str) -> bool: @@ -27,12 +44,32 @@ def _looks_like_url(value: str) -> bool: return candidate.startswith("http://") or candidate.startswith("https://") +def _render_report(report: ValidationReport) -> str: + """Human-readable report summary.""" + lines = [ + f"Validation report for {report.target} (signature: {report.signature.value})", + f" policy {report.policy_version} · levels run: " + + ", ".join(level.name.lower() for level in report.levels_run), + ] + for result in report.results: + lines.append( + f" {result.status.value.upper():5s} {result.check_id} ({result.duration_s:.2f}s)" + ) + if result.status.value in ("fail", "error", "skip"): + for line in result.evidence: + lines.append(f" {line}") + if result.remediation: + lines.append(f" remediation: {result.remediation}") + lines.append(f"Verdict: {report.verdict.value.upper()}") + return "\n".join(lines) + + def validate( target: Annotated[ str | None, typer.Argument( help=( - "Path to the environment directory (default: current directory) " + "Path to the package directory (default: current directory) " "or a running OpenEnv URL (http://... or https://...)" ), ), @@ -44,54 +81,67 @@ def validate( help="Validate a running OpenEnv server by base URL (e.g. http://localhost:8000)", ), ] = None, - json_output: Annotated[ + level: Annotated[ + str, + typer.Option( + "--level", + help="Validation level ceiling: static, runtime, or semantic", + ), + ] = "semantic", + skip_build: Annotated[ bool, typer.Option( - "--json", - help="Output local validation report as JSON (runtime validation is JSON by default)", + "--skip-build", + help="Skip the image build; build-dependent checks are SKIPped with a reason", ), ] = False, + policy_version: Annotated[ + str, + typer.Option("--policy", help="Severity policy version to apply"), + ] = "v1", + json_output: Annotated[ + bool, + typer.Option("--json", help="Print the validation report as JSON"), + ] = False, + output: Annotated[ + Path | None, + typer.Option("--output", help="Write the JSON report to a file"), + ] = None, timeout: Annotated[ float, typer.Option( "--timeout", - help="HTTP timeout in seconds for runtime validation", + help="HTTP timeout in seconds for --url runtime validation", min=0.1, ), ] = 5.0, - verbose: Annotated[ - bool, typer.Option("--verbose", "-v", help="Show detailed information") - ] = False, ) -> None: """ - Validate local environments and running OpenEnv servers. + Validate a local package or a running OpenEnv server. - Local validation checks if an environment is properly configured with: - - Required files (pyproject.toml, openenv.yaml, server/app.py, etc.) - - Docker deployment support - - uv run server capability - - python -m module execution + Local validation detects the package format by its well-known file (the + formats this build can parse; currently openenv.yaml), parses it into the + normalized manifest, runs the applicable graders up to the requested level, + applies the severity policy, and emits a report. - Runtime validation checks if a live OpenEnv server conforms to the - versioned runtime API contract and returns a criteria-based JSON report. + Exit codes: 0 pass/warn - 1 fail - 2 unrecognized/unsupported package - 3 + internal error. Examples: - ```bash - # Validate current directory (recommended) - $ cd my_env - $ openenv validate + ```bash + # Validate the current directory up to the semantic level + openenv validate - # Validate a running environment and return JSON criteria - $ openenv validate --url http://localhost:8000 - $ openenv validate https://my-env.hf.space + # Fast inner loop: static checks only, no image build + openenv validate envs/echo_env --level static --skip-build - # Validate with detailed output - $ openenv validate --verbose + # Machine-readable report + openenv validate envs/echo_env --json - # Validate specific environment - $ openenv validate envs/echo_env - ``` + # Probe a running server (legacy runtime probe) + openenv validate --url http://localhost:8000 + ``` """ runtime_target = url if ( @@ -103,7 +153,7 @@ def validate( "Error: Cannot combine a local path argument with --url runtime validation", err=True, ) - raise typer.Exit(1) + raise typer.Exit(EXIT_FAIL) if target is not None and _looks_like_url(target): if runtime_target is not None and runtime_target != target: @@ -111,7 +161,7 @@ def validate( "Error: Conflicting runtime targets provided via argument and --url", err=True, ) - raise typer.Exit(1) + raise typer.Exit(EXIT_FAIL) runtime_target = target if runtime_target is not None: @@ -119,79 +169,46 @@ def validate( report = validate_running_environment(runtime_target, timeout_s=timeout) except ValueError as exc: typer.echo(f"Error: {exc}", err=True) - raise typer.Exit(1) from exc + raise typer.Exit(EXIT_FAIL) from exc typer.echo(json.dumps(report, indent=2)) if not report.get("passed", False): - raise typer.Exit(1) + raise typer.Exit(EXIT_FAIL) return - # Determine environment path (default to current directory) - if target is None: - env_path_obj = Path.cwd() - else: - env_path_obj = Path(target) - - if not env_path_obj.exists(): - typer.echo(f"Error: Path does not exist: {env_path_obj}", err=True) - raise typer.Exit(1) - - if not env_path_obj.is_dir(): - typer.echo(f"Error: Path is not a directory: {env_path_obj}", err=True) - raise typer.Exit(1) - - # Check for openenv.yaml to confirm this is an environment directory - openenv_yaml = env_path_obj / "openenv.yaml" - if not openenv_yaml.exists(): + if level not in _LEVELS: typer.echo( - f"Error: Not an OpenEnv environment directory (missing openenv.yaml): {env_path_obj}", + f"Error: unknown level {level!r}; expected one of {sorted(_LEVELS)}", err=True, ) - typer.echo( - "Hint: Run this command from the environment root directory or specify the path", - err=True, + raise typer.Exit(EXIT_INTERNAL) + + package_root = Path(target) if target is not None else Path.cwd() + if not package_root.is_dir(): + typer.echo(f"Error: not a package directory: {package_root}", err=True) + raise typer.Exit(EXIT_UNSUPPORTED) + + try: + validation_report = run_validation( + package_root, + max_level=_LEVELS[level], + skip_build=skip_build, + policy=load_policy(policy_version), ) - raise typer.Exit(1) - - env_name = env_path_obj.name - if env_name.endswith("_env"): - base_name = env_name[:-4] - else: - base_name = env_name - - # Run validation - is_valid, issues = validate_multi_mode_deployment(env_path_obj) - modes = get_deployment_modes(env_path_obj) - + except (SignatureError, UnsupportedPackageError) as exc: + typer.echo(f"Error: {exc}", err=True) + raise typer.Exit(EXIT_UNSUPPORTED) from exc + except PolicyError as exc: + typer.echo(f"Internal error: {exc}", err=True) + raise typer.Exit(EXIT_INTERNAL) from exc + + report_json = validation_report.model_dump_json(indent=2) + if output is not None: + output.write_text(report_json + "\n") if json_output: - report = build_local_validation_json_report( - env_name=base_name, - env_path=env_path_obj, - is_valid=is_valid, - issues=issues, - deployment_modes=modes if verbose else None, - ) - typer.echo(json.dumps(report, indent=2)) - if not is_valid: - raise typer.Exit(1) - return + typer.echo(report_json) + else: + typer.echo(_render_report(validation_report)) - # Show validation report - report = format_validation_report(base_name, is_valid, issues) - typer.echo(report) - - # Show deployment modes if verbose - if verbose: - typer.echo("\nSupported deployment modes:") - for mode, supported in modes.items(): - status = "[YES]" if supported else "[NO]" - typer.echo(f" {status} {mode}") - - if is_valid: - typer.echo("\nUsage examples:") - typer.echo(f" cd {env_path_obj.name} && uv run server") - typer.echo(f" cd {env_path_obj.name} && openenv build") - typer.echo(f" cd {env_path_obj.name} && openenv push") - - if not is_valid: - raise typer.Exit(1) + if validation_report.verdict is Verdict.FAIL: + raise typer.Exit(EXIT_FAIL) diff --git a/src/openenv/validation/__init__.py b/src/openenv/validation/__init__.py index 5b6c06c4e..c44348e25 100644 --- a/src/openenv/validation/__init__.py +++ b/src/openenv/validation/__init__.py @@ -10,6 +10,7 @@ from .manifest import ( CapabilitiesSpec, JudgePin, + ManifestError, NetworkPolicy, NormalizedManifest, OracleDeclaration, @@ -30,7 +31,9 @@ ) from .providers import ExecResult, RunningSubject, ValidationProvider from .report import CheckResult, ValidationReport +from .runner import run_validation from .signature import ( + detect_signature, SignatureError, UNSUPPORTED_CATEGORIES, UnsupportedPackageError, @@ -60,6 +63,7 @@ "JudgePin", "Lane", "Level", + "ManifestError", "NetworkPolicy", "NormalizedManifest", "OracleDeclaration", @@ -84,5 +88,7 @@ "Verdict", "VerifierBinding", "apply_policy", + "detect_signature", "load_policy", + "run_validation", ] diff --git a/src/openenv/validation/graders/static_/__init__.py b/src/openenv/validation/graders/static_/__init__.py new file mode 100644 index 000000000..c4472ae04 --- /dev/null +++ b/src/openenv/validation/graders/static_/__init__.py @@ -0,0 +1,5 @@ +"""Static-level (L1) graders.""" + +from .manifest import StaticManifestGrader + +__all__ = ["StaticManifestGrader"] diff --git a/src/openenv/validation/graders/static_/manifest.py b/src/openenv/validation/graders/static_/manifest.py new file mode 100644 index 000000000..1c77a9b19 --- /dev/null +++ b/src/openenv/validation/graders/static_/manifest.py @@ -0,0 +1,73 @@ +"""The `static.manifest` grader: the manifest is schema-valid and within policy bounds.""" + +import time + +from ...manifest import NormalizedManifest +from ...policy import DeclarationBounds +from ...report import CheckResult +from ...types import CheckStatus, Level + + +class StaticManifestGrader: + """ + Checks the parsed manifest's declared tolerances against the policy bounds. + + Schema validity itself is established before this grader runs (an unparseable + manifest never reaches grading — the runner records the `static.manifest` FAIL + directly from the parser's [`~openenv.validation.manifest.ManifestError`]). What + remains to grade is that author-declared values sit within what the severity + policy allows an author to declare. + """ + + check_id = "static.manifest" + level = Level.STATIC + requires_capabilities: frozenset[str] = frozenset() + requires_provider: frozenset = frozenset() + depends_on: tuple[str, ...] = () + + def __init__(self, bounds: DeclarationBounds): + self._bounds = bounds + + def applies_to(self, manifest: NormalizedManifest) -> bool: + return True + + def run(self, subject) -> CheckResult: + started = time.monotonic() + reward = subject.manifest.reward + bounds = self._bounds + problems = [] + if reward.oracle_tolerance > bounds.max_oracle_tolerance: + problems.append( + f"declared oracle_tolerance {reward.oracle_tolerance} exceeds the " + f"policy maximum {bounds.max_oracle_tolerance}" + ) + if reward.floor_margin < bounds.min_floor_margin: + problems.append( + f"declared floor_margin {reward.floor_margin} is below the " + f"policy minimum {bounds.min_floor_margin}" + ) + if ( + reward.variance_tolerance is not None + and reward.variance_tolerance > bounds.max_variance_tolerance + ): + problems.append( + f"declared variance_tolerance {reward.variance_tolerance} exceeds " + f"the policy maximum {bounds.max_variance_tolerance}" + ) + return CheckResult( + check_id=self.check_id, + status=CheckStatus.FAIL if problems else CheckStatus.PASS, + measured={ + "oracle_tolerance": reward.oracle_tolerance, + "floor_margin": reward.floor_margin, + "variance_tolerance": reward.variance_tolerance, + "bounds": bounds.model_dump(), + }, + evidence=problems or ["manifest is schema-valid and within policy bounds"], + remediation=( + "declare tolerances within the severity policy's bounds" + if problems + else None + ), + duration_s=time.monotonic() - started, + ) diff --git a/src/openenv/validation/manifest.py b/src/openenv/validation/manifest.py index 6135e4db4..47e9319b3 100644 --- a/src/openenv/validation/manifest.py +++ b/src/openenv/validation/manifest.py @@ -12,6 +12,28 @@ from .types import SignatureKind +class ManifestError(Exception): + """ + A package's declarations do not produce a valid normalized manifest. + + Raised by parsers when the format-specific source (e.g. `openenv.yaml`) is + readable but its content fails the manifest schema. The runner surfaces this as a + `static.manifest` FAIL in the report — a graded failure with remediation, not a + crash and not exit code 2. + + Attributes: + errors (`list[str]`): + Human-readable schema violations, one per finding. + remediation (`str`, *optional*): + What the author changes to go green. + """ + + def __init__(self, errors: list[str], *, remediation: str | None = None): + super().__init__("; ".join(errors)) + self.errors = errors + self.remediation = remediation + + class RewardDeclaration(BaseModel): """ Author-declared reward contract. Graders normalize measurements against it. diff --git a/src/openenv/validation/parsers/__init__.py b/src/openenv/validation/parsers/__init__.py index 0f7692919..3f6df4fd5 100644 --- a/src/openenv/validation/parsers/__init__.py +++ b/src/openenv/validation/parsers/__init__.py @@ -9,7 +9,7 @@ from typing import Protocol, runtime_checkable from ..manifest import NormalizedManifest -from ..signature import UnsupportedPackageError +from ..signature import detect_signature, UnsupportedPackageError from ..types import SignatureKind @@ -67,3 +67,18 @@ def parser_for(self, signature: SignatureKind) -> Parser: f"no parser registered for signature {signature.value!r}", ) return parser + + def parse(self, package_root: Path) -> NormalizedManifest: + """ + Detect the package signature and dispatch to its parser. + + The only place a [`~openenv.validation.types.SignatureKind`] selects behavior. + + Args: + package_root (`Path`): + The package directory to detect and parse. + + Returns: + [`~openenv.validation.NormalizedManifest`]: the normalized manifest. + """ + return self.parser_for(detect_signature(package_root)).parse(package_root) diff --git a/src/openenv/validation/parsers/openenv_yaml.py b/src/openenv/validation/parsers/openenv_yaml.py new file mode 100644 index 000000000..a9b7070ed --- /dev/null +++ b/src/openenv/validation/parsers/openenv_yaml.py @@ -0,0 +1,89 @@ +"""Parser for the served OpenEnv format (`openenv.yaml`).""" + +from pathlib import Path + +import yaml +from pydantic import ValidationError + +from ..manifest import ManifestError, NormalizedManifest +from ..types import SignatureKind + +VALIDATION_BLOCK_REMEDIATION = ( + "add a `validation:` block to openenv.yaml declaring reward (range, " + "oracle_tolerance, floor_margin), resources (cpu, memory_mb, disk_mb, " + "episode_timeout_s), capabilities (verifier, oracle, ...), and types (tags); " + "see the manifest schema at src/openenv/validation/schemas/manifest.schema.json" +) + + +class OpenEnvYamlParser: + """ + Parses `openenv.yaml` into the normalized manifest. + + Pure read: never imports or executes package code. Top-level `name`/`version` + identify the environment; everything validation-specific lives under the + `validation:` block, which maps 1:1 onto [`~openenv.validation.NormalizedManifest`] + sections. + """ + + signature = SignatureKind.OPENENV_SERVED + + def parse(self, package_root: Path) -> NormalizedManifest: + """ + Parse a served-environment package. + + Args: + package_root (`Path`): + Directory containing `openenv.yaml`. + + Returns: + [`~openenv.validation.NormalizedManifest`]: the normalized manifest. + """ + source = package_root / "openenv.yaml" + try: + raw = yaml.safe_load(source.read_text()) + except yaml.YAMLError as exc: + raise ManifestError([f"openenv.yaml is not valid YAML: {exc}"]) from exc + if not isinstance(raw, dict): + raise ManifestError(["openenv.yaml must be a YAML mapping"]) + + validation = raw.get("validation") + if validation is None: + raise ManifestError( + ["openenv.yaml has no `validation:` block"], + remediation=VALIDATION_BLOCK_REMEDIATION, + ) + if not isinstance(validation, dict): + raise ManifestError(["`validation:` must be a mapping"]) + + data: dict = { + "manifest_schema_version": "1", + "signature": SignatureKind.OPENENV_SERVED, + "version": raw.get("version"), + "judge": validation.get("judge"), + "task_distribution": validation.get("task_distribution"), + } + # Required sections are omitted when absent so schema errors read as + # "field required" rather than "not a valid dictionary"; network is + # omitted when absent so the manifest default (mode public) applies. + for key, value in ( + ("name", raw.get("name")), + ("reward", validation.get("reward")), + ("resources", validation.get("resources")), + ("capabilities", validation.get("capabilities")), + ("types", validation.get("types")), + ("network", validation.get("network")), + ): + if value is not None: + data[key] = value + + try: + return NormalizedManifest.model_validate(data) + except ValidationError as exc: + errors = [ + f"{'.'.join(str(part) for part in err['loc']) or 'manifest'}: {err['msg']}" + for err in exc.errors() + ] + raise ManifestError( + errors, remediation=VALIDATION_BLOCK_REMEDIATION + ) from exc diff --git a/src/openenv/validation/report.py b/src/openenv/validation/report.py index ff7525063..206867104 100644 --- a/src/openenv/validation/report.py +++ b/src/openenv/validation/report.py @@ -51,6 +51,9 @@ class ValidationReport(BaseModel): Pin `policy_version` and a local run reproduces the hub verdict, modulo hub-lane checks. + + `manifest` is `None` only when the package's declarations failed the manifest + schema — the report then carries the `static.manifest` FAIL explaining why. """ model_config = ConfigDict(extra="forbid") @@ -59,7 +62,7 @@ class ValidationReport(BaseModel): target: str source_digest: str signature: SignatureKind - manifest: NormalizedManifest + manifest: NormalizedManifest | None policy_version: str lane: Lane levels_run: list[Level] diff --git a/src/openenv/validation/runner.py b/src/openenv/validation/runner.py new file mode 100644 index 000000000..6eb6e285b --- /dev/null +++ b/src/openenv/validation/runner.py @@ -0,0 +1,165 @@ +"""Validation orchestration: parse → grade → apply policy → report. + +Levels run in order and accumulate into one report per run — a level with a +policy-fail finding still lets later levels run where dependencies permit, so an +author gets maximum information per run. +""" + +import hashlib +import tempfile +import time +from pathlib import Path + +from .graders import GraderRegistry, Subject +from .graders.static_ import StaticManifestGrader +from .manifest import ManifestError, NormalizedManifest +from .parsers import ParserRegistry +from .parsers.openenv_yaml import OpenEnvYamlParser +from .policy import apply_policy, load_policy, SeverityPolicy +from .providers import ValidationProvider +from .report import CheckResult, ValidationReport +from .signature import detect_signature +from .types import CheckStatus, Lane, Level + +REPORT_SCHEMA_VERSION = "1" + +_DIGEST_EXCLUDED_DIRS = {".git", "__pycache__", ".venv", ".worktrees"} + + +def source_digest(package_root: Path) -> str: + """ + Deterministic sha256 over the package tree (relative paths + file contents). + + Args: + package_root (`Path`): + The package directory. + + Returns: + `str`: a 64-character hex digest. + """ + digest = hashlib.sha256() + files = sorted( + path + for path in package_root.rglob("*") + if path.is_file() + and not any( + part in _DIGEST_EXCLUDED_DIRS + for part in path.relative_to(package_root).parts + ) + ) + for path in files: + digest.update(str(path.relative_to(package_root)).encode()) + digest.update(b"\0") + digest.update(path.read_bytes()) + digest.update(b"\0") + return digest.hexdigest() + + +def default_parser_registry() -> ParserRegistry: + """The parsers shipped in this build.""" + registry = ParserRegistry() + registry.register(OpenEnvYamlParser()) + return registry + + +def default_grader_registry(policy: SeverityPolicy) -> GraderRegistry: + """The core graders shipped in this build, configured against a policy.""" + registry = GraderRegistry() + registry.register(StaticManifestGrader(policy.bounds)) + return registry + + +def _run_grader(grader, subject: Subject) -> CheckResult: + """Run one grader; a crash is an ERROR result (fails closed), never an abort.""" + started = time.monotonic() + try: + return grader.run(subject) + except Exception as exc: # noqa: BLE001 — grader crash is evidence, not fatal + return CheckResult( + check_id=grader.check_id, + status=CheckStatus.ERROR, + evidence=[f"grader crashed: {exc!r}"], + duration_s=time.monotonic() - started, + ) + + +def run_validation( + target: Path, + *, + max_level: Level = Level.SEMANTIC, + provider: ValidationProvider | None = None, + skip_build: bool = False, + policy: SeverityPolicy | None = None, +) -> ValidationReport: + """ + Validate a package end to end and return the report. + + Raises [`~openenv.validation.SignatureError`] for ambiguous/unrecognized packages + and [`~openenv.validation.UnsupportedPackageError`] for recognized-but-unsupported + ones (CLI exit code 2). A package whose declarations fail the manifest schema + yields a normal report with a `static.manifest` FAIL (exit code 1). + + Args: + target (`Path`): + The package directory. + max_level ([`~openenv.validation.types.Level`], *optional*, defaults to `Level.SEMANTIC`): + Level ceiling; graders above it are not selected. + provider ([`~openenv.validation.ValidationProvider`], *optional*): + Sandbox provider for runtime+ levels. Unused until the runtime phase + lands; `None` selects the default provider then. + skip_build (`bool`, *optional*, defaults to `False`): + Skip the image build; build-dependent checks SKIP with a reason. + policy ([`~openenv.validation.SeverityPolicy`], *optional*): + Severity policy; `None` loads the committed default version. + + Returns: + [`~openenv.validation.ValidationReport`]: the completed report. + """ + target = Path(target) + policy = policy or load_policy() + signature = detect_signature(target) + + parser = default_parser_registry().parser_for(signature) + manifest: NormalizedManifest | None = None + results: list[CheckResult] = [] + levels_run: list[Level] = [Level.STATIC] + + parse_started = time.monotonic() + try: + manifest = parser.parse(target) + except ManifestError as exc: + results.append( + CheckResult( + check_id="static.manifest", + status=CheckStatus.FAIL, + measured={"schema_errors": len(exc.errors)}, + evidence=exc.errors, + remediation=exc.remediation, + duration_s=time.monotonic() - parse_started, + ) + ) + + if manifest is not None: + graders = default_grader_registry(policy).select(manifest, max_level) + subject = Subject( + root=target, + manifest=manifest, + image_ref=None, + running=None, + outputs_dir=Path(tempfile.mkdtemp(prefix="openenv-validate-")), + ) + results.extend(_run_grader(grader, subject) for grader in graders) + levels_run = sorted({Level.STATIC, *(g.level for g in graders)}) + + return ValidationReport( + report_schema_version=REPORT_SCHEMA_VERSION, + target=str(target), + source_digest=source_digest(target), + signature=signature, + manifest=manifest, + policy_version=policy.policy_version, + lane=Lane.LOCAL, + levels_run=levels_run, + results=results, + verdict=apply_policy(results, policy, Lane.LOCAL), + ) diff --git a/src/openenv/validation/schemas/report.schema.json b/src/openenv/validation/schemas/report.schema.json index 29519e1e5..2dcea7371 100644 --- a/src/openenv/validation/schemas/report.schema.json +++ b/src/openenv/validation/schemas/report.schema.json @@ -519,7 +519,7 @@ } }, "additionalProperties": false, - "description": "One report per validation run, maximum information per run.\n\nPin `policy_version` and a local run reproduces the hub verdict, modulo hub-lane\nchecks.", + "description": "One report per validation run, maximum information per run.\n\nPin `policy_version` and a local run reproduces the hub verdict, modulo hub-lane\nchecks.\n\n`manifest` is `None` only when the package's declarations failed the manifest\nschema \u2014 the report then carries the `static.manifest` FAIL explaining why.", "properties": { "lane": { "$ref": "#/$defs/Lane" @@ -532,7 +532,14 @@ "type": "array" }, "manifest": { - "$ref": "#/$defs/NormalizedManifest" + "anyOf": [ + { + "$ref": "#/$defs/NormalizedManifest" + }, + { + "type": "null" + } + ] }, "policy_version": { "title": "Policy Version", diff --git a/src/openenv/validation/signature.py b/src/openenv/validation/signature.py index 4841567e6..5c81734f6 100644 --- a/src/openenv/validation/signature.py +++ b/src/openenv/validation/signature.py @@ -1,13 +1,12 @@ -"""Signature detection contracts: well-known files, never a guess. +"""Signature detection: well-known files, never a guess.""" -`detect_signature` itself lands with the walking skeleton (slice 1); this module ships -the detection rules and error contract so parsers, registries, and the CLI exit-code -contract can be written against them. -""" +from pathlib import Path from .types import SignatureKind -WELL_KNOWN_FILES: dict[SignatureKind, str] = {} +WELL_KNOWN_FILES: dict[SignatureKind, str] = { + SignatureKind.OPENENV_SERVED: "openenv.yaml", +} """Signature detection table: the formats THIS build can parse. Entries are added alongside their parsers (`openenv.yaml` with the served-env @@ -62,3 +61,40 @@ def __init__(self, category: str, reason: str): super().__init__(f"unsupported package ({category}): {reason}") self.category = category self.reason = reason + + +def detect_signature(package_root: Path) -> SignatureKind: + """ + Detect a package's format from its well-known file. Never a guess. + + Only formats with implemented parsers are recognized; anything else is + refused as unrecognized. + + Args: + package_root (`Path`): + Directory to inspect. + + Returns: + [`~openenv.validation.types.SignatureKind`]: the single matching signature. + Zero or two+ matches raise [`~openenv.validation.signature.SignatureError`]. + """ + if not package_root.is_dir(): + raise SignatureError(f"not a package directory: {package_root}") + matches = [ + kind + for kind, filename in WELL_KNOWN_FILES.items() + if (package_root / filename).is_file() + ] + if not matches: + expected = ", ".join(sorted(WELL_KNOWN_FILES.values())) + raise SignatureError( + f"unrecognized package: {package_root} contains none of the well-known " + f"files this build can parse ({expected})" + ) + if len(matches) > 1: + found = ", ".join(sorted(WELL_KNOWN_FILES[m] for m in matches)) + raise SignatureError( + f"ambiguous package: {package_root} matches multiple signatures ({found}); " + "a package must carry exactly one well-known file" + ) + return matches[0] diff --git a/tests/fixtures/validation/empty_solution_max_reward/openenv.yaml b/tests/fixtures/validation/empty_solution_max_reward/openenv.yaml index 2b3c86eb2..eb0bc880e 100644 --- a/tests/fixtures/validation/empty_solution_max_reward/openenv.yaml +++ b/tests/fixtures/validation/empty_solution_max_reward/openenv.yaml @@ -1,4 +1,5 @@ spec_version: 1 +version: 0.1.0 name: empty-solution-max-reward type: env runtime: python diff --git a/tests/fixtures/validation/leaky_egress/openenv.yaml b/tests/fixtures/validation/leaky_egress/openenv.yaml index a7209b0a7..3a3fbd8c8 100644 --- a/tests/fixtures/validation/leaky_egress/openenv.yaml +++ b/tests/fixtures/validation/leaky_egress/openenv.yaml @@ -1,4 +1,5 @@ spec_version: 1 +version: 0.1.0 name: leaky-egress type: env runtime: python diff --git a/tests/fixtures/validation/leaky_observation/openenv.yaml b/tests/fixtures/validation/leaky_observation/openenv.yaml index b09ad6a15..ea2e2982e 100644 --- a/tests/fixtures/validation/leaky_observation/openenv.yaml +++ b/tests/fixtures/validation/leaky_observation/openenv.yaml @@ -1,4 +1,5 @@ spec_version: 1 +version: 0.1.0 name: leaky-observation type: env runtime: python diff --git a/tests/fixtures/validation/no_oracle/openenv.yaml b/tests/fixtures/validation/no_oracle/openenv.yaml index 4d0a7d596..b08c8bee3 100644 --- a/tests/fixtures/validation/no_oracle/openenv.yaml +++ b/tests/fixtures/validation/no_oracle/openenv.yaml @@ -1,4 +1,5 @@ spec_version: 1 +version: 0.1.0 name: no-oracle type: env runtime: python diff --git a/tests/fixtures/validation/nondeterministic/openenv.yaml b/tests/fixtures/validation/nondeterministic/openenv.yaml index ffe25499f..c0647fb0c 100644 --- a/tests/fixtures/validation/nondeterministic/openenv.yaml +++ b/tests/fixtures/validation/nondeterministic/openenv.yaml @@ -1,4 +1,5 @@ spec_version: 1 +version: 0.1.0 name: nondeterministic type: env runtime: python diff --git a/tests/fixtures/validation/served_min_pass/openenv.yaml b/tests/fixtures/validation/served_min_pass/openenv.yaml index 8ad02becb..3643b91a2 100644 --- a/tests/fixtures/validation/served_min_pass/openenv.yaml +++ b/tests/fixtures/validation/served_min_pass/openenv.yaml @@ -1,4 +1,5 @@ spec_version: 1 +version: 0.1.0 name: served-min-pass type: env runtime: python diff --git a/tests/fixtures/validation/unpinned_judge/openenv.yaml b/tests/fixtures/validation/unpinned_judge/openenv.yaml index 8cd7b5680..a7d4cf296 100644 --- a/tests/fixtures/validation/unpinned_judge/openenv.yaml +++ b/tests/fixtures/validation/unpinned_judge/openenv.yaml @@ -1,4 +1,5 @@ spec_version: 1 +version: 0.1.0 name: unpinned-judge type: env runtime: python diff --git a/tests/fixtures/validation/unrecognized_package/README.md b/tests/fixtures/validation/unrecognized_package/README.md new file mode 100644 index 000000000..717f86268 --- /dev/null +++ b/tests/fixtures/validation/unrecognized_package/README.md @@ -0,0 +1,2 @@ +A directory with no well-known package file: `openenv validate` must refuse it +as unrecognized (exit 2), never guess a format. diff --git a/tests/test_validation/test_checkpoints.py b/tests/test_validation/test_checkpoints.py index fb757731b..26b974fd5 100644 --- a/tests/test_validation/test_checkpoints.py +++ b/tests/test_validation/test_checkpoints.py @@ -5,16 +5,23 @@ procedural. """ +from pathlib import Path + import pytest from conftest import ( EXPECTED_POLICY, + FIXTURES, INVALID_MANIFEST_FIXTURES, load_fixture_manifest, VALID_MANIFEST_FIXTURES, ) from openenv.validation.manifest import NormalizedManifest from openenv.validation.policy import load_policy +from openenv.validation.report import ValidationReport from pydantic import ValidationError +from typer.testing import CliRunner + +REPO_ROOT = Path(__file__).parent.parent.parent def test_checkpoint_0_contracts_are_executable(): @@ -34,3 +41,55 @@ def test_checkpoint_0_contracts_are_executable(): policy = load_policy("v1") assert {e.check_id for e in policy.entries} == set(EXPECTED_POLICY) + + +def test_checkpoint_1_walking_skeleton(): + """Slice 1: signature -> parse -> grade -> policy -> report -> exit code. + + The three checkpoint commands from structure.md §10, run through the CLI: + a real env exits 0 with a schema-valid report; a broken manifest exits 1 + failing static.manifest; an ambiguous package exits 2. + """ + from openenv.cli.__main__ import app + + runner = CliRunner() + + ok = runner.invoke( + app, + [ + "validate", + str(REPO_ROOT / "envs" / "echo_env"), + "--level", + "static", + "--skip-build", + "--json", + ], + ) + assert ok.exit_code == 0, ok.output + report = ValidationReport.model_validate_json(ok.output) + assert report.verdict.value == "pass" + + broken = runner.invoke( + app, + [ + "validate", + str(FIXTURES / "broken_manifest"), + "--level", + "static", + "--skip-build", + ], + ) + assert broken.exit_code == 1, broken.output + assert "static.manifest" in broken.output + + unrecognized = runner.invoke( + app, + [ + "validate", + str(FIXTURES / "unrecognized_package"), + "--level", + "static", + "--skip-build", + ], + ) + assert unrecognized.exit_code == 2, unrecognized.output diff --git a/tests/test_validation/test_cli_validate.py b/tests/test_validation/test_cli_validate.py new file mode 100644 index 000000000..7bc1c67f7 --- /dev/null +++ b/tests/test_validation/test_cli_validate.py @@ -0,0 +1,89 @@ +"""CLI contract: one command, exit codes 0/1/2/3, schema-valid JSON reports.""" + +import pytest +from conftest import FIXTURES +from openenv.cli.__main__ import app +from openenv.validation.report import ValidationReport +from typer.testing import CliRunner + +runner = CliRunner() + + +def _validate(*args): + return runner.invoke(app, ["validate", *args]) + + +def test_valid_package_exits_zero(): + result = _validate( + str(FIXTURES / "served_min_pass"), "--level", "static", "--skip-build" + ) + assert result.exit_code == 0, result.output + assert "Verdict: PASS" in result.output + + +def test_json_report_is_schema_valid(): + result = _validate( + str(FIXTURES / "served_min_pass"), "--level", "static", "--skip-build", "--json" + ) + assert result.exit_code == 0, result.output + report = ValidationReport.model_validate_json(result.output) + assert report.verdict.value == "pass" + + +def test_output_writes_the_json_report(tmp_path): + out = tmp_path / "report.json" + result = _validate( + str(FIXTURES / "served_min_pass"), + "--level", + "static", + "--skip-build", + "--output", + str(out), + ) + assert result.exit_code == 0, result.output + ValidationReport.model_validate_json(out.read_text()) + + +def test_failing_manifest_exits_one(): + result = _validate( + str(FIXTURES / "broken_manifest"), "--level", "static", "--skip-build" + ) + assert result.exit_code == 1, result.output + assert "static.manifest" in result.output + + +def test_unrecognized_package_exits_two(): + result = _validate( + str(FIXTURES / "unrecognized_package"), "--level", "static", "--skip-build" + ) + assert result.exit_code == 2, result.output + assert "unrecognized" in result.output + + +def test_format_without_a_parser_exits_two(): + # Harbor packages are refused as unrecognized until the Harbor parser lands. + result = _validate( + str(FIXTURES / "harbor_task_min"), "--level", "static", "--skip-build" + ) + assert result.exit_code == 2, result.output + assert "unrecognized" in result.output + + +def test_nonexistent_path_exits_two(tmp_path): + result = _validate(str(tmp_path / "nope")) + assert result.exit_code == 2, result.output + + +def test_unknown_level_is_an_internal_error(): + result = _validate(str(FIXTURES / "served_min_pass"), "--level", "cosmic") + assert result.exit_code == 3, result.output + + +@pytest.mark.parametrize("fixture", ["served_min_pass", "broken_manifest"]) +def test_unpinned_judge_fails_from_slice_1_onward(fixture): + # Sanity anchor for the F2 note: unpinned_judge FAILs static.manifest already. + result = _validate( + str(FIXTURES / "unpinned_judge"), "--level", "static", "--skip-build" + ) + assert result.exit_code == 1 + assert "judge pin" in result.output diff --git a/tests/test_validation/test_openenv_yaml_parser.py b/tests/test_validation/test_openenv_yaml_parser.py new file mode 100644 index 000000000..29049f48e --- /dev/null +++ b/tests/test_validation/test_openenv_yaml_parser.py @@ -0,0 +1,96 @@ +"""OpenEnvYamlParser: fixture openenv.yaml files parse to their committed goldens.""" + +import pytest +from conftest import FIXTURES, load_fixture_manifest +from openenv.validation.manifest import ManifestError, NormalizedManifest +from openenv.validation.parsers.openenv_yaml import OpenEnvYamlParser +from openenv.validation.runner import default_parser_registry +from openenv.validation.signature import SignatureError + +# Served fixtures whose normalized_manifest.json is the golden parse result. +GOLDEN_FIXTURES = [ + "served_min_pass", + "empty_solution_max_reward", + "no_oracle", + "leaky_observation", + "nondeterministic", + "leaky_egress", +] + + +@pytest.mark.parametrize("name", GOLDEN_FIXTURES) +def test_parse_matches_the_committed_golden_manifest(name): + parsed = OpenEnvYamlParser().parse(FIXTURES / name) + golden = NormalizedManifest.model_validate(load_fixture_manifest(name)) + assert parsed == golden + + +def test_broken_manifest_raises_manifest_error_with_field_evidence(): + with pytest.raises(ManifestError) as exc_info: + OpenEnvYamlParser().parse(FIXTURES / "broken_manifest") + evidence = "\n".join(exc_info.value.errors) + assert "reward.range" in evidence or "strictly increasing" in evidence + assert "resources" in evidence + + +def test_unpinned_judge_raises_manifest_error_on_the_judge_pin(): + with pytest.raises(ManifestError) as exc_info: + OpenEnvYamlParser().parse(FIXTURES / "unpinned_judge") + assert any("judge pin" in error for error in exc_info.value.errors) + + +def test_missing_validation_block_has_remediation(tmp_path): + (tmp_path / "openenv.yaml").write_text("spec_version: 1\nname: bare\n") + with pytest.raises(ManifestError) as exc_info: + OpenEnvYamlParser().parse(tmp_path) + assert "validation" in exc_info.value.errors[0] + assert exc_info.value.remediation is not None + + +def test_parse_is_a_pure_read(tmp_path): + # Package code must never be imported or executed: a booby-trapped module + # alongside a valid openenv.yaml parses without tripping. + src = (FIXTURES / "served_min_pass" / "openenv.yaml").read_text() + (tmp_path / "openenv.yaml").write_text(src) + (tmp_path / "server.py").write_text( + "raise SystemExit('parser imported package code')\n" + ) + parsed = OpenEnvYamlParser().parse(tmp_path) + assert parsed.name == "served-min-pass" + + +def test_default_registry_dispatches_openenv_yaml_end_to_end(): + manifest = default_parser_registry().parse(FIXTURES / "served_min_pass") + assert manifest.name == "served-min-pass" + + +def test_default_registry_refuses_formats_without_parsers(): + # task.toml is not in the detection table until the Harbor parser lands, so + # a Harbor package is refused as unrecognized rather than half-supported. + with pytest.raises(SignatureError, match="unrecognized"): + default_parser_registry().parse(FIXTURES / "harbor_task_min") + + +def test_network_policy_parses_and_defaults_to_public(tmp_path): + src = (FIXTURES / "served_min_pass" / "openenv.yaml").read_text() + (tmp_path / "openenv.yaml").write_text( + src + + " network:\n mode: allowlist\n allowed_hosts: [api.example.com, 10.0.0.0/8]\n" + ) + parsed = OpenEnvYamlParser().parse(tmp_path) + assert parsed.network.mode == "allowlist" + assert parsed.network.allowed_hosts == ["api.example.com", "10.0.0.0/8"] + + default = OpenEnvYamlParser().parse(FIXTURES / "served_min_pass") + assert default.network.mode == "public" + assert default.network.allowed_hosts == [] + + +def test_allowed_hosts_without_allowlist_mode_is_a_manifest_error(tmp_path): + src = (FIXTURES / "served_min_pass" / "openenv.yaml").read_text() + (tmp_path / "openenv.yaml").write_text( + src + " network:\n allowed_hosts: [api.example.com]\n" + ) + with pytest.raises(ManifestError) as exc_info: + OpenEnvYamlParser().parse(tmp_path) + assert any("allowlist" in error for error in exc_info.value.errors) diff --git a/tests/test_validation/test_runner.py b/tests/test_validation/test_runner.py new file mode 100644 index 000000000..94d79be5c --- /dev/null +++ b/tests/test_validation/test_runner.py @@ -0,0 +1,80 @@ +"""run_validation: parse -> grade -> policy -> report, one report per run.""" + +import shutil + +import pytest +from conftest import FIXTURES +from openenv.validation.policy import load_policy +from openenv.validation.report import ValidationReport +from openenv.validation.runner import run_validation, source_digest +from openenv.validation.signature import SignatureError +from openenv.validation.types import CheckStatus, Lane, Level, Verdict + + +def test_valid_package_passes_static_level(): + report = run_validation( + FIXTURES / "served_min_pass", max_level=Level.STATIC, skip_build=True + ) + assert report.verdict is Verdict.PASS + assert report.lane is Lane.LOCAL + assert report.levels_run == [Level.STATIC] + assert report.manifest is not None and report.manifest.name == "served-min-pass" + by_id = {r.check_id: r for r in report.results} + assert by_id["static.manifest"].status is CheckStatus.PASS + + +def test_report_round_trips_through_its_schema(): + report = run_validation( + FIXTURES / "served_min_pass", max_level=Level.STATIC, skip_build=True + ) + assert ValidationReport.model_validate_json(report.model_dump_json()) == report + + +def test_broken_manifest_fails_static_manifest_with_evidence(): + report = run_validation( + FIXTURES / "broken_manifest", max_level=Level.STATIC, skip_build=True + ) + assert report.verdict is Verdict.FAIL + assert report.manifest is None + (result,) = report.results + assert result.check_id == "static.manifest" + assert result.status is CheckStatus.FAIL + assert result.evidence, "schema errors must surface as evidence" + + +def test_out_of_bounds_declaration_fails_static_manifest(tmp_path): + src = (FIXTURES / "served_min_pass" / "openenv.yaml").read_text() + (tmp_path / "openenv.yaml").write_text( + src.replace("floor_margin: 0.5", "floor_margin: 0.01") + ) + report = run_validation(tmp_path, max_level=Level.STATIC, skip_build=True) + assert report.verdict is Verdict.FAIL + (result,) = report.results + assert result.status is CheckStatus.FAIL + assert "floor_margin" in "\n".join(result.evidence) + + +def test_ambiguous_package_raises_signature_error(): + with pytest.raises(SignatureError, match="ambiguous"): + run_validation(FIXTURES / "ambiguous_package", max_level=Level.STATIC) + + +def test_report_embeds_the_pinned_policy_version(): + policy = load_policy("v1") + report = run_validation( + FIXTURES / "served_min_pass", + max_level=Level.STATIC, + skip_build=True, + policy=policy, + ) + assert report.policy_version == policy.policy_version + + +def test_source_digest_is_deterministic_and_content_sensitive(tmp_path): + copy = tmp_path / "pkg" + shutil.copytree(FIXTURES / "served_min_pass", copy) + first = source_digest(copy) + assert first == source_digest(copy) + assert len(first) == 64 + (copy / "extra.txt").write_text("changed\n") + assert source_digest(copy) != first diff --git a/tests/test_validation/test_signature_detection.py b/tests/test_validation/test_signature_detection.py new file mode 100644 index 000000000..698f266e6 --- /dev/null +++ b/tests/test_validation/test_signature_detection.py @@ -0,0 +1,50 @@ +"""detect_signature: exactly one well-known file of a parseable format, never a guess.""" + +import pytest +from conftest import FIXTURES +from openenv.validation.signature import ( + detect_signature, + SignatureError, + WELL_KNOWN_FILES, +) +from openenv.validation.types import SignatureKind + + +def test_openenv_yaml_detects_the_served_format(): + assert ( + detect_signature(FIXTURES / "served_min_pass") is SignatureKind.OPENENV_SERVED + ) + + +def test_the_table_lists_exactly_the_implemented_parsers(): + # Only the served-env parser exists in this build. task.toml joins with the + # Harbor parser (slice 6) and task.md with the PostTrain parser (slice F1). + assert set(WELL_KNOWN_FILES) == {SignatureKind.OPENENV_SERVED} + + +@pytest.mark.parametrize("fixture", ["harbor_task_min", "posttrain_task_min"]) +def test_formats_without_parsers_are_unrecognized_not_guessed(fixture): + # These packages are real formats, but this build cannot parse them — + # refusing as unrecognized beats claiming support. + with pytest.raises(SignatureError, match="unrecognized"): + detect_signature(FIXTURES / fixture) + + +def test_no_well_known_file_is_unrecognized(): + with pytest.raises(SignatureError, match="unrecognized"): + detect_signature(FIXTURES / "unrecognized_package") + + +def test_two_well_known_files_are_ambiguous_never_a_guess(tmp_path, monkeypatch): + # Ambiguity needs two parseable formats, which this build doesn't have yet — + # exercised against a patched table so the contract is enforced from day one. + monkeypatch.setitem(WELL_KNOWN_FILES, SignatureKind.HARBOR_TASK, "task.toml") + (tmp_path / "openenv.yaml").write_text("spec_version: 1\n") + (tmp_path / "task.toml").write_text('schema_version = "1.1"\n') + with pytest.raises(SignatureError, match="ambiguous"): + detect_signature(tmp_path) + + +def test_missing_directory_is_a_signature_error(tmp_path): + with pytest.raises(SignatureError, match="not a package directory"): + detect_signature(tmp_path / "does_not_exist")