From e0d50d096f69c0725525a3ea14d0ad4e387a57ed Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 24 Aug 2026 15:20:21 +0000 Subject: [PATCH 01/14] Add RFC 008 slice 1: walking skeleton for openenv validate Wire detect_signature, OpenEnvYamlParser, static.manifest, run_validation, policy application, and the 0/1/2/3 CLI exit-code contract so later slices only add parsers, graders, or providers. Co-authored-by: Zach Wentz --- envs/echo_env/openenv.yaml | 17 ++ src/openenv/cli/__main__.py | 6 +- src/openenv/cli/commands/validate.py | 223 ++++++++++-------- src/openenv/validation/__init__.py | 9 +- .../validation/graders/static/__init__.py | 5 + .../validation/graders/static/manifest.py | 80 +++++++ src/openenv/validation/manifest.py | 22 ++ src/openenv/validation/parsers/__init__.py | 18 +- .../validation/parsers/openenv_yaml.py | 95 ++++++++ src/openenv/validation/report.py | 20 ++ src/openenv/validation/runner.py | 171 ++++++++++++++ src/openenv/validation/signature.py | 75 +++++- .../validation/ambiguous_package/README.md | 3 + .../validation/ambiguous_package/openenv.yaml | 6 + .../validation/ambiguous_package/task.toml | 6 + .../validation/no_oracle/openenv.yaml | 1 + .../validation/served_min_pass/openenv.yaml | 1 + .../validation/unpinned_judge/openenv.yaml | 1 + .../validation/unrecognized_package/README.md | 2 + tests/test_cli/test_validate.py | 111 +++++---- tests/test_validation/test_cli_validate.py | 94 ++++++++ .../test_openenv_yaml_parser.py | 96 ++++++++ tests/test_validation/test_runner.py | 80 +++++++ .../test_signature_detection.py | 51 ++++ 24 files changed, 1039 insertions(+), 154 deletions(-) create mode 100644 src/openenv/validation/graders/static/__init__.py create mode 100644 src/openenv/validation/graders/static/manifest.py create mode 100644 src/openenv/validation/parsers/openenv_yaml.py create mode 100644 src/openenv/validation/runner.py create mode 100644 tests/fixtures/validation/ambiguous_package/README.md create mode 100644 tests/fixtures/validation/ambiguous_package/openenv.yaml create mode 100644 tests/fixtures/validation/ambiguous_package/task.toml create mode 100644 tests/fixtures/validation/unrecognized_package/README.md create mode 100644 tests/test_validation/test_cli_validate.py create mode 100644 tests/test_validation/test_openenv_yaml_parser.py create mode 100644 tests/test_validation/test_runner.py create mode 100644 tests/test_validation/test_signature_detection.py diff --git a/envs/echo_env/openenv.yaml b/envs/echo_env/openenv.yaml index 6f178906da..34968ac1dc 100644 --- a/envs/echo_env/openenv.yaml +++ b/envs/echo_env/openenv.yaml @@ -1,6 +1,23 @@ spec_version: 1 name: echo_env +version: 0.1.0 type: space runtime: fastapi app: server.app:app port: 8000 +validation: + reward: + range: [0.0, 1.0] + oracle_tolerance: 0.0 + floor_margin: 0.5 + resources: + cpu: 1.0 + memory_mb: 1024 + disk_mb: 512 + episode_timeout_s: 60.0 + capabilities: + verifier: + kind: reward_channel + declared_tools: [echo_message, echo_with_length] + types: + tags: [demo] diff --git a/src/openenv/cli/__main__.py b/src/openenv/cli/__main__.py index 42ebbdbe97..1fcef4241a 100644 --- a/src/openenv/cli/__main__.py +++ b/src/openenv/cli/__main__.py @@ -37,9 +37,9 @@ app.command(name="build", help="Build Docker images for OpenEnv environments")( build.build ) -app.command( - name="validate", help="Validate environment structure and deployment readiness" -)(validate.validate) +app.command(name="validate", help="Validate a package against the OpenEnv quality bar")( + validate.validate +) app.command( name="push", help="Push an OpenEnv environment to Hugging Face Spaces or custom registry", diff --git a/src/openenv/cli/commands/validate.py b/src/openenv/cli/commands/validate.py index b3926a7698..04555d9ae1 100644 --- a/src/openenv/cli/commands/validate.py +++ b/src/openenv/cli/commands/validate.py @@ -3,8 +3,9 @@ """ OpenEnv validate command. -This module provides the 'openenv validate' command to check if environments -are properly configured for multi-mode deployment. +Local packages run through the RFC 008 validation pipeline (signature → parse → +grade → severity policy → report). Running servers are probed via --url until the +runtime graders land. """ import json @@ -12,27 +13,65 @@ from typing import Annotated import typer -from openenv.cli._validation import ( - build_local_validation_json_report, - format_validation_report, - get_deployment_modes, - validate_multi_mode_deployment, - validate_running_environment, +from openenv.cli._validation import validate_running_environment +from openenv.validation import ( + Level, + load_policy, + PolicyError, + run_validation, + SignatureError, + UnsupportedPackageError, + ValidationReport, + Verdict, + write_report, ) +# Exit-code contract (RFC 008): +EXIT_PASS = 0 # verdict PASS or WARN +EXIT_FAIL = 1 # verdict FAIL +EXIT_UNSUPPORTED = 2 # SignatureError / UnsupportedPackageError +EXIT_INTERNAL = 3 # internal error + +_LEVELS = { + "static": Level.STATIC, + "runtime": Level.RUNTIME, + "semantic": Level.SEMANTIC, +} + + def _looks_like_url(value: str) -> bool: """Return True when the value appears to be a URL target.""" candidate = value.strip().lower() return candidate.startswith("http://") or candidate.startswith("https://") +def _render_report(report: ValidationReport) -> str: + """Human-readable report summary.""" + lines = [ + f"Validation report for {report.target} (signature: {report.signature.value})", + f" policy {report.policy_version} · levels run: " + + ", ".join(level.name.lower() for level in report.levels_run), + ] + for result in report.results: + lines.append( + f" {result.status.value.upper():5s} {result.check_id} ({result.duration_s:.2f}s)" + ) + if result.status.value in ("fail", "error", "skip"): + for line in result.evidence: + lines.append(f" {line}") + if result.remediation: + lines.append(f" remediation: {result.remediation}") + lines.append(f"Verdict: {report.verdict.value.upper()}") + return "\n".join(lines) + + def validate( target: Annotated[ str | None, typer.Argument( help=( - "Path to the environment directory (default: current directory) " + "Path to the package directory (default: current directory) " "or a running OpenEnv URL (http://... or https://...)" ), ), @@ -44,54 +83,67 @@ def validate( help="Validate a running OpenEnv server by base URL (e.g. http://localhost:8000)", ), ] = None, - json_output: Annotated[ + level: Annotated[ + str, + typer.Option( + "--level", + help="Validation level ceiling: static, runtime, or semantic", + ), + ] = "semantic", + skip_build: Annotated[ bool, typer.Option( - "--json", - help="Output local validation report as JSON (runtime validation is JSON by default)", + "--skip-build", + help="Skip the image build; build-dependent checks are SKIPped with a reason", ), ] = False, + policy_version: Annotated[ + str, + typer.Option("--policy", help="Severity policy version to apply"), + ] = "v1", + json_output: Annotated[ + bool, + typer.Option("--json", help="Print the validation report as JSON"), + ] = False, + output: Annotated[ + Path | None, + typer.Option("--output", help="Write the JSON report to a file"), + ] = None, timeout: Annotated[ float, typer.Option( "--timeout", - help="HTTP timeout in seconds for runtime validation", + help="HTTP timeout in seconds for --url runtime validation", min=0.1, ), ] = 5.0, - verbose: Annotated[ - bool, typer.Option("--verbose", "-v", help="Show detailed information") - ] = False, ) -> None: """ - Validate local environments and running OpenEnv servers. + Validate a local package or a running OpenEnv server. - Local validation checks if an environment is properly configured with: - - Required files (pyproject.toml, openenv.yaml, server/app.py, etc.) - - Docker deployment support - - uv run server capability - - python -m module execution + Local validation detects the package format by its well-known file (the + formats this build can parse; currently `openenv.yaml`), parses it into the + normalized manifest, runs the applicable graders up to the requested level, + applies the severity policy, and emits a report. - Runtime validation checks if a live OpenEnv server conforms to the - versioned runtime API contract and returns a criteria-based JSON report. + Exit codes: 0 pass/warn · 1 fail · 2 unrecognized/unsupported package · 3 + internal error. Examples: - ```bash - # Validate current directory (recommended) - $ cd my_env - $ openenv validate + ```bash + # Validate the current directory up to the semantic level + openenv validate - # Validate a running environment and return JSON criteria - $ openenv validate --url http://localhost:8000 - $ openenv validate https://my-env.hf.space + # Fast inner loop: static checks only, no image build + openenv validate envs/echo_env --level static --skip-build - # Validate with detailed output - $ openenv validate --verbose + # Machine-readable report + openenv validate envs/echo_env --json - # Validate specific environment - $ openenv validate envs/echo_env - ``` + # Probe a running server (legacy runtime probe) + openenv validate --url http://localhost:8000 + ``` """ runtime_target = url if ( @@ -103,7 +155,7 @@ def validate( "Error: Cannot combine a local path argument with --url runtime validation", err=True, ) - raise typer.Exit(1) + raise typer.Exit(EXIT_FAIL) if target is not None and _looks_like_url(target): if runtime_target is not None and runtime_target != target: @@ -111,7 +163,7 @@ def validate( "Error: Conflicting runtime targets provided via argument and --url", err=True, ) - raise typer.Exit(1) + raise typer.Exit(EXIT_FAIL) runtime_target = target if runtime_target is not None: @@ -119,79 +171,44 @@ def validate( report = validate_running_environment(runtime_target, timeout_s=timeout) except ValueError as exc: typer.echo(f"Error: {exc}", err=True) - raise typer.Exit(1) from exc + raise typer.Exit(EXIT_FAIL) from exc typer.echo(json.dumps(report, indent=2)) if not report.get("passed", False): - raise typer.Exit(1) + raise typer.Exit(EXIT_FAIL) return - # Determine environment path (default to current directory) - if target is None: - env_path_obj = Path.cwd() - else: - env_path_obj = Path(target) - - if not env_path_obj.exists(): - typer.echo(f"Error: Path does not exist: {env_path_obj}", err=True) - raise typer.Exit(1) - - if not env_path_obj.is_dir(): - typer.echo(f"Error: Path is not a directory: {env_path_obj}", err=True) - raise typer.Exit(1) - - # Check for openenv.yaml to confirm this is an environment directory - openenv_yaml = env_path_obj / "openenv.yaml" - if not openenv_yaml.exists(): + if level not in _LEVELS: typer.echo( - f"Error: Not an OpenEnv environment directory (missing openenv.yaml): {env_path_obj}", + f"Error: unknown level {level!r}; expected one of {sorted(_LEVELS)}", err=True, ) - typer.echo( - "Hint: Run this command from the environment root directory or specify the path", - err=True, + raise typer.Exit(EXIT_INTERNAL) + + package_root = Path(target) if target is not None else Path.cwd() + if not package_root.is_dir(): + typer.echo(f"Error: not a package directory: {package_root}", err=True) + raise typer.Exit(EXIT_UNSUPPORTED) + + try: + validation_report = run_validation( + package_root, + max_level=_LEVELS[level], + skip_build=skip_build, + policy=load_policy(policy_version), ) - raise typer.Exit(1) - - env_name = env_path_obj.name - if env_name.endswith("_env"): - base_name = env_name[:-4] - else: - base_name = env_name - - # Run validation - is_valid, issues = validate_multi_mode_deployment(env_path_obj) - modes = get_deployment_modes(env_path_obj) - + except (SignatureError, UnsupportedPackageError) as exc: + typer.echo(f"Error: {exc}", err=True) + raise typer.Exit(EXIT_UNSUPPORTED) from exc + except PolicyError as exc: + typer.echo(f"Internal error: {exc}", err=True) + raise typer.Exit(EXIT_INTERNAL) from exc + + report_json = write_report(validation_report, output) if json_output: - report = build_local_validation_json_report( - env_name=base_name, - env_path=env_path_obj, - is_valid=is_valid, - issues=issues, - deployment_modes=modes if verbose else None, - ) - typer.echo(json.dumps(report, indent=2)) - if not is_valid: - raise typer.Exit(1) - return + typer.echo(report_json) + else: + typer.echo(_render_report(validation_report)) - # Show validation report - report = format_validation_report(base_name, is_valid, issues) - typer.echo(report) - - # Show deployment modes if verbose - if verbose: - typer.echo("\nSupported deployment modes:") - for mode, supported in modes.items(): - status = "[YES]" if supported else "[NO]" - typer.echo(f" {status} {mode}") - - if is_valid: - typer.echo("\nUsage examples:") - typer.echo(f" cd {env_path_obj.name} && uv run server") - typer.echo(f" cd {env_path_obj.name} && openenv build") - typer.echo(f" cd {env_path_obj.name} && openenv push") - - if not is_valid: - raise typer.Exit(1) + if validation_report.verdict is Verdict.FAIL: + raise typer.Exit(EXIT_FAIL) diff --git a/src/openenv/validation/__init__.py b/src/openenv/validation/__init__.py index 2e8bd3e4a1..eb577c78b3 100644 --- a/src/openenv/validation/__init__.py +++ b/src/openenv/validation/__init__.py @@ -4,6 +4,7 @@ from .manifest import ( CapabilitiesSpec, JudgePin, + ManifestError, NetworkPolicy, NormalizedManifest, OracleDeclaration, @@ -23,8 +24,10 @@ SeverityPolicy, ) from .providers import ExecResult, RunningSubject, ValidationProvider -from .report import CheckResult, ValidationReport +from .report import CheckResult, ValidationReport, write_report +from .runner import run_validation from .signature import ( + detect_signature, SignatureError, UNSUPPORTED_CATEGORIES, UnsupportedPackageError, @@ -54,6 +57,7 @@ "JudgePin", "Lane", "Level", + "ManifestError", "NetworkPolicy", "NormalizedManifest", "OracleDeclaration", @@ -78,5 +82,8 @@ "Verdict", "VerifierBinding", "apply_policy", + "detect_signature", "load_policy", + "run_validation", + "write_report", ] diff --git a/src/openenv/validation/graders/static/__init__.py b/src/openenv/validation/graders/static/__init__.py new file mode 100644 index 0000000000..c4472ae049 --- /dev/null +++ b/src/openenv/validation/graders/static/__init__.py @@ -0,0 +1,5 @@ +"""Static-level (L1) graders.""" + +from .manifest import StaticManifestGrader + +__all__ = ["StaticManifestGrader"] diff --git a/src/openenv/validation/graders/static/manifest.py b/src/openenv/validation/graders/static/manifest.py new file mode 100644 index 0000000000..6224e33fdb --- /dev/null +++ b/src/openenv/validation/graders/static/manifest.py @@ -0,0 +1,80 @@ +"""The `static.manifest` grader: the manifest is schema-valid and within policy bounds.""" + +import time + +from ...manifest import NormalizedManifest +from ...policy import DeclarationBounds +from ...report import CheckResult +from ...types import CheckStatus, Level, ProviderCapability + + +class StaticManifestGrader: + """ + Checks the parsed manifest's declared tolerances against the policy bounds. + + Schema validity itself is established before this grader runs (an unparseable + manifest never reaches grading — the runner records the `static.manifest` FAIL + directly from the parser's [`~openenv.validation.manifest.ManifestError`]). What + remains to grade is that author-declared values sit within what the severity + policy allows an author to declare. + """ + + check_id = "static.manifest" + level = Level.STATIC + requires_capabilities: frozenset[str] = frozenset() + requires_provider: frozenset[ProviderCapability] = frozenset() + depends_on: tuple[str, ...] = () + + def __init__(self, bounds: DeclarationBounds): + self._bounds = bounds + + def applies_to(self, manifest: NormalizedManifest) -> bool: + return True + + def run(self, subject) -> CheckResult: + started = time.monotonic() + reward = subject.manifest.reward + bounds = self._bounds + problems = [] + if reward.oracle_tolerance > bounds.max_oracle_tolerance: + problems.append( + f"declared oracle_tolerance {reward.oracle_tolerance} exceeds the " + f"policy maximum {bounds.max_oracle_tolerance}" + ) + if reward.floor_margin < bounds.min_floor_margin: + problems.append( + f"declared floor_margin {reward.floor_margin} is below the " + f"policy minimum {bounds.min_floor_margin}" + ) + if ( + reward.variance_tolerance is not None + and reward.variance_tolerance > bounds.max_variance_tolerance + ): + problems.append( + f"declared variance_tolerance {reward.variance_tolerance} exceeds " + f"the policy maximum {bounds.max_variance_tolerance}" + ) + timeout = subject.manifest.resources.episode_timeout_s + if timeout > bounds.max_episode_timeout_s: + problems.append( + f"declared episode_timeout_s {timeout} exceeds the " + f"policy maximum {bounds.max_episode_timeout_s}" + ) + return CheckResult( + check_id=self.check_id, + status=CheckStatus.FAIL if problems else CheckStatus.PASS, + measured={ + "oracle_tolerance": reward.oracle_tolerance, + "floor_margin": reward.floor_margin, + "variance_tolerance": reward.variance_tolerance, + "episode_timeout_s": timeout, + "bounds": bounds.model_dump(), + }, + evidence=problems or ["manifest is schema-valid and within policy bounds"], + remediation=( + "declare tolerances within the severity policy's bounds" + if problems + else None + ), + duration_s=time.monotonic() - started, + ) diff --git a/src/openenv/validation/manifest.py b/src/openenv/validation/manifest.py index 3c31abbdf1..2e6adbc2b8 100644 --- a/src/openenv/validation/manifest.py +++ b/src/openenv/validation/manifest.py @@ -12,6 +12,28 @@ from .types import SignatureKind +class ManifestError(Exception): + """ + A package's declarations do not produce a valid normalized manifest. + + Raised by parsers when the format-specific source (e.g. `openenv.yaml`) is + readable but its content fails the manifest schema. The runner surfaces this as a + `static.manifest` FAIL in the report — a graded failure with remediation, not a + crash and not exit code 2. + + Attributes: + errors (`list[str]`): + Human-readable schema violations, one per finding. + remediation (`str`, *optional*): + What the author changes to go green. + """ + + def __init__(self, errors: list[str], *, remediation: str | None = None): + super().__init__("; ".join(errors)) + self.errors = errors + self.remediation = remediation + + class RewardDeclaration(BaseModel): """ Author-declared reward contract. Graders normalize measurements against it. diff --git a/src/openenv/validation/parsers/__init__.py b/src/openenv/validation/parsers/__init__.py index 0f76929196..a8b9fb3215 100644 --- a/src/openenv/validation/parsers/__init__.py +++ b/src/openenv/validation/parsers/__init__.py @@ -9,7 +9,7 @@ from typing import Protocol, runtime_checkable from ..manifest import NormalizedManifest -from ..signature import UnsupportedPackageError +from ..signature import detect_signature, UnsupportedPackageError from ..types import SignatureKind @@ -67,3 +67,19 @@ def parser_for(self, signature: SignatureKind) -> Parser: f"no parser registered for signature {signature.value!r}", ) return parser + + def parse(self, package_root: Path) -> NormalizedManifest: + """ + Detect the package signature and dispatch to its parser. + + This is the only place a [`~openenv.validation.types.SignatureKind`] selects + behavior. + + Args: + package_root (`Path`): + The package directory to detect and parse. + + Returns: + [`~openenv.validation.manifest.NormalizedManifest`]: the normalized manifest. + """ + return self.parser_for(detect_signature(package_root)).parse(package_root) diff --git a/src/openenv/validation/parsers/openenv_yaml.py b/src/openenv/validation/parsers/openenv_yaml.py new file mode 100644 index 0000000000..c983ce0051 --- /dev/null +++ b/src/openenv/validation/parsers/openenv_yaml.py @@ -0,0 +1,95 @@ +"""Parser for the served OpenEnv format (`openenv.yaml`).""" + +from pathlib import Path + +import yaml +from pydantic import ValidationError + +from ..manifest import ManifestError, NormalizedManifest +from ..types import SignatureKind + +VALIDATION_BLOCK_REMEDIATION = ( + "add a `validation:` block to openenv.yaml declaring reward (range, " + "oracle_tolerance, floor_margin), resources (cpu, memory_mb, disk_mb, " + "episode_timeout_s), capabilities (verifier, oracle, ...), and types (tags); " + "see the manifest schema at src/openenv/validation/schemas/manifest.schema.json" +) + + +def _format_validation_error(exc: ValidationError) -> list[str]: + """Render pydantic errors as `field: message` evidence lines.""" + lines = [] + for err in exc.errors(): + loc = ".".join(str(part) for part in err["loc"]) or "manifest" + lines.append(f"{loc}: {err['msg']}") + return lines + + +class OpenEnvYamlParser: + """ + Parses `openenv.yaml` into the normalized manifest. + + A parse is a pure read: it never imports or executes package code. Top-level + `name`/`version` identify the environment; everything validation-specific lives + under the `validation:` block, which maps onto + [`~openenv.validation.manifest.NormalizedManifest`] sections. + """ + + signature = SignatureKind.OPENENV_SERVED + + def parse(self, package_root: Path) -> NormalizedManifest: + """ + Parse a served-environment package. + + Args: + package_root (`Path`): + Directory containing `openenv.yaml`. + + Returns: + [`~openenv.validation.manifest.NormalizedManifest`]: the normalized manifest. + """ + source = Path(package_root) / "openenv.yaml" + try: + raw = yaml.safe_load(source.read_text(encoding="utf-8")) + except yaml.YAMLError as exc: + raise ManifestError([f"openenv.yaml is not valid YAML: {exc}"]) from exc + if not isinstance(raw, dict): + raise ManifestError(["openenv.yaml must be a YAML mapping"]) + + validation = raw.get("validation") + if validation is None: + raise ManifestError( + ["openenv.yaml has no `validation:` block"], + remediation=VALIDATION_BLOCK_REMEDIATION, + ) + if not isinstance(validation, dict): + raise ManifestError(["`validation:` must be a mapping"]) + + data: dict = { + "manifest_schema_version": "1", + "signature": SignatureKind.OPENENV_SERVED, + "version": raw.get("version"), + "judge": validation.get("judge"), + "task_distribution": validation.get("task_distribution"), + } + # Omit absent required sections so schema errors read as "field required" + # rather than "not a valid dictionary". Omit absent `network` so the + # manifest default (mode public) applies. + for key, value in ( + ("name", raw.get("name")), + ("reward", validation.get("reward")), + ("resources", validation.get("resources")), + ("capabilities", validation.get("capabilities")), + ("types", validation.get("types")), + ("network", validation.get("network")), + ): + if value is not None: + data[key] = value + + try: + return NormalizedManifest.model_validate(data) + except ValidationError as exc: + raise ManifestError( + _format_validation_error(exc), + remediation=VALIDATION_BLOCK_REMEDIATION, + ) from exc diff --git a/src/openenv/validation/report.py b/src/openenv/validation/report.py index 08c098e5cb..3fc1d0ad19 100644 --- a/src/openenv/validation/report.py +++ b/src/openenv/validation/report.py @@ -6,6 +6,7 @@ ids are reserved in this schema so operator reports and local reports share it. """ +from pathlib import Path from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field @@ -68,3 +69,22 @@ class ValidationReport(BaseModel): levels_run: list[Level] results: list[CheckResult] verdict: Verdict + + +def write_report(report: ValidationReport, path: Path | None = None) -> str: + """ + Serialize a validation report to schema-versioned JSON. + + Args: + report ([`~openenv.validation.report.ValidationReport`]): + The completed report. + path (`Path`, *optional*): + When set, the JSON is also written to this file. + + Returns: + `str`: the JSON payload, without a trailing newline. + """ + payload = report.model_dump_json(indent=2) + if path is not None: + path.write_text(payload + "\n") + return payload diff --git a/src/openenv/validation/runner.py b/src/openenv/validation/runner.py new file mode 100644 index 0000000000..a1ada20645 --- /dev/null +++ b/src/openenv/validation/runner.py @@ -0,0 +1,171 @@ +"""Validation orchestration: parse → grade → apply policy → report. + +Levels run in order and accumulate into one report per run — a level with a +policy-fail finding still lets later levels run where dependencies permit, so an +author gets maximum information per run. +""" + +import hashlib +import tempfile +import time +from pathlib import Path + +from .graders import GraderRegistry, Subject +from .graders.static import StaticManifestGrader +from .manifest import ManifestError, NormalizedManifest +from .parsers import ParserRegistry +from .parsers.openenv_yaml import OpenEnvYamlParser +from .policy import apply_policy, load_policy, SeverityPolicy +from .providers import ValidationProvider +from .report import CheckResult, ValidationReport +from .signature import detect_signature +from .types import CheckStatus, Lane, Level + +REPORT_SCHEMA_VERSION = "1" + +_DIGEST_EXCLUDED_DIRS = {".git", "__pycache__", ".venv", ".worktrees", "outputs"} + + +def source_digest(package_root: Path) -> str: + """ + Deterministic sha256 over the package tree (relative paths + file contents). + + Args: + package_root (`Path`): + The package directory. + + Returns: + `str`: a 64-character hex digest. + """ + digest = hashlib.sha256() + files = sorted( + path + for path in package_root.rglob("*") + if path.is_file() + and not any( + part in _DIGEST_EXCLUDED_DIRS + for part in path.relative_to(package_root).parts + ) + ) + for path in files: + digest.update(str(path.relative_to(package_root)).encode()) + digest.update(b"\0") + digest.update(path.read_bytes()) + digest.update(b"\0") + return digest.hexdigest() + + +def default_parser_registry() -> ParserRegistry: + """The parsers shipped in this build.""" + registry = ParserRegistry() + registry.register(OpenEnvYamlParser()) + return registry + + +def default_grader_registry(policy: SeverityPolicy) -> GraderRegistry: + """The core graders shipped in this build, configured against a policy.""" + registry = GraderRegistry() + registry.register(StaticManifestGrader(policy.bounds)) + return registry + + +def _run_grader(grader, subject: Subject) -> CheckResult: + """Run one grader; a crash is an ERROR result (fails closed), never an abort.""" + started = time.monotonic() + try: + return grader.run(subject) + except Exception as exc: # noqa: BLE001 — grader crash is evidence, not fatal + return CheckResult( + check_id=grader.check_id, + status=CheckStatus.ERROR, + evidence=[f"grader crashed: {exc!r}"], + duration_s=time.monotonic() - started, + ) + + +def run_validation( + target: Path, + *, + max_level: Level = Level.SEMANTIC, + provider: ValidationProvider | None = None, + skip_build: bool = False, + policy: SeverityPolicy | None = None, +) -> ValidationReport: + """ + Validate a package end to end and return the report. + + Raises [`~openenv.validation.signature.SignatureError`] for ambiguous or + unrecognized packages and + [`~openenv.validation.signature.UnsupportedPackageError`] for + recognized-but-unsupported ones (CLI exit code 2). A package whose declarations + fail the manifest schema yields a normal report with a `static.manifest` FAIL + (exit code 1). + + Args: + target (`Path`): + The package directory. + max_level ([`~openenv.validation.types.Level`], *optional*, defaults to `Level.SEMANTIC`): + Level ceiling; graders above it are not selected. + provider ([`~openenv.validation.providers.ValidationProvider`], *optional*): + Sandbox provider for runtime+ levels. Unused until the runtime phase + lands; `None` selects the default provider then. + skip_build (`bool`, *optional*, defaults to `False`): + Skip the image build; build-dependent checks SKIP with a reason. + Slice 1 has no build-dependent graders, so this only records + `image_ref=None` on the subject. + policy ([`~openenv.validation.policy.SeverityPolicy`], *optional*): + Severity policy; `None` loads the committed default version. + + Returns: + [`~openenv.validation.report.ValidationReport`]: the completed report. + """ + del provider, skip_build # reserved: runtime build + provider selection + target = Path(target) + policy = policy or load_policy() + signature = detect_signature(target) + + parser = default_parser_registry().parser_for(signature) + manifest: NormalizedManifest | None = None + results: list[CheckResult] = [] + levels_run: list[Level] = [Level.STATIC] + + parse_started = time.monotonic() + try: + manifest = parser.parse(target) + except ManifestError as exc: + results.append( + CheckResult( + check_id="static.manifest", + status=CheckStatus.FAIL, + measured={"schema_errors": len(exc.errors)}, + evidence=exc.errors, + remediation=exc.remediation, + duration_s=time.monotonic() - parse_started, + ) + ) + + if manifest is not None: + graders = default_grader_registry(policy).select(manifest, max_level) + with tempfile.TemporaryDirectory(prefix="openenv-validate-") as outputs: + subject = Subject( + root=target, + manifest=manifest, + image_ref=None, + running=None, + outputs_dir=Path(outputs), + ) + results.extend(_run_grader(grader, subject) for grader in graders) + levels_run = sorted({Level.STATIC, *(g.level for g in graders)}) + + return ValidationReport( + report_schema_version=REPORT_SCHEMA_VERSION, + target=str(target), + source_digest=source_digest(target), + signature=signature, + manifest=manifest, + policy_version=policy.policy_version, + lane=Lane.LOCAL, + levels_run=levels_run, + results=results, + verdict=apply_policy(results, policy, Lane.LOCAL), + ) diff --git a/src/openenv/validation/signature.py b/src/openenv/validation/signature.py index b93825429d..400e7ff056 100644 --- a/src/openenv/validation/signature.py +++ b/src/openenv/validation/signature.py @@ -1,9 +1,13 @@ -"""Well-known files and unsupported-package errors.""" +"""Well-known files, signature detection, and unsupported-package errors.""" + +from pathlib import Path from .types import SignatureKind -WELL_KNOWN_FILES: dict[SignatureKind, str] = {} -"""Formats this build can parse. Empty until a parser is registered. Values are the enum filenames.""" +WELL_KNOWN_FILES: dict[SignatureKind, str] = { + SignatureKind.OPENENV_SERVED: "openenv.yaml", +} +"""Formats this build can parse. Values are the enum filenames.""" UNSUPPORTED_CATEGORIES: dict[str, str] = { "hosted-verifier": ( @@ -51,3 +55,68 @@ def __init__(self, category: str, reason: str): super().__init__(f"unsupported package ({category}): {reason}") self.category = category self.reason = reason + + +def _has_yaml_frontmatter(path: Path) -> bool: + """Return True when `task.md` starts with YAML frontmatter (`---`).""" + try: + return path.read_text(encoding="utf-8").lstrip().startswith("---") + except OSError: + return False + + +def _file_present(package_root: Path, kind: SignatureKind) -> bool: + """Return True when this signature's well-known file is present and countable.""" + path = package_root / kind.value + if not path.is_file(): + return False + if kind is SignatureKind.POSTTRAIN_TASK: + return _has_yaml_frontmatter(path) + return True + + +def detect_signature(package_root: Path) -> SignatureKind: + """ + Detect a package's format from its well-known file. Never a guess. + + Only formats with implemented parsers are returned. A package carrying two or + more well-known files of any named format is refused as ambiguous — even when + only one of those formats has a parser in this build — because choosing the + parseable one would be a guess. Zero implemented-parser matches is unrecognized. + + Args: + package_root (`Path`): + Directory to inspect. + + Returns: + [`~openenv.validation.types.SignatureKind`]: the single matching signature. + + Raises: + [`~openenv.validation.signature.SignatureError`]: + The path is not a directory, zero implemented parsers match, or two or + more named-format well-known files are present. + """ + package_root = Path(package_root) + if not package_root.is_dir(): + raise SignatureError(f"not a package directory: {package_root}") + + named_matches = [ + kind for kind in SignatureKind if _file_present(package_root, kind) + ] + if len(named_matches) > 1: + found = ", ".join(sorted(kind.value for kind in named_matches)) + raise SignatureError( + f"ambiguous package: {package_root} matches multiple signatures ({found}); " + "a package must carry exactly one well-known file" + ) + + implemented = [ + kind for kind in WELL_KNOWN_FILES if _file_present(package_root, kind) + ] + if not implemented: + expected = ", ".join(sorted(WELL_KNOWN_FILES.values())) + raise SignatureError( + f"unrecognized package: {package_root} contains none of the well-known " + f"files this build can parse ({expected})" + ) + return implemented[0] diff --git a/tests/fixtures/validation/ambiguous_package/README.md b/tests/fixtures/validation/ambiguous_package/README.md new file mode 100644 index 0000000000..76569fea5c --- /dev/null +++ b/tests/fixtures/validation/ambiguous_package/README.md @@ -0,0 +1,3 @@ +A package that carries two well-known files (`openenv.yaml` and `task.toml`). +`openenv validate` must refuse it as ambiguous (exit 2, SignatureError) — never +guess which format to parse. diff --git a/tests/fixtures/validation/ambiguous_package/openenv.yaml b/tests/fixtures/validation/ambiguous_package/openenv.yaml new file mode 100644 index 0000000000..73f313cf9e --- /dev/null +++ b/tests/fixtures/validation/ambiguous_package/openenv.yaml @@ -0,0 +1,6 @@ +spec_version: 1 +name: ambiguous-package +type: env +runtime: python +app: server.app:app +port: 8000 diff --git a/tests/fixtures/validation/ambiguous_package/task.toml b/tests/fixtures/validation/ambiguous_package/task.toml new file mode 100644 index 0000000000..2bb29b893e --- /dev/null +++ b/tests/fixtures/validation/ambiguous_package/task.toml @@ -0,0 +1,6 @@ +schema_version = "1.1" + +[task] +name = "ambiguous-package" +version = "0.1.0" +tags = ["demo"] diff --git a/tests/fixtures/validation/no_oracle/openenv.yaml b/tests/fixtures/validation/no_oracle/openenv.yaml index 4d0a7d596a..b08c8bee35 100644 --- a/tests/fixtures/validation/no_oracle/openenv.yaml +++ b/tests/fixtures/validation/no_oracle/openenv.yaml @@ -1,4 +1,5 @@ spec_version: 1 +version: 0.1.0 name: no-oracle type: env runtime: python diff --git a/tests/fixtures/validation/served_min_pass/openenv.yaml b/tests/fixtures/validation/served_min_pass/openenv.yaml index 8ad02becb6..3643b91a21 100644 --- a/tests/fixtures/validation/served_min_pass/openenv.yaml +++ b/tests/fixtures/validation/served_min_pass/openenv.yaml @@ -1,4 +1,5 @@ spec_version: 1 +version: 0.1.0 name: served-min-pass type: env runtime: python diff --git a/tests/fixtures/validation/unpinned_judge/openenv.yaml b/tests/fixtures/validation/unpinned_judge/openenv.yaml index 8cd7b5680f..a7d4cf2962 100644 --- a/tests/fixtures/validation/unpinned_judge/openenv.yaml +++ b/tests/fixtures/validation/unpinned_judge/openenv.yaml @@ -1,4 +1,5 @@ spec_version: 1 +version: 0.1.0 name: unpinned-judge type: env runtime: python diff --git a/tests/fixtures/validation/unrecognized_package/README.md b/tests/fixtures/validation/unrecognized_package/README.md new file mode 100644 index 0000000000..717f862689 --- /dev/null +++ b/tests/fixtures/validation/unrecognized_package/README.md @@ -0,0 +1,2 @@ +A directory with no well-known package file: `openenv validate` must refuse it +as unrecognized (exit 2), never guess a format. diff --git a/tests/test_cli/test_validate.py b/tests/test_cli/test_validate.py index 3900f42123..0fd9f9467d 100644 --- a/tests/test_cli/test_validate.py +++ b/tests/test_cli/test_validate.py @@ -9,7 +9,10 @@ from unittest.mock import patch from openenv.cli.__main__ import app -from openenv.cli._validation import validate_running_environment +from openenv.cli._validation import ( + validate_multi_mode_deployment, + validate_running_environment, +) from typer.testing import CliRunner @@ -186,34 +189,57 @@ def test_validate_command_runtime_target_outputs_json() -> None: mock_validate.assert_called_once_with("https://example.com", timeout_s=5.0) -def test_validate_command_local_path_still_works(tmp_path: Path) -> None: - """CLI local validation remains backward compatible.""" +def test_validate_command_local_path_without_validation_block_fails( + tmp_path: Path, +) -> None: + """Packages without a `validation:` block fail static.manifest (exit 1).""" env_dir = tmp_path / "test_env" _write_minimal_valid_env(env_dir) - result = runner.invoke(app, ["validate", str(env_dir)]) + result = runner.invoke( + app, ["validate", str(env_dir), "--level", "static", "--skip-build"] + ) - assert result.exit_code == 0 - assert "[OK]" in result.output + assert result.exit_code == 1 + assert "static.manifest" in result.output + assert "validation" in result.output def test_validate_command_local_json_output(tmp_path: Path) -> None: - """CLI can emit JSON report for local validation via --json.""" + """CLI --json emits a schema-valid ValidationReport for a local package.""" env_dir = tmp_path / "test_env" _write_minimal_valid_env(env_dir) + yaml_text = (env_dir / "openenv.yaml").read_text() + (env_dir / "openenv.yaml").write_text( + yaml_text + + "version: 0.1.0\n" + + "validation:\n" + + " reward:\n" + + " range: [0.0, 1.0]\n" + + " floor_margin: 0.5\n" + + " resources:\n" + + " cpu: 1.0\n" + + " memory_mb: 1024\n" + + " disk_mb: 512\n" + + " episode_timeout_s: 60.0\n" + + " capabilities:\n" + + " verifier:\n" + + " kind: reward_channel\n" + + " types:\n" + + " tags: [demo]\n" + ) - result = runner.invoke(app, ["validate", str(env_dir), "--json"]) + result = runner.invoke( + app, ["validate", str(env_dir), "--level", "static", "--skip-build", "--json"] + ) - assert result.exit_code == 0 + assert result.exit_code == 0, result.output payload = json.loads(result.output) - assert payload["validation_type"] == "local_environment" - assert payload["passed"] is True - assert payload["summary"]["passed_count"] == 1 - assert payload["summary"]["total_count"] == 1 - assert payload["summary"]["failed_criteria"] == [] + assert payload["report_schema_version"] == "1" + assert payload["verdict"] == "pass" -def test_validate_command_rejects_environment_package_as_runtime_dependency( +def test_multi_mode_rejects_environment_package_as_runtime_dependency( tmp_path: Path, ) -> None: """An openenv-* environment package is not the OpenEnv runtime package.""" @@ -229,16 +255,16 @@ def test_validate_command_rejects_environment_package_as_runtime_dependency( 'server = "server.app:main"\n' ) - result = runner.invoke(app, ["validate", str(env_dir)]) + is_valid, issues = validate_multi_mode_deployment(env_dir) - assert result.exit_code != 0 - assert "Missing required dependency: openenv>=0.2.0" in result.output + assert is_valid is False + assert "Missing required dependency: openenv>=0.2.0" in issues -def test_validate_command_accepts_dockerfile_managed_openenv_runtime( +def test_multi_mode_accepts_dockerfile_managed_openenv_runtime( tmp_path: Path, ) -> None: - """Local validation accepts envs that install OpenEnv in Dockerfile.""" + """Multi-mode helper accepts envs that install OpenEnv in Dockerfile.""" env_dir = tmp_path / "test_env" _write_minimal_valid_env(env_dir) (env_dir / "pyproject.toml").write_text( @@ -254,14 +280,14 @@ def test_validate_command_accepts_dockerfile_managed_openenv_runtime( 'RUN pip install --no-cache-dir --no-deps "openenv>=0.2.2"\n' ) - result = runner.invoke(app, ["validate", str(env_dir), "--json"]) + is_valid, issues = validate_multi_mode_deployment(env_dir) - assert result.exit_code == 0 - assert json.loads(result.output)["passed"] is True + assert is_valid is True + assert issues == [] -def test_validate_command_accepts_main_call_with_arguments(tmp_path: Path) -> None: - """Local validation accepts a guarded main(...) call with arguments.""" +def test_multi_mode_accepts_main_call_with_arguments(tmp_path: Path) -> None: + """Multi-mode helper accepts a guarded main(...) call with arguments.""" env_dir = tmp_path / "test_env" _write_minimal_valid_env( env_dir, @@ -269,15 +295,14 @@ def test_validate_command_accepts_main_call_with_arguments(tmp_path: Path) -> No main_invocation="main(port=8000)", ) - result = runner.invoke(app, ["validate", str(env_dir)]) + is_valid, issues = validate_multi_mode_deployment(env_dir) - assert result.exit_code == 0 - assert "main() function not callable" not in result.output - assert "[OK]" in result.output + assert is_valid is True + assert not any("main() function not callable" in issue for issue in issues) -def test_validate_command_rejects_nested_main_guard(tmp_path: Path) -> None: - """Local validation requires the __main__ guard at module scope.""" +def test_multi_mode_rejects_nested_main_guard(tmp_path: Path) -> None: + """Multi-mode helper requires the __main__ guard at module scope.""" env_dir = tmp_path / "test_env" _write_minimal_valid_env(env_dir) (env_dir / "server" / "app.py").write_text( @@ -287,14 +312,14 @@ def test_validate_command_rejects_nested_main_guard(tmp_path: Path) -> None: " main()\n" ) - result = runner.invoke(app, ["validate", str(env_dir)]) + is_valid, issues = validate_multi_mode_deployment(env_dir) - assert result.exit_code != 0 - assert "main() function not callable" in result.output + assert is_valid is False + assert any("main() function not callable" in issue for issue in issues) -def test_validate_command_accepts_later_top_level_main_guard(tmp_path: Path) -> None: - """Local validation scans each top-level __main__ guard for a main() call.""" +def test_multi_mode_accepts_later_top_level_main_guard(tmp_path: Path) -> None: + """Multi-mode helper scans each top-level __main__ guard for a main() call.""" env_dir = tmp_path / "test_env" _write_minimal_valid_env(env_dir) (env_dir / "server" / "app.py").write_text( @@ -305,13 +330,13 @@ def test_validate_command_accepts_later_top_level_main_guard(tmp_path: Path) -> " main()\n" ) - result = runner.invoke(app, ["validate", str(env_dir)]) + is_valid, issues = validate_multi_mode_deployment(env_dir) - assert result.exit_code == 0 - assert "[OK]" in result.output + assert is_valid is True + assert issues == [] -def test_validate_command_syntax_error_fallback_requires_dunder_main( +def test_multi_mode_syntax_error_fallback_requires_dunder_main( tmp_path: Path, ) -> None: """Syntax-error fallback still requires the literal __main__ guard string.""" @@ -321,10 +346,10 @@ def test_validate_command_syntax_error_fallback_requires_dunder_main( "def main(:\n return None\n\nif __name__ ==\n main(\n" ) - result = runner.invoke(app, ["validate", str(env_dir)]) + is_valid, issues = validate_multi_mode_deployment(env_dir) - assert result.exit_code != 0 - assert "main() function not callable" in result.output + assert is_valid is False + assert any("main() function not callable" in issue for issue in issues) def test_validate_command_rejects_mixed_path_and_url(tmp_path: Path) -> None: diff --git a/tests/test_validation/test_cli_validate.py b/tests/test_validation/test_cli_validate.py new file mode 100644 index 0000000000..54cf726367 --- /dev/null +++ b/tests/test_validation/test_cli_validate.py @@ -0,0 +1,94 @@ +"""CLI contract: one command, exit codes 0/1/2/3, schema-valid JSON reports.""" + +from conftest import FIXTURES +from openenv.cli.__main__ import app +from openenv.validation.report import ValidationReport +from typer.testing import CliRunner + +runner = CliRunner() + + +def _validate(*args): + return runner.invoke(app, ["validate", *args]) + + +def test_valid_package_exits_zero(): + result = _validate( + str(FIXTURES / "served_min_pass"), "--level", "static", "--skip-build" + ) + assert result.exit_code == 0, result.output + assert "Verdict: PASS" in result.output + + +def test_json_report_is_schema_valid(): + result = _validate( + str(FIXTURES / "served_min_pass"), "--level", "static", "--skip-build", "--json" + ) + assert result.exit_code == 0, result.output + report = ValidationReport.model_validate_json(result.output) + assert report.verdict.value == "pass" + + +def test_output_writes_the_json_report(tmp_path): + out = tmp_path / "report.json" + result = _validate( + str(FIXTURES / "served_min_pass"), + "--level", + "static", + "--skip-build", + "--output", + str(out), + ) + assert result.exit_code == 0, result.output + ValidationReport.model_validate_json(out.read_text()) + + +def test_failing_manifest_exits_one(): + result = _validate( + str(FIXTURES / "broken_manifest"), "--level", "static", "--skip-build" + ) + assert result.exit_code == 1, result.output + assert "static.manifest" in result.output + + +def test_ambiguous_package_exits_two(): + result = _validate( + str(FIXTURES / "ambiguous_package"), "--level", "static", "--skip-build" + ) + assert result.exit_code == 2, result.output + assert "ambiguous" in result.output + + +def test_unrecognized_package_exits_two(): + result = _validate( + str(FIXTURES / "unrecognized_package"), "--level", "static", "--skip-build" + ) + assert result.exit_code == 2, result.output + assert "unrecognized" in result.output + + +def test_format_without_a_parser_exits_two(): + # Harbor packages are refused as unrecognized until the Harbor parser lands. + result = _validate( + str(FIXTURES / "harbor_task_min"), "--level", "static", "--skip-build" + ) + assert result.exit_code == 2, result.output + assert "unrecognized" in result.output + + +def test_nonexistent_path_exits_two(tmp_path): + result = _validate(str(tmp_path / "nope")) + assert result.exit_code == 2, result.output + + +def test_unknown_level_is_an_internal_error(): + result = _validate(str(FIXTURES / "served_min_pass"), "--level", "cosmic") + assert result.exit_code == 3, result.output + + +def test_unpinned_judge_fails_static_manifest(): + result = _validate( + str(FIXTURES / "unpinned_judge"), "--level", "static", "--skip-build" + ) + assert result.exit_code == 1 + assert "judge pin" in result.output diff --git a/tests/test_validation/test_openenv_yaml_parser.py b/tests/test_validation/test_openenv_yaml_parser.py new file mode 100644 index 0000000000..29049f48ec --- /dev/null +++ b/tests/test_validation/test_openenv_yaml_parser.py @@ -0,0 +1,96 @@ +"""OpenEnvYamlParser: fixture openenv.yaml files parse to their committed goldens.""" + +import pytest +from conftest import FIXTURES, load_fixture_manifest +from openenv.validation.manifest import ManifestError, NormalizedManifest +from openenv.validation.parsers.openenv_yaml import OpenEnvYamlParser +from openenv.validation.runner import default_parser_registry +from openenv.validation.signature import SignatureError + +# Served fixtures whose normalized_manifest.json is the golden parse result. +GOLDEN_FIXTURES = [ + "served_min_pass", + "empty_solution_max_reward", + "no_oracle", + "leaky_observation", + "nondeterministic", + "leaky_egress", +] + + +@pytest.mark.parametrize("name", GOLDEN_FIXTURES) +def test_parse_matches_the_committed_golden_manifest(name): + parsed = OpenEnvYamlParser().parse(FIXTURES / name) + golden = NormalizedManifest.model_validate(load_fixture_manifest(name)) + assert parsed == golden + + +def test_broken_manifest_raises_manifest_error_with_field_evidence(): + with pytest.raises(ManifestError) as exc_info: + OpenEnvYamlParser().parse(FIXTURES / "broken_manifest") + evidence = "\n".join(exc_info.value.errors) + assert "reward.range" in evidence or "strictly increasing" in evidence + assert "resources" in evidence + + +def test_unpinned_judge_raises_manifest_error_on_the_judge_pin(): + with pytest.raises(ManifestError) as exc_info: + OpenEnvYamlParser().parse(FIXTURES / "unpinned_judge") + assert any("judge pin" in error for error in exc_info.value.errors) + + +def test_missing_validation_block_has_remediation(tmp_path): + (tmp_path / "openenv.yaml").write_text("spec_version: 1\nname: bare\n") + with pytest.raises(ManifestError) as exc_info: + OpenEnvYamlParser().parse(tmp_path) + assert "validation" in exc_info.value.errors[0] + assert exc_info.value.remediation is not None + + +def test_parse_is_a_pure_read(tmp_path): + # Package code must never be imported or executed: a booby-trapped module + # alongside a valid openenv.yaml parses without tripping. + src = (FIXTURES / "served_min_pass" / "openenv.yaml").read_text() + (tmp_path / "openenv.yaml").write_text(src) + (tmp_path / "server.py").write_text( + "raise SystemExit('parser imported package code')\n" + ) + parsed = OpenEnvYamlParser().parse(tmp_path) + assert parsed.name == "served-min-pass" + + +def test_default_registry_dispatches_openenv_yaml_end_to_end(): + manifest = default_parser_registry().parse(FIXTURES / "served_min_pass") + assert manifest.name == "served-min-pass" + + +def test_default_registry_refuses_formats_without_parsers(): + # task.toml is not in the detection table until the Harbor parser lands, so + # a Harbor package is refused as unrecognized rather than half-supported. + with pytest.raises(SignatureError, match="unrecognized"): + default_parser_registry().parse(FIXTURES / "harbor_task_min") + + +def test_network_policy_parses_and_defaults_to_public(tmp_path): + src = (FIXTURES / "served_min_pass" / "openenv.yaml").read_text() + (tmp_path / "openenv.yaml").write_text( + src + + " network:\n mode: allowlist\n allowed_hosts: [api.example.com, 10.0.0.0/8]\n" + ) + parsed = OpenEnvYamlParser().parse(tmp_path) + assert parsed.network.mode == "allowlist" + assert parsed.network.allowed_hosts == ["api.example.com", "10.0.0.0/8"] + + default = OpenEnvYamlParser().parse(FIXTURES / "served_min_pass") + assert default.network.mode == "public" + assert default.network.allowed_hosts == [] + + +def test_allowed_hosts_without_allowlist_mode_is_a_manifest_error(tmp_path): + src = (FIXTURES / "served_min_pass" / "openenv.yaml").read_text() + (tmp_path / "openenv.yaml").write_text( + src + " network:\n allowed_hosts: [api.example.com]\n" + ) + with pytest.raises(ManifestError) as exc_info: + OpenEnvYamlParser().parse(tmp_path) + assert any("allowlist" in error for error in exc_info.value.errors) diff --git a/tests/test_validation/test_runner.py b/tests/test_validation/test_runner.py new file mode 100644 index 0000000000..2c6491adb2 --- /dev/null +++ b/tests/test_validation/test_runner.py @@ -0,0 +1,80 @@ +"""run_validation: parse → grade → policy → report, one report per run.""" + +import shutil + +import pytest +from conftest import FIXTURES +from openenv.validation.policy import load_policy +from openenv.validation.report import ValidationReport +from openenv.validation.runner import run_validation, source_digest +from openenv.validation.signature import SignatureError +from openenv.validation.types import CheckStatus, Lane, Level, Verdict + + +def test_valid_package_passes_static_level(): + report = run_validation( + FIXTURES / "served_min_pass", max_level=Level.STATIC, skip_build=True + ) + assert report.verdict is Verdict.PASS + assert report.lane is Lane.LOCAL + assert report.levels_run == [Level.STATIC] + assert report.manifest is not None and report.manifest.name == "served-min-pass" + by_id = {r.check_id: r for r in report.results} + assert by_id["static.manifest"].status is CheckStatus.PASS + + +def test_report_round_trips_through_its_schema(): + report = run_validation( + FIXTURES / "served_min_pass", max_level=Level.STATIC, skip_build=True + ) + assert ValidationReport.model_validate_json(report.model_dump_json()) == report + + +def test_broken_manifest_fails_static_manifest_with_evidence(): + report = run_validation( + FIXTURES / "broken_manifest", max_level=Level.STATIC, skip_build=True + ) + assert report.verdict is Verdict.FAIL + assert report.manifest is None + (result,) = report.results + assert result.check_id == "static.manifest" + assert result.status is CheckStatus.FAIL + assert result.evidence, "schema errors must surface as evidence" + + +def test_out_of_bounds_declaration_fails_static_manifest(tmp_path): + src = (FIXTURES / "served_min_pass" / "openenv.yaml").read_text() + (tmp_path / "openenv.yaml").write_text( + src.replace("floor_margin: 0.5", "floor_margin: 0.01") + ) + report = run_validation(tmp_path, max_level=Level.STATIC, skip_build=True) + assert report.verdict is Verdict.FAIL + (result,) = report.results + assert result.status is CheckStatus.FAIL + assert "floor_margin" in "\n".join(result.evidence) + + +def test_ambiguous_package_raises_signature_error(): + with pytest.raises(SignatureError, match="ambiguous"): + run_validation(FIXTURES / "ambiguous_package", max_level=Level.STATIC) + + +def test_report_embeds_the_pinned_policy_version(): + policy = load_policy("v1") + report = run_validation( + FIXTURES / "served_min_pass", + max_level=Level.STATIC, + skip_build=True, + policy=policy, + ) + assert report.policy_version == policy.policy_version + + +def test_source_digest_is_deterministic_and_content_sensitive(tmp_path): + copy = tmp_path / "pkg" + shutil.copytree(FIXTURES / "served_min_pass", copy) + first = source_digest(copy) + assert first == source_digest(copy) + assert len(first) == 64 + (copy / "extra.txt").write_text("changed\n") + assert source_digest(copy) != first diff --git a/tests/test_validation/test_signature_detection.py b/tests/test_validation/test_signature_detection.py new file mode 100644 index 0000000000..9c89de9c48 --- /dev/null +++ b/tests/test_validation/test_signature_detection.py @@ -0,0 +1,51 @@ +"""detect_signature: exactly one well-known file of a parseable format, never a guess.""" + +import pytest +from conftest import FIXTURES +from openenv.validation.signature import ( + detect_signature, + SignatureError, + WELL_KNOWN_FILES, +) +from openenv.validation.types import SignatureKind + + +def test_openenv_yaml_detects_the_served_format(): + assert ( + detect_signature(FIXTURES / "served_min_pass") is SignatureKind.OPENENV_SERVED + ) + + +def test_the_table_lists_exactly_the_implemented_parsers(): + # Only the served-env parser exists in this build. task.toml joins with the + # Harbor parser and task.md with the PostTrain parser. + assert set(WELL_KNOWN_FILES) == {SignatureKind.OPENENV_SERVED} + + +@pytest.mark.parametrize("fixture", ["harbor_task_min", "posttrain_task_min"]) +def test_formats_without_parsers_are_unrecognized_not_guessed(fixture): + # These packages are real formats, but this build cannot parse them — + # refusing as unrecognized beats claiming support. + with pytest.raises(SignatureError, match="unrecognized"): + detect_signature(FIXTURES / fixture) + + +def test_no_well_known_file_is_unrecognized(): + with pytest.raises(SignatureError, match="unrecognized"): + detect_signature(FIXTURES / "unrecognized_package") + + +def test_two_well_known_files_are_ambiguous_never_a_guess(): + with pytest.raises(SignatureError, match="ambiguous"): + detect_signature(FIXTURES / "ambiguous_package") + + +def test_task_md_without_frontmatter_is_not_a_named_format(tmp_path): + (tmp_path / "openenv.yaml").write_text("spec_version: 1\nname: x\n") + (tmp_path / "task.md").write_text("# just a readme\n") + assert detect_signature(tmp_path) is SignatureKind.OPENENV_SERVED + + +def test_missing_directory_is_a_signature_error(tmp_path): + with pytest.raises(SignatureError, match="not a package directory"): + detect_signature(tmp_path / "does_not_exist") From 7d9c2dc4007dab0da90d26faa2cc44e5392fcc5c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 24 Aug 2026 15:21:45 +0000 Subject: [PATCH 02/14] Distinguish missing vs invalid validation-block remediation Schema failures already have a validation: block; tell authors to fix it rather than add another one. Co-authored-by: Zach Wentz --- src/openenv/validation/parsers/openenv_yaml.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/openenv/validation/parsers/openenv_yaml.py b/src/openenv/validation/parsers/openenv_yaml.py index c983ce0051..7108682bde 100644 --- a/src/openenv/validation/parsers/openenv_yaml.py +++ b/src/openenv/validation/parsers/openenv_yaml.py @@ -15,6 +15,11 @@ "see the manifest schema at src/openenv/validation/schemas/manifest.schema.json" ) +SCHEMA_REMEDIATION = ( + "fix the `validation:` block in openenv.yaml so it matches the normalized " + "manifest schema at src/openenv/validation/schemas/manifest.schema.json" +) + def _format_validation_error(exc: ValidationError) -> list[str]: """Render pydantic errors as `field: message` evidence lines.""" @@ -91,5 +96,5 @@ def parse(self, package_root: Path) -> NormalizedManifest: except ValidationError as exc: raise ManifestError( _format_validation_error(exc), - remediation=VALIDATION_BLOCK_REMEDIATION, + remediation=SCHEMA_REMEDIATION, ) from exc From 6774733cd4990e5788963c4160da4e413962fac2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 26 Aug 2026 00:38:37 +0000 Subject: [PATCH 03/14] refactor(validation): drop later-slice weight from the walking skeleton Slice 1 has to prove signature detection, parse, static.manifest, policy, report, and the 0/1/2/3 exit codes. Later-slice fixture yamls and a second detect-and-parse wrapper were not part of that proof. - Delete `ParserRegistry.parse` and the unused `provider` argument on `run_validation`. - Register `StaticManifestGrader` inline and stop creating an outputs tempdir no grader writes. - Delete `openenv.yaml` from `leaky_*`, `no_oracle`, `nondeterministic`, and `empty_solution_max_reward`. - Point checkpoint and CLI tests at `python -m openenv.cli`, not CliRunner. - Map unexpected local-validate exceptions to exit code 3. `skip_build` stays on `run_validation` because the CLI flag is in the RFC contract. It does nothing until a build-dependent grader lands. `openenv validate` on a served package keeps the same 0/1/2 exits. A crash in `run_validation` now exits 3 instead of falling through `__main__` as 1. Pin before this commit: `envs/echo_env` exit 0, `broken_manifest` exit 1, `ambiguous_package` exit 2, 239 tests green. Real-CLI re-run follows on this commit. Co-authored-by: Zach Wentz --- src/openenv/cli/commands/validate.py | 3 + src/openenv/validation/parsers/__init__.py | 18 +----- src/openenv/validation/runner.py | 45 ++++++--------- .../validation/no_oracle/openenv.yaml | 23 -------- tests/test_cli/test_validate.py | 43 +++++++++++--- tests/test_validation/test_cli_validate.py | 57 +++++++++++-------- .../test_openenv_yaml_parser.py | 41 +++++-------- 7 files changed, 103 insertions(+), 127 deletions(-) delete mode 100644 tests/fixtures/validation/no_oracle/openenv.yaml diff --git a/src/openenv/cli/commands/validate.py b/src/openenv/cli/commands/validate.py index 04555d9ae1..6defaeea44 100644 --- a/src/openenv/cli/commands/validate.py +++ b/src/openenv/cli/commands/validate.py @@ -203,6 +203,9 @@ def validate( except PolicyError as exc: typer.echo(f"Internal error: {exc}", err=True) raise typer.Exit(EXIT_INTERNAL) from exc + except Exception as exc: + typer.echo(f"Internal error: {exc}", err=True) + raise typer.Exit(EXIT_INTERNAL) from exc report_json = write_report(validation_report, output) if json_output: diff --git a/src/openenv/validation/parsers/__init__.py b/src/openenv/validation/parsers/__init__.py index a8b9fb3215..0f76929196 100644 --- a/src/openenv/validation/parsers/__init__.py +++ b/src/openenv/validation/parsers/__init__.py @@ -9,7 +9,7 @@ from typing import Protocol, runtime_checkable from ..manifest import NormalizedManifest -from ..signature import detect_signature, UnsupportedPackageError +from ..signature import UnsupportedPackageError from ..types import SignatureKind @@ -67,19 +67,3 @@ def parser_for(self, signature: SignatureKind) -> Parser: f"no parser registered for signature {signature.value!r}", ) return parser - - def parse(self, package_root: Path) -> NormalizedManifest: - """ - Detect the package signature and dispatch to its parser. - - This is the only place a [`~openenv.validation.types.SignatureKind`] selects - behavior. - - Args: - package_root (`Path`): - The package directory to detect and parse. - - Returns: - [`~openenv.validation.manifest.NormalizedManifest`]: the normalized manifest. - """ - return self.parser_for(detect_signature(package_root)).parse(package_root) diff --git a/src/openenv/validation/runner.py b/src/openenv/validation/runner.py index a1ada20645..1ee7b3ac38 100644 --- a/src/openenv/validation/runner.py +++ b/src/openenv/validation/runner.py @@ -6,7 +6,6 @@ """ import hashlib -import tempfile import time from pathlib import Path @@ -16,7 +15,6 @@ from .parsers import ParserRegistry from .parsers.openenv_yaml import OpenEnvYamlParser from .policy import apply_policy, load_policy, SeverityPolicy -from .providers import ValidationProvider from .report import CheckResult, ValidationReport from .signature import detect_signature from .types import CheckStatus, Lane, Level @@ -62,13 +60,6 @@ def default_parser_registry() -> ParserRegistry: return registry -def default_grader_registry(policy: SeverityPolicy) -> GraderRegistry: - """The core graders shipped in this build, configured against a policy.""" - registry = GraderRegistry() - registry.register(StaticManifestGrader(policy.bounds)) - return registry - - def _run_grader(grader, subject: Subject) -> CheckResult: """Run one grader; a crash is an ERROR result (fails closed), never an abort.""" started = time.monotonic() @@ -87,7 +78,6 @@ def run_validation( target: Path, *, max_level: Level = Level.SEMANTIC, - provider: ValidationProvider | None = None, skip_build: bool = False, policy: SeverityPolicy | None = None, ) -> ValidationReport: @@ -106,20 +96,16 @@ def run_validation( The package directory. max_level ([`~openenv.validation.types.Level`], *optional*, defaults to `Level.SEMANTIC`): Level ceiling; graders above it are not selected. - provider ([`~openenv.validation.providers.ValidationProvider`], *optional*): - Sandbox provider for runtime+ levels. Unused until the runtime phase - lands; `None` selects the default provider then. skip_build (`bool`, *optional*, defaults to `False`): Skip the image build; build-dependent checks SKIP with a reason. - Slice 1 has no build-dependent graders, so this only records - `image_ref=None` on the subject. + Slice 1 has no build-dependent graders, so this is accepted and unused. policy ([`~openenv.validation.policy.SeverityPolicy`], *optional*): Severity policy; `None` loads the committed default version. Returns: [`~openenv.validation.report.ValidationReport`]: the completed report. """ - del provider, skip_build # reserved: runtime build + provider selection + del skip_build target = Path(target) policy = policy or load_policy() signature = detect_signature(target) @@ -127,7 +113,6 @@ def run_validation( parser = default_parser_registry().parser_for(signature) manifest: NormalizedManifest | None = None results: list[CheckResult] = [] - levels_run: list[Level] = [Level.STATIC] parse_started = time.monotonic() try: @@ -145,17 +130,19 @@ def run_validation( ) if manifest is not None: - graders = default_grader_registry(policy).select(manifest, max_level) - with tempfile.TemporaryDirectory(prefix="openenv-validate-") as outputs: - subject = Subject( - root=target, - manifest=manifest, - image_ref=None, - running=None, - outputs_dir=Path(outputs), - ) - results.extend(_run_grader(grader, subject) for grader in graders) - levels_run = sorted({Level.STATIC, *(g.level for g in graders)}) + graders = GraderRegistry() + graders.register(StaticManifestGrader(policy.bounds)) + subject = Subject( + root=target, + manifest=manifest, + image_ref=None, + running=None, + outputs_dir=target, + ) + results.extend( + _run_grader(grader, subject) + for grader in graders.select(manifest, max_level) + ) return ValidationReport( report_schema_version=REPORT_SCHEMA_VERSION, @@ -165,7 +152,7 @@ def run_validation( manifest=manifest, policy_version=policy.policy_version, lane=Lane.LOCAL, - levels_run=levels_run, + levels_run=[Level.STATIC], results=results, verdict=apply_policy(results, policy, Lane.LOCAL), ) diff --git a/tests/fixtures/validation/no_oracle/openenv.yaml b/tests/fixtures/validation/no_oracle/openenv.yaml deleted file mode 100644 index b08c8bee35..0000000000 --- a/tests/fixtures/validation/no_oracle/openenv.yaml +++ /dev/null @@ -1,23 +0,0 @@ -spec_version: 1 -version: 0.1.0 -name: no-oracle -type: env -runtime: python -app: server.app:app -port: 8000 -validation: - reward: - range: [0.0, 1.0] - oracle_tolerance: 0.0 - floor_margin: 0.5 - resources: - cpu: 1.0 - memory_mb: 1024 - disk_mb: 512 - episode_timeout_s: 60.0 - capabilities: - verifier: - kind: reward_channel - declared_tools: [echo] - types: - tags: [demo] diff --git a/tests/test_cli/test_validate.py b/tests/test_cli/test_validate.py index 0fd9f9467d..4b69d16a36 100644 --- a/tests/test_cli/test_validate.py +++ b/tests/test_cli/test_validate.py @@ -5,6 +5,8 @@ from __future__ import annotations import json +import subprocess +import sys from pathlib import Path from unittest.mock import patch @@ -196,13 +198,24 @@ def test_validate_command_local_path_without_validation_block_fails( env_dir = tmp_path / "test_env" _write_minimal_valid_env(env_dir) - result = runner.invoke( - app, ["validate", str(env_dir), "--level", "static", "--skip-build"] + result = subprocess.run( + [ + sys.executable, + "-m", + "openenv.cli", + "validate", + str(env_dir), + "--level", + "static", + "--skip-build", + ], + capture_output=True, + text=True, ) - assert result.exit_code == 1 - assert "static.manifest" in result.output - assert "validation" in result.output + assert result.returncode == 1 + assert "static.manifest" in result.stdout + assert "validation" in result.stdout def test_validate_command_local_json_output(tmp_path: Path) -> None: @@ -229,12 +242,24 @@ def test_validate_command_local_json_output(tmp_path: Path) -> None: + " tags: [demo]\n" ) - result = runner.invoke( - app, ["validate", str(env_dir), "--level", "static", "--skip-build", "--json"] + result = subprocess.run( + [ + sys.executable, + "-m", + "openenv.cli", + "validate", + str(env_dir), + "--level", + "static", + "--skip-build", + "--json", + ], + capture_output=True, + text=True, ) - assert result.exit_code == 0, result.output - payload = json.loads(result.output) + assert result.returncode == 0, result.stdout + result.stderr + payload = json.loads(result.stdout) assert payload["report_schema_version"] == "1" assert payload["verdict"] == "pass" diff --git a/tests/test_validation/test_cli_validate.py b/tests/test_validation/test_cli_validate.py index 54cf726367..dcdc33f2ad 100644 --- a/tests/test_validation/test_cli_validate.py +++ b/tests/test_validation/test_cli_validate.py @@ -1,31 +1,38 @@ """CLI contract: one command, exit codes 0/1/2/3, schema-valid JSON reports.""" +import subprocess +import sys +from pathlib import Path + from conftest import FIXTURES -from openenv.cli.__main__ import app from openenv.validation.report import ValidationReport -from typer.testing import CliRunner -runner = CliRunner() +REPO_ROOT = Path(__file__).parent.parent.parent -def _validate(*args): - return runner.invoke(app, ["validate", *args]) +def _validate(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, "-m", "openenv.cli", "validate", *args], + capture_output=True, + text=True, + cwd=REPO_ROOT, + ) def test_valid_package_exits_zero(): result = _validate( str(FIXTURES / "served_min_pass"), "--level", "static", "--skip-build" ) - assert result.exit_code == 0, result.output - assert "Verdict: PASS" in result.output + assert result.returncode == 0, result.stdout + result.stderr + assert "Verdict: PASS" in result.stdout def test_json_report_is_schema_valid(): result = _validate( str(FIXTURES / "served_min_pass"), "--level", "static", "--skip-build", "--json" ) - assert result.exit_code == 0, result.output - report = ValidationReport.model_validate_json(result.output) + assert result.returncode == 0, result.stdout + result.stderr + report = ValidationReport.model_validate_json(result.stdout) assert report.verdict.value == "pass" @@ -39,7 +46,7 @@ def test_output_writes_the_json_report(tmp_path): "--output", str(out), ) - assert result.exit_code == 0, result.output + assert result.returncode == 0, result.stdout + result.stderr ValidationReport.model_validate_json(out.read_text()) @@ -47,48 +54,52 @@ def test_failing_manifest_exits_one(): result = _validate( str(FIXTURES / "broken_manifest"), "--level", "static", "--skip-build" ) - assert result.exit_code == 1, result.output - assert "static.manifest" in result.output + assert result.returncode == 1, result.stdout + result.stderr + assert "static.manifest" in result.stdout def test_ambiguous_package_exits_two(): result = _validate( str(FIXTURES / "ambiguous_package"), "--level", "static", "--skip-build" ) - assert result.exit_code == 2, result.output - assert "ambiguous" in result.output + assert result.returncode == 2, result.stdout + result.stderr + assert "ambiguous" in result.stderr def test_unrecognized_package_exits_two(): result = _validate( str(FIXTURES / "unrecognized_package"), "--level", "static", "--skip-build" ) - assert result.exit_code == 2, result.output - assert "unrecognized" in result.output + assert result.returncode == 2, result.stdout + result.stderr + assert "unrecognized" in result.stderr def test_format_without_a_parser_exits_two(): - # Harbor packages are refused as unrecognized until the Harbor parser lands. result = _validate( str(FIXTURES / "harbor_task_min"), "--level", "static", "--skip-build" ) - assert result.exit_code == 2, result.output - assert "unrecognized" in result.output + assert result.returncode == 2, result.stdout + result.stderr + assert "unrecognized" in result.stderr def test_nonexistent_path_exits_two(tmp_path): result = _validate(str(tmp_path / "nope")) - assert result.exit_code == 2, result.output + assert result.returncode == 2, result.stdout + result.stderr def test_unknown_level_is_an_internal_error(): result = _validate(str(FIXTURES / "served_min_pass"), "--level", "cosmic") - assert result.exit_code == 3, result.output + assert result.returncode == 3, result.stdout + result.stderr + + +def test_unknown_policy_is_an_internal_error(): + result = _validate(str(FIXTURES / "served_min_pass"), "--policy", "nope") + assert result.returncode == 3, result.stdout + result.stderr def test_unpinned_judge_fails_static_manifest(): result = _validate( str(FIXTURES / "unpinned_judge"), "--level", "static", "--skip-build" ) - assert result.exit_code == 1 - assert "judge pin" in result.output + assert result.returncode == 1 + assert "judge pin" in result.stdout diff --git a/tests/test_validation/test_openenv_yaml_parser.py b/tests/test_validation/test_openenv_yaml_parser.py index 29049f48ec..6e09d72c35 100644 --- a/tests/test_validation/test_openenv_yaml_parser.py +++ b/tests/test_validation/test_openenv_yaml_parser.py @@ -1,27 +1,17 @@ -"""OpenEnvYamlParser: fixture openenv.yaml files parse to their committed goldens.""" +"""OpenEnvYamlParser: openenv.yaml maps onto the committed served-min-pass golden.""" import pytest from conftest import FIXTURES, load_fixture_manifest from openenv.validation.manifest import ManifestError, NormalizedManifest from openenv.validation.parsers.openenv_yaml import OpenEnvYamlParser from openenv.validation.runner import default_parser_registry -from openenv.validation.signature import SignatureError - -# Served fixtures whose normalized_manifest.json is the golden parse result. -GOLDEN_FIXTURES = [ - "served_min_pass", - "empty_solution_max_reward", - "no_oracle", - "leaky_observation", - "nondeterministic", - "leaky_egress", -] - - -@pytest.mark.parametrize("name", GOLDEN_FIXTURES) -def test_parse_matches_the_committed_golden_manifest(name): - parsed = OpenEnvYamlParser().parse(FIXTURES / name) - golden = NormalizedManifest.model_validate(load_fixture_manifest(name)) +from openenv.validation.signature import detect_signature +from openenv.validation.types import SignatureKind + + +def test_parse_matches_the_committed_golden_manifest(): + parsed = OpenEnvYamlParser().parse(FIXTURES / "served_min_pass") + golden = NormalizedManifest.model_validate(load_fixture_manifest("served_min_pass")) assert parsed == golden @@ -60,17 +50,16 @@ def test_parse_is_a_pure_read(tmp_path): def test_default_registry_dispatches_openenv_yaml_end_to_end(): - manifest = default_parser_registry().parse(FIXTURES / "served_min_pass") + signature = detect_signature(FIXTURES / "served_min_pass") + assert signature is SignatureKind.OPENENV_SERVED + manifest = ( + default_parser_registry() + .parser_for(signature) + .parse(FIXTURES / "served_min_pass") + ) assert manifest.name == "served-min-pass" -def test_default_registry_refuses_formats_without_parsers(): - # task.toml is not in the detection table until the Harbor parser lands, so - # a Harbor package is refused as unrecognized rather than half-supported. - with pytest.raises(SignatureError, match="unrecognized"): - default_parser_registry().parse(FIXTURES / "harbor_task_min") - - def test_network_policy_parses_and_defaults_to_public(tmp_path): src = (FIXTURES / "served_min_pass" / "openenv.yaml").read_text() (tmp_path / "openenv.yaml").write_text( From 001ccf218e3548d627a05bad2905105b91f73982 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 26 Aug 2026 01:23:42 +0000 Subject: [PATCH 04/14] Add verify-openenv-validate skill for CLI validation proofs Seed a project-local skill that drives `openenv validate` the way an author does, with a feature map starting at echo_env static validate. Co-authored-by: Zach Wentz --- .../skills/verify-openenv-validate/SKILL.md | 114 ++++++++++++++++++ .../artifacts/.gitignore | 2 + .../bin/check-report.py | 31 +++++ .../verify-openenv-validate/bin/cleanup.sh | 31 +++++ .../verify-openenv-validate/bin/doctor.sh | 55 +++++++++ .../bin/validate-run.sh | 64 ++++++++++ .../features/README.md | 45 +++++++ .../features/ambiguous-package.md | 30 +++++ .../features/broken-manifest.md | 32 +++++ .../features/echo-env-static-validate.md | 37 ++++++ .../features/json-report-output.md | 33 +++++ .../features/unrecognized-package.md | 28 +++++ 12 files changed, 502 insertions(+) create mode 100644 .cursor/skills/verify-openenv-validate/SKILL.md create mode 100644 .cursor/skills/verify-openenv-validate/artifacts/.gitignore create mode 100755 .cursor/skills/verify-openenv-validate/bin/check-report.py create mode 100755 .cursor/skills/verify-openenv-validate/bin/cleanup.sh create mode 100755 .cursor/skills/verify-openenv-validate/bin/doctor.sh create mode 100755 .cursor/skills/verify-openenv-validate/bin/validate-run.sh create mode 100644 .cursor/skills/verify-openenv-validate/features/README.md create mode 100644 .cursor/skills/verify-openenv-validate/features/ambiguous-package.md create mode 100644 .cursor/skills/verify-openenv-validate/features/broken-manifest.md create mode 100644 .cursor/skills/verify-openenv-validate/features/echo-env-static-validate.md create mode 100644 .cursor/skills/verify-openenv-validate/features/json-report-output.md create mode 100644 .cursor/skills/verify-openenv-validate/features/unrecognized-package.md diff --git a/.cursor/skills/verify-openenv-validate/SKILL.md b/.cursor/skills/verify-openenv-validate/SKILL.md new file mode 100644 index 0000000000..61e67bc1a3 --- /dev/null +++ b/.cursor/skills/verify-openenv-validate/SKILL.md @@ -0,0 +1,114 @@ +--- +name: verify-openenv-validate +description: "Drive the OpenEnv CLI `openenv validate` the way an author does: local package path, stdout/stderr, exit codes, and a schema-valid ValidationReport. Use when proving echo_env or fixture validation, checking RFC 008 slice-1 checkpoints, or verifying validate CLI behavior after validation-pipeline changes." +--- + +# Verify `openenv validate` + +Short-lived CLI. There is no always-on server for local static validation. Launch means the checkout can run `openenv validate`; each drive is one isolated command that writes its own evidence. + +Primary user surface: `uv run openenv validate `. The equivalent spawn used by this repo's checkpoint tests is `uv run python -m openenv.cli validate `. Prefer the user-facing `openenv` form unless a recipe names the module form. + +`--url` probes a running FastAPI env (legacy runtime). That is a different surface; do not treat a live `echo_env` server as proof of local static validation. + +## Launch + +From the OpenEnv repo root (`/workspace` in this checkout): + +```bash +export REPO_ROOT="$(git rev-parse --show-toplevel)" +export RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$" +export EVIDENCE_ROOT="$REPO_ROOT/.cursor/skills/verify-openenv-validate/artifacts/$RUN_ID" +export SCRATCH_ROOT="/tmp/openenv-validate-verify-$RUN_ID" +mkdir -p "$EVIDENCE_ROOT" "$SCRATCH_ROOT" +export PATH="$HOME/.local/bin:$PATH" +cd "$REPO_ROOT" +``` + +Ready when this exits 0 and prints the validate help (must mention `--level` and `--skip-build`): + +```bash +uv run openenv validate --help +``` + +If `uv` is missing, install it to `~/.local/bin` and re-run. If the import fails, `uv sync --all-extras` from `$REPO_ROOT` (core extras are enough for this CLI). Do not start `echo_env.server.app` for static recipes. + +Teardown is Cleanup below. Launch does not leave a daemon. + +## Doctor + +Read-only. Run first whenever anything looks off: + +```bash +bash "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/doctor.sh" +``` + +Doctor must print `ok` for each of: `uv` on PATH, `openenv validate --help` exits 0, `envs/echo_env/openenv.yaml` exists and contains a `validation:` block, and `ValidationReport` imports. Exit 0 means the instance is worth driving. Exit 1 means stop and fix; do not drive a half-imported CLI. + +Two static validates may run side by side. They only read package trees and write reports under `$EVIDENCE_ROOT` / `$SCRATCH_ROOT`. Never write into `envs/echo_env/` or `tests/fixtures/validation/`. + +## Drive + +Read `features/README.md`, then the matching feature file. Follow that file's harness section literally. + +Harness is the CLI helper (tmux optional; the helper already isolates stdout/stderr/exit): + +```bash +bash "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/validate-run.sh" \ + -- \ + validate [flags] +``` + +That runs `uv run openenv …` from `$REPO_ROOT`, writes `stdout.txt`, `stderr.txt`, `exit_code.txt`, `command.txt` under `$EVIDENCE_ROOT//`, and copies `--output` reports into the same directory when the recipe names one. + +Stable handles (assert these, not durations or digest strings): + +- Human stdout: `Validation report for `, `PASS static.manifest`, `Verdict: PASS` / `Verdict: FAIL` +- JSON stdout or `--output` file: `report_schema_version` is `"1"`, `verdict` is `pass`/`fail`, a `results[]` entry has `check_id` `static.manifest` +- Exit codes: `0` PASS/WARN, `1` FAIL, `2` SignatureError / unrecognized / unsupported, `3` PolicyError / unknown `--level` +- Signature-error text goes to **stderr** (`ambiguous`, `unrecognized`), not stdout +- Schema-valid JSON means `ValidationReport.model_validate_json(...)` succeeds + +Slice 1 only implements `static.manifest`. `--level static --skip-build` is the inner-loop contract. `--skip-build` is a no-op until build graders exist; still pass it so the user path matches the checkpoint. + +## Evidence + +Proof lives under `$EVIDENCE_ROOT` (see Launch). Cleanup must not delete it. + +Standards: + +- Drive the real CLI (`uv run openenv validate …`), not `CliRunner`, not `run_validation()` as a library call, not pytest as a substitute for the user path. +- Capture the command, stdout, stderr, and exit code for every drive. A passing pytest file is not this skill's proof. +- For PASS packages, also capture a JSON report (`--json` and/or `--output`) and validate it with `ValidationReport`. +- For FAIL packages, human stdout must name `static.manifest` and the process must exit 1. +- For signature refusals, stderr must name the reason and the process must exit 2. There is no report file. +- `--output PATH` proof is the file on disk, not only stdout. Human mode still prints the text report; `--json` prints the JSON instead. +- Do not mock the filesystem package. Use `envs/echo_env` and `tests/fixtures/validation/*` as committed. + +After a JSON report is written, check it: + +```bash +uv run python "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/check-report.py" \ + "$EVIDENCE_ROOT//report.json" +``` + +## Cleanup + +```bash +bash "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/cleanup.sh" +``` + +Removes only `$SCRATCH_ROOT` (`/tmp/openenv-validate-verify-$RUN_ID`). Does not kill by process name. Does not delete `$EVIDENCE_ROOT`. After cleanup, `ls "$EVIDENCE_ROOT/"` must still show the proof files. + +## Helpers + +All scripts are executable. Invoke them as shown; do not invent flags. + +| Script | Purpose | +| --- | --- | +| `bin/doctor.sh` | Read-only readiness. No args. Exit 0/1. | +| `bin/validate-run.sh -- ` | One isolated `uv run openenv` capture. | +| `bin/check-report.py ` | `ValidationReport` parse; prints verdict and `static.manifest` status. | +| `bin/cleanup.sh` | Delete scratch dir only. | + +`validate-run.sh` requires `REPO_ROOT`, `EVIDENCE_ROOT` from Launch. Optional `EXPECTED_EXIT` (default: do not assert). If `EXPECTED_EXIT` is set and the process exits otherwise, the helper exits 1 after still writing the capture files. diff --git a/.cursor/skills/verify-openenv-validate/artifacts/.gitignore b/.cursor/skills/verify-openenv-validate/artifacts/.gitignore new file mode 100644 index 0000000000..d6b7ef32c8 --- /dev/null +++ b/.cursor/skills/verify-openenv-validate/artifacts/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/.cursor/skills/verify-openenv-validate/bin/check-report.py b/.cursor/skills/verify-openenv-validate/bin/check-report.py new file mode 100755 index 0000000000..e88a8af7e5 --- /dev/null +++ b/.cursor/skills/verify-openenv-validate/bin/check-report.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +"""Parse a ValidationReport JSON file and print stable fields.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +from openenv.validation.report import ValidationReport + + +def main() -> int: + if len(sys.argv) != 2: + print("usage: check-report.py ", file=sys.stderr) + return 2 + path = Path(sys.argv[1]) + report = ValidationReport.model_validate_json(path.read_text()) + statuses = {item.check_id: item.status.value for item in report.results} + print(f"schema={report.report_schema_version}") + print(f"target={report.target}") + print(f"signature={report.signature.value}") + print(f"policy={report.policy_version}") + print(f"verdict={report.verdict.value}") + print(f"levels={[level.name.lower() for level in report.levels_run]}") + print(f"static.manifest={statuses.get('static.manifest', 'absent')}") + print(f"manifest={'present' if report.manifest is not None else 'null'}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.cursor/skills/verify-openenv-validate/bin/cleanup.sh b/.cursor/skills/verify-openenv-validate/bin/cleanup.sh new file mode 100755 index 0000000000..89bb0c1593 --- /dev/null +++ b/.cursor/skills/verify-openenv-validate/bin/cleanup.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Remove scratch created by this verification run. Never deletes EVIDENCE_ROOT. +set -euo pipefail + +if [[ -z "${SCRATCH_ROOT:-}" ]]; then + printf 'cleanup.sh needs SCRATCH_ROOT\n' >&2 + exit 2 +fi + +case "${SCRATCH_ROOT}" in + /tmp/openenv-validate-verify-*) + if [[ -d "${SCRATCH_ROOT}" ]]; then + rm -rf "${SCRATCH_ROOT}" + printf 'removed %s\n' "${SCRATCH_ROOT}" + else + printf 'no scratch at %s\n' "${SCRATCH_ROOT}" + fi + ;; + *) + printf 'refusing to remove unexpected SCRATCH_ROOT=%s\n' "${SCRATCH_ROOT}" >&2 + exit 2 + ;; +esac + +if [[ -n "${EVIDENCE_ROOT:-}" ]]; then + if [[ -d "${EVIDENCE_ROOT}" ]]; then + printf 'evidence kept at %s\n' "${EVIDENCE_ROOT}" + else + printf 'warning: EVIDENCE_ROOT missing after cleanup: %s\n' "${EVIDENCE_ROOT}" >&2 + fi +fi diff --git a/.cursor/skills/verify-openenv-validate/bin/doctor.sh b/.cursor/skills/verify-openenv-validate/bin/doctor.sh new file mode 100755 index 0000000000..ee371ed190 --- /dev/null +++ b/.cursor/skills/verify-openenv-validate/bin/doctor.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Read-only readiness for verify-openenv-validate. No args. Exit 0 if worth driving. +set -euo pipefail + +fail() { + printf 'not ok %s\n' "$1" >&2 + exit 1 +} + +ok() { + printf 'ok %s\n' "$1" +} + +REPO_ROOT="${REPO_ROOT:-$(git rev-parse --show-toplevel 2>/dev/null || true)}" +if [[ -z "${REPO_ROOT}" || ! -d "${REPO_ROOT}/envs/echo_env" ]]; then + fail "REPO_ROOT is not an OpenEnv checkout (set REPO_ROOT or run from the repo)" +fi + +export PATH="${HOME}/.local/bin:${PATH}" + +if ! command -v uv >/dev/null 2>&1; then + fail "uv not on PATH" +fi +ok "uv $(command -v uv)" + +echo_yaml="${REPO_ROOT}/envs/echo_env/openenv.yaml" +if [[ ! -f "${echo_yaml}" ]]; then + fail "missing ${echo_yaml}" +fi +if ! grep -q '^validation:' "${echo_yaml}"; then + fail "${echo_yaml} has no validation: block" +fi +ok "envs/echo_env/openenv.yaml has validation:" + +cd "${REPO_ROOT}" +if ! uv run openenv validate --help >/tmp/openenv-validate-help.$$ 2>&1; then + fail "uv run openenv validate --help failed" +fi +if ! grep -q -- '--level' /tmp/openenv-validate-help.$$; then + rm -f /tmp/openenv-validate-help.$$ + fail "validate --help missing --level" +fi +if ! grep -q -- '--skip-build' /tmp/openenv-validate-help.$$; then + rm -f /tmp/openenv-validate-help.$$ + fail "validate --help missing --skip-build" +fi +rm -f /tmp/openenv-validate-help.$$ +ok "openenv validate --help" + +if ! uv run python -c "from openenv.validation.report import ValidationReport" >/dev/null 2>&1; then + fail "ValidationReport import failed" +fi +ok "ValidationReport import" + +printf 'doctor: ready\n' diff --git a/.cursor/skills/verify-openenv-validate/bin/validate-run.sh b/.cursor/skills/verify-openenv-validate/bin/validate-run.sh new file mode 100755 index 0000000000..c8a773f4ac --- /dev/null +++ b/.cursor/skills/verify-openenv-validate/bin/validate-run.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Capture one `uv run openenv` invocation. +# Usage: validate-run.sh -- +# Requires REPO_ROOT and EVIDENCE_ROOT. Optional EXPECTED_EXIT. +set -euo pipefail + +if [[ $# -lt 3 || "$2" != "--" ]]; then + printf 'usage: validate-run.sh -- \n' >&2 + exit 2 +fi + +feature_id="$1" +shift 2 + +if [[ -z "${REPO_ROOT:-}" || -z "${EVIDENCE_ROOT:-}" ]]; then + printf 'validate-run.sh needs REPO_ROOT and EVIDENCE_ROOT\n' >&2 + exit 2 +fi + +out_dir="${EVIDENCE_ROOT}/${feature_id}" +mkdir -p "${out_dir}" + +export PATH="${HOME}/.local/bin:${PATH}" + +{ + printf 'cwd=%s\n' "${REPO_ROOT}" + printf 'uv run openenv' + for arg in "$@"; do + printf ' %q' "${arg}" + done + printf '\n' +} >"${out_dir}/command.txt" + +set +e +( + cd "${REPO_ROOT}" + uv run openenv "$@" +) >"${out_dir}/stdout.txt" 2>"${out_dir}/stderr.txt" +exit_code=$? +set -e + +printf '%s\n' "${exit_code}" >"${out_dir}/exit_code.txt" + +# If the user passed --output, copy the report next to the capture when it exists. +output_path="" +prev="" +for arg in "$@"; do + if [[ "${prev}" == "--output" ]]; then + output_path="${arg}" + fi + prev="${arg}" +done +if [[ -n "${output_path}" && -f "${output_path}" ]]; then + cp "${output_path}" "${out_dir}/report.json" +fi + +printf 'captured %s exit=%s\n' "${feature_id}" "${exit_code}" + +if [[ -n "${EXPECTED_EXIT:-}" && "${exit_code}" != "${EXPECTED_EXIT}" ]]; then + printf 'expected exit %s, got %s\n' "${EXPECTED_EXIT}" "${exit_code}" >&2 + exit 1 +fi + +exit 0 diff --git a/.cursor/skills/verify-openenv-validate/features/README.md b/.cursor/skills/verify-openenv-validate/features/README.md new file mode 100644 index 0000000000..a61ba5789c --- /dev/null +++ b/.cursor/skills/verify-openenv-validate/features/README.md @@ -0,0 +1,45 @@ +# `openenv validate` verification map + +This directory is the maintained source for verifying the user-facing behavior of `openenv validate`. Read this index before driving the CLI, then use the matching feature file as the recipe. + +## Baseline preconditions + +- Work from `$REPO_ROOT` (OpenEnv checkout). `uv` is on `PATH` (`~/.local/bin`). +- `bash .cursor/skills/verify-openenv-validate/bin/doctor.sh` exits 0. +- `EVIDENCE_ROOT` and `SCRATCH_ROOT` are set from the skill Launch section. +- Do not mutate `envs/echo_env/` or `tests/fixtures/validation/`. +- Do not start an environment server unless a feature file names `--url`. +- Treat every command as literal. Keep package paths and flags unchanged. + +## Driving conventions + +- Start every recipe from the baseline unless its preconditions say otherwise. +- Drive through `bin/validate-run.sh` so stdout, stderr, and the exit code are captured together. +- Prefer `uv run openenv validate …` (user path). The module form `python -m openenv.cli validate` is the same CLI; use it only when a recipe says so. +- Local static recipes always pass `--level static --skip-build`. +- Restore nothing after a drive: these commands are read-only on package trees. Scratch reports under `$SCRATCH_ROOT` are removed in cleanup. Proof under `$EVIDENCE_ROOT` stays. + +## Proof and skip reporting + +- CLI proof includes the command, stdout, stderr, and exit code. +- JSON proof is a file that `bin/check-report.py` accepts. +- Record the feature ID with every artifact directory name. +- Report an unreachable path with the attempted command and the unmet precondition. +- Do not report a skipped entry point as verified through a different path (human vs `--json` vs `--output` are distinct when the feature lists them). + +## Feature entry contract + +Each feature file starts with an H1 title and one paragraph describing the user-visible behavior. It then uses exactly four H2 sections in this order. + +1. `Sub-features` +2. `How to get to it (user POV)` +3. `Driving it with validate-run` +4. `Gotchas` + +## Features + +- [Echo env static validate](./echo-env-static-validate.md) is the slice-1 happy path: `envs/echo_env` exits 0 with a schema-valid report and `static.manifest` PASS. +- [Broken manifest](./broken-manifest.md) is a graded FAIL: invalid `validation:` exits 1 and names `static.manifest`. +- [Ambiguous package](./ambiguous-package.md) is a signature refusal: two well-known files exit 2 on stderr, no report. +- [JSON report output](./json-report-output.md) is the machine-readable path: `--json` and `--output` emit `ValidationReport`. +- [Unrecognized package](./unrecognized-package.md) is a signature refusal: no well-known file exits 2 on stderr. diff --git a/.cursor/skills/verify-openenv-validate/features/ambiguous-package.md b/.cursor/skills/verify-openenv-validate/features/ambiguous-package.md new file mode 100644 index 0000000000..5d46818459 --- /dev/null +++ b/.cursor/skills/verify-openenv-validate/features/ambiguous-package.md @@ -0,0 +1,30 @@ +# Ambiguous package + +`openenv validate` refuses `tests/fixtures/validation/ambiguous_package` because it carries two well-known files (`openenv.yaml` and `task.toml`). It prints a signature error on stderr and exits 2 without guessing a format. + +## Sub-features + +- `ambiguous-refuse` exits 2. +- `ambiguous-stderr` says the package is ambiguous. +- `ambiguous-no-report` does not print a `Verdict:` line or a `ValidationReport` JSON object. + +## How to get to it (user POV) + +- Run `openenv validate tests/fixtures/validation/ambiguous_package --level static --skip-build`. + +## Driving it with validate-run + +Preconditions: + +- Doctor exited 0. +- The fixture directory contains both `openenv.yaml` and `task.toml`. + +- **Signature refusal.** Run `EXPECTED_EXIT=2 bash "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/validate-run.sh" ambiguous-package -- validate tests/fixtures/validation/ambiguous_package --level static --skip-build`. Exit code `2`. Stderr contains `ambiguous` (case-insensitive). Stdout has no `Verdict:` and is not parseable as `ValidationReport`. +- **Proof.** Keep `$EVIDENCE_ROOT/ambiguous-package/{command,stdout,stderr,exit_code}.txt`. + +## Gotchas + +- The error is on **stderr**. Checking only stdout will miss it. +- Do not "fix" the fixture by deleting `task.toml` to make validate pass. Ambiguity is the feature. +- Harbor `task.toml` is not parsed in slice 1. Ambiguity is detected from well-known filenames before any Harbor parser exists. +- Exit 1 with a `static.manifest` FAIL would mean the CLI picked a format. That is a contract break. diff --git a/.cursor/skills/verify-openenv-validate/features/broken-manifest.md b/.cursor/skills/verify-openenv-validate/features/broken-manifest.md new file mode 100644 index 0000000000..122076b502 --- /dev/null +++ b/.cursor/skills/verify-openenv-validate/features/broken-manifest.md @@ -0,0 +1,32 @@ +# Broken manifest + +`openenv validate` on `tests/fixtures/validation/broken_manifest` treats an invalid `validation:` block as a graded failure: it prints a report that FAILs `static.manifest` and exits 1. + +## Sub-features + +- `broken-human` prints `FAIL static.manifest` and `Verdict: FAIL`. +- `broken-exit` exits 1 (FAIL), not 2 (signature) or 3 (internal). +- `broken-json` emits a schema-valid report with `verdict` `fail` and `manifest` null or a failed check naming the schema problem. + +## How to get to it (user POV) + +- Run `openenv validate tests/fixtures/validation/broken_manifest --level static --skip-build`. +- Add `--json` when asserting the report object. + +## Driving it with validate-run + +Preconditions: + +- Doctor exited 0. +- `tests/fixtures/validation/broken_manifest/openenv.yaml` exists (reward range `[1.0, 0.0]`). + +- **Human FAIL.** Run `EXPECTED_EXIT=1 bash "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/validate-run.sh" broken-manifest -- validate tests/fixtures/validation/broken_manifest --level static --skip-build`. Exit code `1`. Stdout contains `static.manifest` and `Verdict: FAIL`. +- **JSON FAIL.** Run `EXPECTED_EXIT=1 bash "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/validate-run.sh" broken-manifest-json -- validate tests/fixtures/validation/broken_manifest --level static --skip-build --json`. Exit code `1`. `check-report.py` on stdout accepts the JSON and prints `verdict=fail` and `static.manifest=fail`. +- **Proof.** Keep both artifact directories. This is a successful verification of a failing package: exit 1 is the expected user-visible result. + +## Gotchas + +- Do not treat exit 1 as "the CLI is broken." The CLI worked; the package failed the bar. +- A `ManifestError` is graded, not a `SignatureError`. Exit 2 here is a regression. +- Remediation text distinguishes a missing `validation:` block from an invalid one. This fixture is invalid (range inverted), not missing. +- `--json` still exits 1. Capture stdout anyway; the report is the proof. diff --git a/.cursor/skills/verify-openenv-validate/features/echo-env-static-validate.md b/.cursor/skills/verify-openenv-validate/features/echo-env-static-validate.md new file mode 100644 index 0000000000..d638e362fc --- /dev/null +++ b/.cursor/skills/verify-openenv-validate/features/echo-env-static-validate.md @@ -0,0 +1,37 @@ +# Echo env static validate + +`openenv validate` on `envs/echo_env` at the static level accepts the committed `openenv.yaml` `validation:` block, prints a PASS report, and exits 0. + +## Sub-features + +- `echo-human` prints the text report with `PASS static.manifest` and `Verdict: PASS`. +- `echo-json` prints a schema-valid `ValidationReport` whose `verdict` is `pass`. +- `echo-output` writes the same report to `--output` while still printing the human summary. + +## How to get to it (user POV) + +- From the repo root, run `openenv validate envs/echo_env --level static --skip-build`. +- Add `--json` for stdout JSON. +- Add `--output ` to write the JSON report to disk. + +## Driving it with validate-run + +Preconditions: + +- Doctor exited 0. +- `envs/echo_env/openenv.yaml` contains `name: echo_env` and a `validation:` block. +- `$SCRATCH_ROOT` exists and is writable. + +- **Human entry.** Validate echo_env as an author would. Run `bash "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/validate-run.sh" echo-env-static-validate -- validate envs/echo_env --level static --skip-build`. Exit code `0`. Stdout contains `Validation report for`, `PASS static.manifest`, `signature: openenv.yaml`, `levels run: static`, and `Verdict: PASS`. Stderr is empty or lacks `Error:`. +- **JSON entry.** Repeat with `--json`. Run `EXPECTED_EXIT=0 bash "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/validate-run.sh" echo-env-static-validate-json -- validate envs/echo_env --level static --skip-build --json`. Exit code `0`. Stdout is JSON. Run `uv run python "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/check-report.py" "$EVIDENCE_ROOT/echo-env-static-validate-json/stdout.txt"`. It prints `verdict=pass` and `static.manifest=pass`. +- **Output file.** Write a report beside the human printout. Run `EXPECTED_EXIT=0 bash "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/validate-run.sh" echo-env-static-validate-output -- validate envs/echo_env --level static --skip-build --output "$SCRATCH_ROOT/echo-env-report.json"`. Exit code `0`. Stdout is the human report (`Verdict: PASS`), not JSON. `$SCRATCH_ROOT/echo-env-report.json` exists. Copy it to `$EVIDENCE_ROOT/echo-env-static-validate-output/report.json` if the helper did not. `check-report.py` on that file prints `verdict=pass` and `static.manifest=pass`. +- **Proof.** Keep `$EVIDENCE_ROOT/echo-env-static-validate/`, `echo-env-static-validate-json/`, and `echo-env-static-validate-output/`. Each directory has `command.txt`, `stdout.txt`, `stderr.txt`, and `exit_code.txt`. + +## Gotchas + +- Default `--level` is `semantic`, not `static`. Slice 1 has no semantic graders; still pass `--level static` so the proof matches the checkpoint and `levels_run` is `[static]`. +- `--skip-build` does not skip a build today. Pass it anyway; the user path in the CLI help names it. +- `source_digest` and `duration_s` change every run. Do not pin them. +- `manifest` is present and non-null on this path. A null `manifest` means parse failed — that is a FAIL package, not echo_env. +- Do not use `PYTHONPATH=src:envs` as a substitute for `uv run`. The installed package is enough for `openenv validate`. +- A green `pytest tests/test_validation/test_checkpoints.py` is not this feature's proof. Drive the CLI. diff --git a/.cursor/skills/verify-openenv-validate/features/json-report-output.md b/.cursor/skills/verify-openenv-validate/features/json-report-output.md new file mode 100644 index 0000000000..e61b624c6a --- /dev/null +++ b/.cursor/skills/verify-openenv-validate/features/json-report-output.md @@ -0,0 +1,33 @@ +# JSON report output + +`--json` prints the validation report as schema-versioned JSON on stdout. `--output ` writes that JSON to disk. An author can pin `policy_version` and re-read the file as `ValidationReport`. + +## Sub-features + +- `json-stdout` replaces the human summary with JSON when `--json` is set. +- `json-file` writes JSON to `--output` even without `--json`. +- `json-schema` round-trips through `ValidationReport.model_validate_json`. + +## How to get to it (user POV) + +- Run `openenv validate envs/echo_env --level static --skip-build --json`. +- Run `openenv validate envs/echo_env --level static --skip-build --output report.json`. +- Combine `--json --output report.json` to print and write the same payload. + +## Driving it with validate-run + +Preconditions: + +- Doctor exited 0. +- Echo env static validate is reachable (same package as that feature). + +- **JSON stdout.** Run `EXPECTED_EXIT=0 bash "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/validate-run.sh" json-report-stdout -- validate envs/echo_env --level static --skip-build --json`. Exit code `0`. `check-report.py` on `$EVIDENCE_ROOT/json-report-stdout/stdout.txt` prints `verdict=pass` `static.manifest=pass` `schema=1`. +- **Output file only.** Run `EXPECTED_EXIT=0 bash "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/validate-run.sh" json-report-file -- validate envs/echo_env --level static --skip-build --output "$SCRATCH_ROOT/report.json"`. Exit code `0`. Stdout contains `Verdict: PASS` and does not start with `{`. `check-report.py "$SCRATCH_ROOT/report.json"` succeeds. Copy the file to `$EVIDENCE_ROOT/json-report-file/report.json` if needed. +- **Both.** Run `EXPECTED_EXIT=0 bash "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/validate-run.sh" json-report-both -- validate envs/echo_env --level static --skip-build --json --output "$SCRATCH_ROOT/report-both.json"`. Exit code `0`. Stdout JSON and the file both pass `check-report.py` with the same `verdict` and `check_id`s. +- **Proof.** Keep the three artifact directories. File proof is the bytes on disk, not the helper's return code alone. + +## Gotchas + +- Human mode plus `--output` still prints the text report. Do not require stdout to be JSON unless `--json` was passed. +- `--json` on a FAIL package still prints JSON and exits 1. Schema validity does not imply `verdict=pass`. +- Do not pretty-print or re-serialize the report before `check-report.py`. Validate the CLI's bytes. diff --git a/.cursor/skills/verify-openenv-validate/features/unrecognized-package.md b/.cursor/skills/verify-openenv-validate/features/unrecognized-package.md new file mode 100644 index 0000000000..44af01583f --- /dev/null +++ b/.cursor/skills/verify-openenv-validate/features/unrecognized-package.md @@ -0,0 +1,28 @@ +# Unrecognized package + +`openenv validate` refuses a directory with no well-known package file. It prints an unrecognized-package error on stderr and exits 2. + +## Sub-features + +- `unrecognized-refuse` exits 2 on `tests/fixtures/validation/unrecognized_package`. +- `unrecognized-stderr` says the package is unrecognized. +- `unrecognized-no-guess` does not invent an `openenv.yaml` or emit `Verdict:`. + +## How to get to it (user POV) + +- Run `openenv validate tests/fixtures/validation/unrecognized_package --level static --skip-build`. + +## Driving it with validate-run + +Preconditions: + +- Doctor exited 0. +- The fixture directory has no `openenv.yaml`, `task.toml`, or `task.md`. + +- **Signature refusal.** Run `EXPECTED_EXIT=2 bash "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/validate-run.sh" unrecognized-package -- validate tests/fixtures/validation/unrecognized_package --level static --skip-build`. Exit code `2`. Stderr contains `unrecognized` (case-insensitive). Stdout has no `Verdict:`. +- **Proof.** Keep `$EVIDENCE_ROOT/unrecognized-package/{command,stdout,stderr,exit_code}.txt`. + +## Gotchas + +- A Harbor-only `task.toml` package (`tests/fixtures/validation/harbor_task_min`) also exits 2 in slice 1 because no Harbor parser ships yet. That is "unsupported format", not this fixture. Do not use it as a stand-in for unrecognized. +- A missing directory (path does not exist) also exits 2 (`not a package directory`). That is a different message. Use the committed empty-ish fixture. From 06a4be729ff0cd7e3a020fb649ba9621e759597c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 26 Aug 2026 02:03:53 +0000 Subject: [PATCH 05/14] refactor(validation): drop sermons and a duplicate CLI fixture Slice 1 already proves the pipeline. Module essays, later-slice comments, and a second invented happy-path fixture made the walking skeleton heavier to read without changing the contract. - Trim module and helper prose in runner, signature, parser, StaticManifestGrader, and the validate CLI. - Delete the tmp-dir `--json` happy path from tests/test_cli/test_validate.py. served_min_pass and test_cli_validate already pin that. - Keep the missing-`validation:` CLI test. That path has no fixture. ParserRegistry, GraderRegistry, skip_build, and the verify-openenv-validate skill stay. They are the spine later slices register into, not leftover weight. openenv validate behavior is unchanged. Authors still get the same 0/1/2/3 exits and report text. Pre-change pin: echo_env exit 0, broken_manifest exit 1, ambiguous and unrecognized exit 2, harbor_task_min unrecognized, cosmic --level exit 3. Tests and the verify skill run after this commit. Co-authored-by: Zach Wentz --- src/openenv/cli/commands/validate.py | 15 ++---- .../validation/graders/static/manifest.py | 10 +--- .../validation/parsers/openenv_yaml.py | 3 -- src/openenv/validation/runner.py | 8 +--- src/openenv/validation/signature.py | 2 - tests/test_cli/test_validate.py | 46 ------------------- .../test_openenv_yaml_parser.py | 2 - .../test_signature_detection.py | 4 -- 8 files changed, 7 insertions(+), 83 deletions(-) diff --git a/src/openenv/cli/commands/validate.py b/src/openenv/cli/commands/validate.py index 6defaeea44..ac6a3bc444 100644 --- a/src/openenv/cli/commands/validate.py +++ b/src/openenv/cli/commands/validate.py @@ -3,9 +3,7 @@ """ OpenEnv validate command. -Local packages run through the RFC 008 validation pipeline (signature → parse → -grade → severity policy → report). Running servers are probed via --url until the -runtime graders land. +Local packages run the RFC 008 pipeline. `--url` still probes a running server. """ import json @@ -27,11 +25,10 @@ ) -# Exit-code contract (RFC 008): -EXIT_PASS = 0 # verdict PASS or WARN -EXIT_FAIL = 1 # verdict FAIL -EXIT_UNSUPPORTED = 2 # SignatureError / UnsupportedPackageError -EXIT_INTERNAL = 3 # internal error +EXIT_PASS = 0 +EXIT_FAIL = 1 +EXIT_UNSUPPORTED = 2 +EXIT_INTERNAL = 3 _LEVELS = { "static": Level.STATIC, @@ -41,13 +38,11 @@ def _looks_like_url(value: str) -> bool: - """Return True when the value appears to be a URL target.""" candidate = value.strip().lower() return candidate.startswith("http://") or candidate.startswith("https://") def _render_report(report: ValidationReport) -> str: - """Human-readable report summary.""" lines = [ f"Validation report for {report.target} (signature: {report.signature.value})", f" policy {report.policy_version} · levels run: " diff --git a/src/openenv/validation/graders/static/manifest.py b/src/openenv/validation/graders/static/manifest.py index 6224e33fdb..fabc0239d8 100644 --- a/src/openenv/validation/graders/static/manifest.py +++ b/src/openenv/validation/graders/static/manifest.py @@ -9,15 +9,7 @@ class StaticManifestGrader: - """ - Checks the parsed manifest's declared tolerances against the policy bounds. - - Schema validity itself is established before this grader runs (an unparseable - manifest never reaches grading — the runner records the `static.manifest` FAIL - directly from the parser's [`~openenv.validation.manifest.ManifestError`]). What - remains to grade is that author-declared values sit within what the severity - policy allows an author to declare. - """ + """Checks declared tolerances against the severity policy bounds.""" check_id = "static.manifest" level = Level.STATIC diff --git a/src/openenv/validation/parsers/openenv_yaml.py b/src/openenv/validation/parsers/openenv_yaml.py index 7108682bde..1e63634aad 100644 --- a/src/openenv/validation/parsers/openenv_yaml.py +++ b/src/openenv/validation/parsers/openenv_yaml.py @@ -77,9 +77,6 @@ def parse(self, package_root: Path) -> NormalizedManifest: "judge": validation.get("judge"), "task_distribution": validation.get("task_distribution"), } - # Omit absent required sections so schema errors read as "field required" - # rather than "not a valid dictionary". Omit absent `network` so the - # manifest default (mode public) applies. for key, value in ( ("name", raw.get("name")), ("reward", validation.get("reward")), diff --git a/src/openenv/validation/runner.py b/src/openenv/validation/runner.py index 1ee7b3ac38..5b3808066a 100644 --- a/src/openenv/validation/runner.py +++ b/src/openenv/validation/runner.py @@ -1,9 +1,4 @@ -"""Validation orchestration: parse → grade → apply policy → report. - -Levels run in order and accumulate into one report per run — a level with a -policy-fail finding still lets later levels run where dependencies permit, so an -author gets maximum information per run. -""" +"""Validation orchestration: parse → grade → apply policy → report.""" import hashlib import time @@ -61,7 +56,6 @@ def default_parser_registry() -> ParserRegistry: def _run_grader(grader, subject: Subject) -> CheckResult: - """Run one grader; a crash is an ERROR result (fails closed), never an abort.""" started = time.monotonic() try: return grader.run(subject) diff --git a/src/openenv/validation/signature.py b/src/openenv/validation/signature.py index 400e7ff056..4589d82302 100644 --- a/src/openenv/validation/signature.py +++ b/src/openenv/validation/signature.py @@ -58,7 +58,6 @@ def __init__(self, category: str, reason: str): def _has_yaml_frontmatter(path: Path) -> bool: - """Return True when `task.md` starts with YAML frontmatter (`---`).""" try: return path.read_text(encoding="utf-8").lstrip().startswith("---") except OSError: @@ -66,7 +65,6 @@ def _has_yaml_frontmatter(path: Path) -> bool: def _file_present(package_root: Path, kind: SignatureKind) -> bool: - """Return True when this signature's well-known file is present and countable.""" path = package_root / kind.value if not path.is_file(): return False diff --git a/tests/test_cli/test_validate.py b/tests/test_cli/test_validate.py index 4b69d16a36..524152df8a 100644 --- a/tests/test_cli/test_validate.py +++ b/tests/test_cli/test_validate.py @@ -218,52 +218,6 @@ def test_validate_command_local_path_without_validation_block_fails( assert "validation" in result.stdout -def test_validate_command_local_json_output(tmp_path: Path) -> None: - """CLI --json emits a schema-valid ValidationReport for a local package.""" - env_dir = tmp_path / "test_env" - _write_minimal_valid_env(env_dir) - yaml_text = (env_dir / "openenv.yaml").read_text() - (env_dir / "openenv.yaml").write_text( - yaml_text - + "version: 0.1.0\n" - + "validation:\n" - + " reward:\n" - + " range: [0.0, 1.0]\n" - + " floor_margin: 0.5\n" - + " resources:\n" - + " cpu: 1.0\n" - + " memory_mb: 1024\n" - + " disk_mb: 512\n" - + " episode_timeout_s: 60.0\n" - + " capabilities:\n" - + " verifier:\n" - + " kind: reward_channel\n" - + " types:\n" - + " tags: [demo]\n" - ) - - result = subprocess.run( - [ - sys.executable, - "-m", - "openenv.cli", - "validate", - str(env_dir), - "--level", - "static", - "--skip-build", - "--json", - ], - capture_output=True, - text=True, - ) - - assert result.returncode == 0, result.stdout + result.stderr - payload = json.loads(result.stdout) - assert payload["report_schema_version"] == "1" - assert payload["verdict"] == "pass" - - def test_multi_mode_rejects_environment_package_as_runtime_dependency( tmp_path: Path, ) -> None: diff --git a/tests/test_validation/test_openenv_yaml_parser.py b/tests/test_validation/test_openenv_yaml_parser.py index 6e09d72c35..782c700286 100644 --- a/tests/test_validation/test_openenv_yaml_parser.py +++ b/tests/test_validation/test_openenv_yaml_parser.py @@ -38,8 +38,6 @@ def test_missing_validation_block_has_remediation(tmp_path): def test_parse_is_a_pure_read(tmp_path): - # Package code must never be imported or executed: a booby-trapped module - # alongside a valid openenv.yaml parses without tripping. src = (FIXTURES / "served_min_pass" / "openenv.yaml").read_text() (tmp_path / "openenv.yaml").write_text(src) (tmp_path / "server.py").write_text( diff --git a/tests/test_validation/test_signature_detection.py b/tests/test_validation/test_signature_detection.py index 9c89de9c48..f782bed4ee 100644 --- a/tests/test_validation/test_signature_detection.py +++ b/tests/test_validation/test_signature_detection.py @@ -17,15 +17,11 @@ def test_openenv_yaml_detects_the_served_format(): def test_the_table_lists_exactly_the_implemented_parsers(): - # Only the served-env parser exists in this build. task.toml joins with the - # Harbor parser and task.md with the PostTrain parser. assert set(WELL_KNOWN_FILES) == {SignatureKind.OPENENV_SERVED} @pytest.mark.parametrize("fixture", ["harbor_task_min", "posttrain_task_min"]) def test_formats_without_parsers_are_unrecognized_not_guessed(fixture): - # These packages are real formats, but this build cannot parse them — - # refusing as unrecognized beats claiming support. with pytest.raises(SignatureError, match="unrecognized"): detect_signature(FIXTURES / fixture) From 8e2cc288e6e415aeccbcb56ae4d7b1aa02a0e3c3 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 26 Aug 2026 02:06:34 +0000 Subject: [PATCH 06/14] chore: strip narration comments from the RFC 008 slice 1 skeleton Keep public API contract docstrings, SPDX headers, the PR2 unsupported-categories note, and the proven pydantic missing-vs-None omit in OpenEnvYamlParser.parse. Co-authored-by: Zach Wentz --- src/openenv/cli/commands/validate.py | 6 ------ .../validation/graders/static/__init__.py | 2 -- .../validation/graders/static/manifest.py | 2 -- src/openenv/validation/parsers/openenv_yaml.py | 6 +++--- src/openenv/validation/runner.py | 5 +---- tests/test_cli/test_validate.py | 18 ------------------ tests/test_validation/test_cli_validate.py | 2 -- .../test_openenv_yaml_parser.py | 2 -- tests/test_validation/test_runner.py | 2 -- .../test_signature_detection.py | 2 -- 10 files changed, 4 insertions(+), 43 deletions(-) diff --git a/src/openenv/cli/commands/validate.py b/src/openenv/cli/commands/validate.py index ac6a3bc444..bfb55bd88f 100644 --- a/src/openenv/cli/commands/validate.py +++ b/src/openenv/cli/commands/validate.py @@ -1,11 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -""" -OpenEnv validate command. - -Local packages run the RFC 008 pipeline. `--url` still probes a running server. -""" - import json from pathlib import Path from typing import Annotated diff --git a/src/openenv/validation/graders/static/__init__.py b/src/openenv/validation/graders/static/__init__.py index c4472ae049..fb198c488e 100644 --- a/src/openenv/validation/graders/static/__init__.py +++ b/src/openenv/validation/graders/static/__init__.py @@ -1,5 +1,3 @@ -"""Static-level (L1) graders.""" - from .manifest import StaticManifestGrader __all__ = ["StaticManifestGrader"] diff --git a/src/openenv/validation/graders/static/manifest.py b/src/openenv/validation/graders/static/manifest.py index fabc0239d8..d5e8677812 100644 --- a/src/openenv/validation/graders/static/manifest.py +++ b/src/openenv/validation/graders/static/manifest.py @@ -1,5 +1,3 @@ -"""The `static.manifest` grader: the manifest is schema-valid and within policy bounds.""" - import time from ...manifest import NormalizedManifest diff --git a/src/openenv/validation/parsers/openenv_yaml.py b/src/openenv/validation/parsers/openenv_yaml.py index 1e63634aad..0fcc156f4b 100644 --- a/src/openenv/validation/parsers/openenv_yaml.py +++ b/src/openenv/validation/parsers/openenv_yaml.py @@ -1,5 +1,3 @@ -"""Parser for the served OpenEnv format (`openenv.yaml`).""" - from pathlib import Path import yaml @@ -22,7 +20,6 @@ def _format_validation_error(exc: ValidationError) -> list[str]: - """Render pydantic errors as `field: message` evidence lines.""" lines = [] for err in exc.errors(): loc = ".".join(str(part) for part in err["loc"]) or "manifest" @@ -77,6 +74,9 @@ def parse(self, package_root: Path) -> NormalizedManifest: "judge": validation.get("judge"), "task_distribution": validation.get("task_distribution"), } + # Omit absent required sections so schema errors read as "field required" + # rather than "not a valid dictionary". Omit absent `network` so the + # manifest default (mode public) applies. for key, value in ( ("name", raw.get("name")), ("reward", validation.get("reward")), diff --git a/src/openenv/validation/runner.py b/src/openenv/validation/runner.py index 5b3808066a..3f9f194404 100644 --- a/src/openenv/validation/runner.py +++ b/src/openenv/validation/runner.py @@ -1,5 +1,3 @@ -"""Validation orchestration: parse → grade → apply policy → report.""" - import hashlib import time from pathlib import Path @@ -59,7 +57,7 @@ def _run_grader(grader, subject: Subject) -> CheckResult: started = time.monotonic() try: return grader.run(subject) - except Exception as exc: # noqa: BLE001 — grader crash is evidence, not fatal + except Exception as exc: return CheckResult( check_id=grader.check_id, status=CheckStatus.ERROR, @@ -92,7 +90,6 @@ def run_validation( Level ceiling; graders above it are not selected. skip_build (`bool`, *optional*, defaults to `False`): Skip the image build; build-dependent checks SKIP with a reason. - Slice 1 has no build-dependent graders, so this is accepted and unused. policy ([`~openenv.validation.policy.SeverityPolicy`], *optional*): Severity policy; `None` loads the committed default version. diff --git a/tests/test_cli/test_validate.py b/tests/test_cli/test_validate.py index 524152df8a..d897572457 100644 --- a/tests/test_cli/test_validate.py +++ b/tests/test_cli/test_validate.py @@ -1,7 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -"""Tests for the openenv validate command and runtime validation utilities.""" - from __future__ import annotations import json @@ -22,8 +20,6 @@ class _MockResponse: - """Minimal mock response object for requests.get/post tests.""" - def __init__(self, status_code: int, payload: dict | None = None): self.status_code = status_code self._payload = payload @@ -40,7 +36,6 @@ def _write_minimal_valid_env( main_signature: str = "def main():", main_invocation: str = "main()", ) -> None: - """Create a minimal local environment that passes local validation.""" (env_dir / "server").mkdir(parents=True) (env_dir / "openenv.yaml").write_text( @@ -62,8 +57,6 @@ def _write_minimal_valid_env( def test_validate_running_environment_success() -> None: - """Runtime validator returns passing criteria for a conforming server.""" - def _fake_get(url: str, timeout: float) -> _MockResponse: if url.endswith("/openapi.json"): return _MockResponse( @@ -118,8 +111,6 @@ def _fake_post(url: str, json: dict, timeout: float) -> _MockResponse: def test_validate_running_environment_failure() -> None: - """Runtime validator marks report as failed when criteria fail.""" - def _fake_get(url: str, timeout: float) -> _MockResponse: if url.endswith("/openapi.json"): return _MockResponse( @@ -171,7 +162,6 @@ def _fake_post(url: str, json: dict, timeout: float) -> _MockResponse: def test_validate_command_runtime_target_outputs_json() -> None: - """CLI validates runtime targets and prints JSON report.""" mock_report = { "target": "https://example.com", "validation_type": "running_environment", @@ -194,7 +184,6 @@ def test_validate_command_runtime_target_outputs_json() -> None: def test_validate_command_local_path_without_validation_block_fails( tmp_path: Path, ) -> None: - """Packages without a `validation:` block fail static.manifest (exit 1).""" env_dir = tmp_path / "test_env" _write_minimal_valid_env(env_dir) @@ -221,7 +210,6 @@ def test_validate_command_local_path_without_validation_block_fails( def test_multi_mode_rejects_environment_package_as_runtime_dependency( tmp_path: Path, ) -> None: - """An openenv-* environment package is not the OpenEnv runtime package.""" env_dir = tmp_path / "test_env" _write_minimal_valid_env(env_dir) (env_dir / "pyproject.toml").write_text( @@ -243,7 +231,6 @@ def test_multi_mode_rejects_environment_package_as_runtime_dependency( def test_multi_mode_accepts_dockerfile_managed_openenv_runtime( tmp_path: Path, ) -> None: - """Multi-mode helper accepts envs that install OpenEnv in Dockerfile.""" env_dir = tmp_path / "test_env" _write_minimal_valid_env(env_dir) (env_dir / "pyproject.toml").write_text( @@ -266,7 +253,6 @@ def test_multi_mode_accepts_dockerfile_managed_openenv_runtime( def test_multi_mode_accepts_main_call_with_arguments(tmp_path: Path) -> None: - """Multi-mode helper accepts a guarded main(...) call with arguments.""" env_dir = tmp_path / "test_env" _write_minimal_valid_env( env_dir, @@ -281,7 +267,6 @@ def test_multi_mode_accepts_main_call_with_arguments(tmp_path: Path) -> None: def test_multi_mode_rejects_nested_main_guard(tmp_path: Path) -> None: - """Multi-mode helper requires the __main__ guard at module scope.""" env_dir = tmp_path / "test_env" _write_minimal_valid_env(env_dir) (env_dir / "server" / "app.py").write_text( @@ -298,7 +283,6 @@ def test_multi_mode_rejects_nested_main_guard(tmp_path: Path) -> None: def test_multi_mode_accepts_later_top_level_main_guard(tmp_path: Path) -> None: - """Multi-mode helper scans each top-level __main__ guard for a main() call.""" env_dir = tmp_path / "test_env" _write_minimal_valid_env(env_dir) (env_dir / "server" / "app.py").write_text( @@ -318,7 +302,6 @@ def test_multi_mode_accepts_later_top_level_main_guard(tmp_path: Path) -> None: def test_multi_mode_syntax_error_fallback_requires_dunder_main( tmp_path: Path, ) -> None: - """Syntax-error fallback still requires the literal __main__ guard string.""" env_dir = tmp_path / "test_env" _write_minimal_valid_env(env_dir) (env_dir / "server" / "app.py").write_text( @@ -332,7 +315,6 @@ def test_multi_mode_syntax_error_fallback_requires_dunder_main( def test_validate_command_rejects_mixed_path_and_url(tmp_path: Path) -> None: - """CLI rejects mixing a local path argument with --url mode.""" env_dir = tmp_path / "test_env" _write_minimal_valid_env(env_dir) diff --git a/tests/test_validation/test_cli_validate.py b/tests/test_validation/test_cli_validate.py index dcdc33f2ad..ba6555d3f7 100644 --- a/tests/test_validation/test_cli_validate.py +++ b/tests/test_validation/test_cli_validate.py @@ -1,5 +1,3 @@ -"""CLI contract: one command, exit codes 0/1/2/3, schema-valid JSON reports.""" - import subprocess import sys from pathlib import Path diff --git a/tests/test_validation/test_openenv_yaml_parser.py b/tests/test_validation/test_openenv_yaml_parser.py index 782c700286..6a750be3cd 100644 --- a/tests/test_validation/test_openenv_yaml_parser.py +++ b/tests/test_validation/test_openenv_yaml_parser.py @@ -1,5 +1,3 @@ -"""OpenEnvYamlParser: openenv.yaml maps onto the committed served-min-pass golden.""" - import pytest from conftest import FIXTURES, load_fixture_manifest from openenv.validation.manifest import ManifestError, NormalizedManifest diff --git a/tests/test_validation/test_runner.py b/tests/test_validation/test_runner.py index 2c6491adb2..6e2b1fd0bf 100644 --- a/tests/test_validation/test_runner.py +++ b/tests/test_validation/test_runner.py @@ -1,5 +1,3 @@ -"""run_validation: parse → grade → policy → report, one report per run.""" - import shutil import pytest diff --git a/tests/test_validation/test_signature_detection.py b/tests/test_validation/test_signature_detection.py index f782bed4ee..08897f97ce 100644 --- a/tests/test_validation/test_signature_detection.py +++ b/tests/test_validation/test_signature_detection.py @@ -1,5 +1,3 @@ -"""detect_signature: exactly one well-known file of a parseable format, never a guess.""" - import pytest from conftest import FIXTURES from openenv.validation.signature import ( From 2200d9eaf034e379149d187150264c429e4635d2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 26 Aug 2026 02:07:32 +0000 Subject: [PATCH 07/14] refactor(validation): keep package one-liners, drop the parser alibi The comment-strip commit deleted the one-line module banners the rest of `openenv.validation` uses, and put back the omit-field sermon in OpenEnvYamlParser.parse. The omit is already proven by the schema-error tests. - Restore one-line module docstrings on runner, signature, parser, StaticManifestGrader, static graders package, and the validate command. - Delete the three-line omit-field comment in OpenEnvYamlParser.parse. No behavior change. ruff check and format on the touched files passed after the restore. Co-authored-by: Zach Wentz --- src/openenv/cli/commands/validate.py | 2 ++ src/openenv/validation/graders/static/__init__.py | 2 ++ src/openenv/validation/graders/static/manifest.py | 2 ++ src/openenv/validation/parsers/openenv_yaml.py | 5 ++--- src/openenv/validation/runner.py | 2 ++ 5 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/openenv/cli/commands/validate.py b/src/openenv/cli/commands/validate.py index bfb55bd88f..a5497bc0d4 100644 --- a/src/openenv/cli/commands/validate.py +++ b/src/openenv/cli/commands/validate.py @@ -1,5 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause +"""OpenEnv validate command.""" + import json from pathlib import Path from typing import Annotated diff --git a/src/openenv/validation/graders/static/__init__.py b/src/openenv/validation/graders/static/__init__.py index fb198c488e..ea1e699d46 100644 --- a/src/openenv/validation/graders/static/__init__.py +++ b/src/openenv/validation/graders/static/__init__.py @@ -1,3 +1,5 @@ +"""Static-level graders.""" + from .manifest import StaticManifestGrader __all__ = ["StaticManifestGrader"] diff --git a/src/openenv/validation/graders/static/manifest.py b/src/openenv/validation/graders/static/manifest.py index d5e8677812..0b5276f18a 100644 --- a/src/openenv/validation/graders/static/manifest.py +++ b/src/openenv/validation/graders/static/manifest.py @@ -1,3 +1,5 @@ +"""The `static.manifest` grader.""" + import time from ...manifest import NormalizedManifest diff --git a/src/openenv/validation/parsers/openenv_yaml.py b/src/openenv/validation/parsers/openenv_yaml.py index 0fcc156f4b..39a27ab3ee 100644 --- a/src/openenv/validation/parsers/openenv_yaml.py +++ b/src/openenv/validation/parsers/openenv_yaml.py @@ -1,3 +1,5 @@ +"""Parser for the served OpenEnv format (`openenv.yaml`).""" + from pathlib import Path import yaml @@ -74,9 +76,6 @@ def parse(self, package_root: Path) -> NormalizedManifest: "judge": validation.get("judge"), "task_distribution": validation.get("task_distribution"), } - # Omit absent required sections so schema errors read as "field required" - # rather than "not a valid dictionary". Omit absent `network` so the - # manifest default (mode public) applies. for key, value in ( ("name", raw.get("name")), ("reward", validation.get("reward")), diff --git a/src/openenv/validation/runner.py b/src/openenv/validation/runner.py index 3f9f194404..583edf6219 100644 --- a/src/openenv/validation/runner.py +++ b/src/openenv/validation/runner.py @@ -1,3 +1,5 @@ +"""Validation orchestration: parse → grade → apply policy → report.""" + import hashlib import time from pathlib import Path From 23442b3388194dc7623c19e17f54dd25dd616479 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 26 Aug 2026 02:08:28 +0000 Subject: [PATCH 08/14] fix(validation): point Subject.outputs_dir at package/outputs ## Why source_digest already excludes an outputs/ directory. The runner still handed graders the package root, so the first replay writer would mutate the tree and change the next digest. ## Scope run_validation now sets Subject.outputs_dir to target / "outputs". Slice 1 still does not create that directory. ## Blast Radius No grader reads or writes outputs_dir today. echo_env and fixture validates are unchanged. ## Verification Convention already pinned in tests/test_validation/test_protocols.py. Runner and CLI pins follow this commit. Co-authored-by: Zach Wentz --- src/openenv/validation/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/openenv/validation/runner.py b/src/openenv/validation/runner.py index 583edf6219..7b79673968 100644 --- a/src/openenv/validation/runner.py +++ b/src/openenv/validation/runner.py @@ -130,7 +130,7 @@ def run_validation( manifest=manifest, image_ref=None, running=None, - outputs_dir=target, + outputs_dir=target / "outputs", ) results.extend( _run_grader(grader, subject) From 4f1c22ab6e69e540dcb62b2fc292f24bbc3d85e2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 26 Aug 2026 14:35:32 +0000 Subject: [PATCH 09/14] revert: drop verify-openenv-validate from the walking skeleton ## Why Slice 1 should ship the CLI pipeline only. The Cursor skill can stack on the next PR. ## Scope Removes .cursor/skills/verify-openenv-validate/. Checkpoints and pytest still pin the CLI. ## Blast Radius Authors and CI are unchanged. Agents lose this skill until the follow-up lands. Co-authored-by: Zach Wentz --- .../skills/verify-openenv-validate/SKILL.md | 114 ------------------ .../artifacts/.gitignore | 2 - .../bin/check-report.py | 31 ----- .../verify-openenv-validate/bin/cleanup.sh | 31 ----- .../verify-openenv-validate/bin/doctor.sh | 55 --------- .../bin/validate-run.sh | 64 ---------- .../features/README.md | 45 ------- .../features/ambiguous-package.md | 30 ----- .../features/broken-manifest.md | 32 ----- .../features/echo-env-static-validate.md | 37 ------ .../features/json-report-output.md | 33 ----- .../features/unrecognized-package.md | 28 ----- 12 files changed, 502 deletions(-) delete mode 100644 .cursor/skills/verify-openenv-validate/SKILL.md delete mode 100644 .cursor/skills/verify-openenv-validate/artifacts/.gitignore delete mode 100755 .cursor/skills/verify-openenv-validate/bin/check-report.py delete mode 100755 .cursor/skills/verify-openenv-validate/bin/cleanup.sh delete mode 100755 .cursor/skills/verify-openenv-validate/bin/doctor.sh delete mode 100755 .cursor/skills/verify-openenv-validate/bin/validate-run.sh delete mode 100644 .cursor/skills/verify-openenv-validate/features/README.md delete mode 100644 .cursor/skills/verify-openenv-validate/features/ambiguous-package.md delete mode 100644 .cursor/skills/verify-openenv-validate/features/broken-manifest.md delete mode 100644 .cursor/skills/verify-openenv-validate/features/echo-env-static-validate.md delete mode 100644 .cursor/skills/verify-openenv-validate/features/json-report-output.md delete mode 100644 .cursor/skills/verify-openenv-validate/features/unrecognized-package.md diff --git a/.cursor/skills/verify-openenv-validate/SKILL.md b/.cursor/skills/verify-openenv-validate/SKILL.md deleted file mode 100644 index 61e67bc1a3..0000000000 --- a/.cursor/skills/verify-openenv-validate/SKILL.md +++ /dev/null @@ -1,114 +0,0 @@ ---- -name: verify-openenv-validate -description: "Drive the OpenEnv CLI `openenv validate` the way an author does: local package path, stdout/stderr, exit codes, and a schema-valid ValidationReport. Use when proving echo_env or fixture validation, checking RFC 008 slice-1 checkpoints, or verifying validate CLI behavior after validation-pipeline changes." ---- - -# Verify `openenv validate` - -Short-lived CLI. There is no always-on server for local static validation. Launch means the checkout can run `openenv validate`; each drive is one isolated command that writes its own evidence. - -Primary user surface: `uv run openenv validate `. The equivalent spawn used by this repo's checkpoint tests is `uv run python -m openenv.cli validate `. Prefer the user-facing `openenv` form unless a recipe names the module form. - -`--url` probes a running FastAPI env (legacy runtime). That is a different surface; do not treat a live `echo_env` server as proof of local static validation. - -## Launch - -From the OpenEnv repo root (`/workspace` in this checkout): - -```bash -export REPO_ROOT="$(git rev-parse --show-toplevel)" -export RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$" -export EVIDENCE_ROOT="$REPO_ROOT/.cursor/skills/verify-openenv-validate/artifacts/$RUN_ID" -export SCRATCH_ROOT="/tmp/openenv-validate-verify-$RUN_ID" -mkdir -p "$EVIDENCE_ROOT" "$SCRATCH_ROOT" -export PATH="$HOME/.local/bin:$PATH" -cd "$REPO_ROOT" -``` - -Ready when this exits 0 and prints the validate help (must mention `--level` and `--skip-build`): - -```bash -uv run openenv validate --help -``` - -If `uv` is missing, install it to `~/.local/bin` and re-run. If the import fails, `uv sync --all-extras` from `$REPO_ROOT` (core extras are enough for this CLI). Do not start `echo_env.server.app` for static recipes. - -Teardown is Cleanup below. Launch does not leave a daemon. - -## Doctor - -Read-only. Run first whenever anything looks off: - -```bash -bash "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/doctor.sh" -``` - -Doctor must print `ok` for each of: `uv` on PATH, `openenv validate --help` exits 0, `envs/echo_env/openenv.yaml` exists and contains a `validation:` block, and `ValidationReport` imports. Exit 0 means the instance is worth driving. Exit 1 means stop and fix; do not drive a half-imported CLI. - -Two static validates may run side by side. They only read package trees and write reports under `$EVIDENCE_ROOT` / `$SCRATCH_ROOT`. Never write into `envs/echo_env/` or `tests/fixtures/validation/`. - -## Drive - -Read `features/README.md`, then the matching feature file. Follow that file's harness section literally. - -Harness is the CLI helper (tmux optional; the helper already isolates stdout/stderr/exit): - -```bash -bash "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/validate-run.sh" \ - -- \ - validate [flags] -``` - -That runs `uv run openenv …` from `$REPO_ROOT`, writes `stdout.txt`, `stderr.txt`, `exit_code.txt`, `command.txt` under `$EVIDENCE_ROOT//`, and copies `--output` reports into the same directory when the recipe names one. - -Stable handles (assert these, not durations or digest strings): - -- Human stdout: `Validation report for `, `PASS static.manifest`, `Verdict: PASS` / `Verdict: FAIL` -- JSON stdout or `--output` file: `report_schema_version` is `"1"`, `verdict` is `pass`/`fail`, a `results[]` entry has `check_id` `static.manifest` -- Exit codes: `0` PASS/WARN, `1` FAIL, `2` SignatureError / unrecognized / unsupported, `3` PolicyError / unknown `--level` -- Signature-error text goes to **stderr** (`ambiguous`, `unrecognized`), not stdout -- Schema-valid JSON means `ValidationReport.model_validate_json(...)` succeeds - -Slice 1 only implements `static.manifest`. `--level static --skip-build` is the inner-loop contract. `--skip-build` is a no-op until build graders exist; still pass it so the user path matches the checkpoint. - -## Evidence - -Proof lives under `$EVIDENCE_ROOT` (see Launch). Cleanup must not delete it. - -Standards: - -- Drive the real CLI (`uv run openenv validate …`), not `CliRunner`, not `run_validation()` as a library call, not pytest as a substitute for the user path. -- Capture the command, stdout, stderr, and exit code for every drive. A passing pytest file is not this skill's proof. -- For PASS packages, also capture a JSON report (`--json` and/or `--output`) and validate it with `ValidationReport`. -- For FAIL packages, human stdout must name `static.manifest` and the process must exit 1. -- For signature refusals, stderr must name the reason and the process must exit 2. There is no report file. -- `--output PATH` proof is the file on disk, not only stdout. Human mode still prints the text report; `--json` prints the JSON instead. -- Do not mock the filesystem package. Use `envs/echo_env` and `tests/fixtures/validation/*` as committed. - -After a JSON report is written, check it: - -```bash -uv run python "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/check-report.py" \ - "$EVIDENCE_ROOT//report.json" -``` - -## Cleanup - -```bash -bash "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/cleanup.sh" -``` - -Removes only `$SCRATCH_ROOT` (`/tmp/openenv-validate-verify-$RUN_ID`). Does not kill by process name. Does not delete `$EVIDENCE_ROOT`. After cleanup, `ls "$EVIDENCE_ROOT/"` must still show the proof files. - -## Helpers - -All scripts are executable. Invoke them as shown; do not invent flags. - -| Script | Purpose | -| --- | --- | -| `bin/doctor.sh` | Read-only readiness. No args. Exit 0/1. | -| `bin/validate-run.sh -- ` | One isolated `uv run openenv` capture. | -| `bin/check-report.py ` | `ValidationReport` parse; prints verdict and `static.manifest` status. | -| `bin/cleanup.sh` | Delete scratch dir only. | - -`validate-run.sh` requires `REPO_ROOT`, `EVIDENCE_ROOT` from Launch. Optional `EXPECTED_EXIT` (default: do not assert). If `EXPECTED_EXIT` is set and the process exits otherwise, the helper exits 1 after still writing the capture files. diff --git a/.cursor/skills/verify-openenv-validate/artifacts/.gitignore b/.cursor/skills/verify-openenv-validate/artifacts/.gitignore deleted file mode 100644 index d6b7ef32c8..0000000000 --- a/.cursor/skills/verify-openenv-validate/artifacts/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore diff --git a/.cursor/skills/verify-openenv-validate/bin/check-report.py b/.cursor/skills/verify-openenv-validate/bin/check-report.py deleted file mode 100755 index e88a8af7e5..0000000000 --- a/.cursor/skills/verify-openenv-validate/bin/check-report.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python3 -"""Parse a ValidationReport JSON file and print stable fields.""" - -from __future__ import annotations - -import sys -from pathlib import Path - -from openenv.validation.report import ValidationReport - - -def main() -> int: - if len(sys.argv) != 2: - print("usage: check-report.py ", file=sys.stderr) - return 2 - path = Path(sys.argv[1]) - report = ValidationReport.model_validate_json(path.read_text()) - statuses = {item.check_id: item.status.value for item in report.results} - print(f"schema={report.report_schema_version}") - print(f"target={report.target}") - print(f"signature={report.signature.value}") - print(f"policy={report.policy_version}") - print(f"verdict={report.verdict.value}") - print(f"levels={[level.name.lower() for level in report.levels_run]}") - print(f"static.manifest={statuses.get('static.manifest', 'absent')}") - print(f"manifest={'present' if report.manifest is not None else 'null'}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/.cursor/skills/verify-openenv-validate/bin/cleanup.sh b/.cursor/skills/verify-openenv-validate/bin/cleanup.sh deleted file mode 100755 index 89bb0c1593..0000000000 --- a/.cursor/skills/verify-openenv-validate/bin/cleanup.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -# Remove scratch created by this verification run. Never deletes EVIDENCE_ROOT. -set -euo pipefail - -if [[ -z "${SCRATCH_ROOT:-}" ]]; then - printf 'cleanup.sh needs SCRATCH_ROOT\n' >&2 - exit 2 -fi - -case "${SCRATCH_ROOT}" in - /tmp/openenv-validate-verify-*) - if [[ -d "${SCRATCH_ROOT}" ]]; then - rm -rf "${SCRATCH_ROOT}" - printf 'removed %s\n' "${SCRATCH_ROOT}" - else - printf 'no scratch at %s\n' "${SCRATCH_ROOT}" - fi - ;; - *) - printf 'refusing to remove unexpected SCRATCH_ROOT=%s\n' "${SCRATCH_ROOT}" >&2 - exit 2 - ;; -esac - -if [[ -n "${EVIDENCE_ROOT:-}" ]]; then - if [[ -d "${EVIDENCE_ROOT}" ]]; then - printf 'evidence kept at %s\n' "${EVIDENCE_ROOT}" - else - printf 'warning: EVIDENCE_ROOT missing after cleanup: %s\n' "${EVIDENCE_ROOT}" >&2 - fi -fi diff --git a/.cursor/skills/verify-openenv-validate/bin/doctor.sh b/.cursor/skills/verify-openenv-validate/bin/doctor.sh deleted file mode 100755 index ee371ed190..0000000000 --- a/.cursor/skills/verify-openenv-validate/bin/doctor.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -# Read-only readiness for verify-openenv-validate. No args. Exit 0 if worth driving. -set -euo pipefail - -fail() { - printf 'not ok %s\n' "$1" >&2 - exit 1 -} - -ok() { - printf 'ok %s\n' "$1" -} - -REPO_ROOT="${REPO_ROOT:-$(git rev-parse --show-toplevel 2>/dev/null || true)}" -if [[ -z "${REPO_ROOT}" || ! -d "${REPO_ROOT}/envs/echo_env" ]]; then - fail "REPO_ROOT is not an OpenEnv checkout (set REPO_ROOT or run from the repo)" -fi - -export PATH="${HOME}/.local/bin:${PATH}" - -if ! command -v uv >/dev/null 2>&1; then - fail "uv not on PATH" -fi -ok "uv $(command -v uv)" - -echo_yaml="${REPO_ROOT}/envs/echo_env/openenv.yaml" -if [[ ! -f "${echo_yaml}" ]]; then - fail "missing ${echo_yaml}" -fi -if ! grep -q '^validation:' "${echo_yaml}"; then - fail "${echo_yaml} has no validation: block" -fi -ok "envs/echo_env/openenv.yaml has validation:" - -cd "${REPO_ROOT}" -if ! uv run openenv validate --help >/tmp/openenv-validate-help.$$ 2>&1; then - fail "uv run openenv validate --help failed" -fi -if ! grep -q -- '--level' /tmp/openenv-validate-help.$$; then - rm -f /tmp/openenv-validate-help.$$ - fail "validate --help missing --level" -fi -if ! grep -q -- '--skip-build' /tmp/openenv-validate-help.$$; then - rm -f /tmp/openenv-validate-help.$$ - fail "validate --help missing --skip-build" -fi -rm -f /tmp/openenv-validate-help.$$ -ok "openenv validate --help" - -if ! uv run python -c "from openenv.validation.report import ValidationReport" >/dev/null 2>&1; then - fail "ValidationReport import failed" -fi -ok "ValidationReport import" - -printf 'doctor: ready\n' diff --git a/.cursor/skills/verify-openenv-validate/bin/validate-run.sh b/.cursor/skills/verify-openenv-validate/bin/validate-run.sh deleted file mode 100755 index c8a773f4ac..0000000000 --- a/.cursor/skills/verify-openenv-validate/bin/validate-run.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash -# Capture one `uv run openenv` invocation. -# Usage: validate-run.sh -- -# Requires REPO_ROOT and EVIDENCE_ROOT. Optional EXPECTED_EXIT. -set -euo pipefail - -if [[ $# -lt 3 || "$2" != "--" ]]; then - printf 'usage: validate-run.sh -- \n' >&2 - exit 2 -fi - -feature_id="$1" -shift 2 - -if [[ -z "${REPO_ROOT:-}" || -z "${EVIDENCE_ROOT:-}" ]]; then - printf 'validate-run.sh needs REPO_ROOT and EVIDENCE_ROOT\n' >&2 - exit 2 -fi - -out_dir="${EVIDENCE_ROOT}/${feature_id}" -mkdir -p "${out_dir}" - -export PATH="${HOME}/.local/bin:${PATH}" - -{ - printf 'cwd=%s\n' "${REPO_ROOT}" - printf 'uv run openenv' - for arg in "$@"; do - printf ' %q' "${arg}" - done - printf '\n' -} >"${out_dir}/command.txt" - -set +e -( - cd "${REPO_ROOT}" - uv run openenv "$@" -) >"${out_dir}/stdout.txt" 2>"${out_dir}/stderr.txt" -exit_code=$? -set -e - -printf '%s\n' "${exit_code}" >"${out_dir}/exit_code.txt" - -# If the user passed --output, copy the report next to the capture when it exists. -output_path="" -prev="" -for arg in "$@"; do - if [[ "${prev}" == "--output" ]]; then - output_path="${arg}" - fi - prev="${arg}" -done -if [[ -n "${output_path}" && -f "${output_path}" ]]; then - cp "${output_path}" "${out_dir}/report.json" -fi - -printf 'captured %s exit=%s\n' "${feature_id}" "${exit_code}" - -if [[ -n "${EXPECTED_EXIT:-}" && "${exit_code}" != "${EXPECTED_EXIT}" ]]; then - printf 'expected exit %s, got %s\n' "${EXPECTED_EXIT}" "${exit_code}" >&2 - exit 1 -fi - -exit 0 diff --git a/.cursor/skills/verify-openenv-validate/features/README.md b/.cursor/skills/verify-openenv-validate/features/README.md deleted file mode 100644 index a61ba5789c..0000000000 --- a/.cursor/skills/verify-openenv-validate/features/README.md +++ /dev/null @@ -1,45 +0,0 @@ -# `openenv validate` verification map - -This directory is the maintained source for verifying the user-facing behavior of `openenv validate`. Read this index before driving the CLI, then use the matching feature file as the recipe. - -## Baseline preconditions - -- Work from `$REPO_ROOT` (OpenEnv checkout). `uv` is on `PATH` (`~/.local/bin`). -- `bash .cursor/skills/verify-openenv-validate/bin/doctor.sh` exits 0. -- `EVIDENCE_ROOT` and `SCRATCH_ROOT` are set from the skill Launch section. -- Do not mutate `envs/echo_env/` or `tests/fixtures/validation/`. -- Do not start an environment server unless a feature file names `--url`. -- Treat every command as literal. Keep package paths and flags unchanged. - -## Driving conventions - -- Start every recipe from the baseline unless its preconditions say otherwise. -- Drive through `bin/validate-run.sh` so stdout, stderr, and the exit code are captured together. -- Prefer `uv run openenv validate …` (user path). The module form `python -m openenv.cli validate` is the same CLI; use it only when a recipe says so. -- Local static recipes always pass `--level static --skip-build`. -- Restore nothing after a drive: these commands are read-only on package trees. Scratch reports under `$SCRATCH_ROOT` are removed in cleanup. Proof under `$EVIDENCE_ROOT` stays. - -## Proof and skip reporting - -- CLI proof includes the command, stdout, stderr, and exit code. -- JSON proof is a file that `bin/check-report.py` accepts. -- Record the feature ID with every artifact directory name. -- Report an unreachable path with the attempted command and the unmet precondition. -- Do not report a skipped entry point as verified through a different path (human vs `--json` vs `--output` are distinct when the feature lists them). - -## Feature entry contract - -Each feature file starts with an H1 title and one paragraph describing the user-visible behavior. It then uses exactly four H2 sections in this order. - -1. `Sub-features` -2. `How to get to it (user POV)` -3. `Driving it with validate-run` -4. `Gotchas` - -## Features - -- [Echo env static validate](./echo-env-static-validate.md) is the slice-1 happy path: `envs/echo_env` exits 0 with a schema-valid report and `static.manifest` PASS. -- [Broken manifest](./broken-manifest.md) is a graded FAIL: invalid `validation:` exits 1 and names `static.manifest`. -- [Ambiguous package](./ambiguous-package.md) is a signature refusal: two well-known files exit 2 on stderr, no report. -- [JSON report output](./json-report-output.md) is the machine-readable path: `--json` and `--output` emit `ValidationReport`. -- [Unrecognized package](./unrecognized-package.md) is a signature refusal: no well-known file exits 2 on stderr. diff --git a/.cursor/skills/verify-openenv-validate/features/ambiguous-package.md b/.cursor/skills/verify-openenv-validate/features/ambiguous-package.md deleted file mode 100644 index 5d46818459..0000000000 --- a/.cursor/skills/verify-openenv-validate/features/ambiguous-package.md +++ /dev/null @@ -1,30 +0,0 @@ -# Ambiguous package - -`openenv validate` refuses `tests/fixtures/validation/ambiguous_package` because it carries two well-known files (`openenv.yaml` and `task.toml`). It prints a signature error on stderr and exits 2 without guessing a format. - -## Sub-features - -- `ambiguous-refuse` exits 2. -- `ambiguous-stderr` says the package is ambiguous. -- `ambiguous-no-report` does not print a `Verdict:` line or a `ValidationReport` JSON object. - -## How to get to it (user POV) - -- Run `openenv validate tests/fixtures/validation/ambiguous_package --level static --skip-build`. - -## Driving it with validate-run - -Preconditions: - -- Doctor exited 0. -- The fixture directory contains both `openenv.yaml` and `task.toml`. - -- **Signature refusal.** Run `EXPECTED_EXIT=2 bash "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/validate-run.sh" ambiguous-package -- validate tests/fixtures/validation/ambiguous_package --level static --skip-build`. Exit code `2`. Stderr contains `ambiguous` (case-insensitive). Stdout has no `Verdict:` and is not parseable as `ValidationReport`. -- **Proof.** Keep `$EVIDENCE_ROOT/ambiguous-package/{command,stdout,stderr,exit_code}.txt`. - -## Gotchas - -- The error is on **stderr**. Checking only stdout will miss it. -- Do not "fix" the fixture by deleting `task.toml` to make validate pass. Ambiguity is the feature. -- Harbor `task.toml` is not parsed in slice 1. Ambiguity is detected from well-known filenames before any Harbor parser exists. -- Exit 1 with a `static.manifest` FAIL would mean the CLI picked a format. That is a contract break. diff --git a/.cursor/skills/verify-openenv-validate/features/broken-manifest.md b/.cursor/skills/verify-openenv-validate/features/broken-manifest.md deleted file mode 100644 index 122076b502..0000000000 --- a/.cursor/skills/verify-openenv-validate/features/broken-manifest.md +++ /dev/null @@ -1,32 +0,0 @@ -# Broken manifest - -`openenv validate` on `tests/fixtures/validation/broken_manifest` treats an invalid `validation:` block as a graded failure: it prints a report that FAILs `static.manifest` and exits 1. - -## Sub-features - -- `broken-human` prints `FAIL static.manifest` and `Verdict: FAIL`. -- `broken-exit` exits 1 (FAIL), not 2 (signature) or 3 (internal). -- `broken-json` emits a schema-valid report with `verdict` `fail` and `manifest` null or a failed check naming the schema problem. - -## How to get to it (user POV) - -- Run `openenv validate tests/fixtures/validation/broken_manifest --level static --skip-build`. -- Add `--json` when asserting the report object. - -## Driving it with validate-run - -Preconditions: - -- Doctor exited 0. -- `tests/fixtures/validation/broken_manifest/openenv.yaml` exists (reward range `[1.0, 0.0]`). - -- **Human FAIL.** Run `EXPECTED_EXIT=1 bash "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/validate-run.sh" broken-manifest -- validate tests/fixtures/validation/broken_manifest --level static --skip-build`. Exit code `1`. Stdout contains `static.manifest` and `Verdict: FAIL`. -- **JSON FAIL.** Run `EXPECTED_EXIT=1 bash "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/validate-run.sh" broken-manifest-json -- validate tests/fixtures/validation/broken_manifest --level static --skip-build --json`. Exit code `1`. `check-report.py` on stdout accepts the JSON and prints `verdict=fail` and `static.manifest=fail`. -- **Proof.** Keep both artifact directories. This is a successful verification of a failing package: exit 1 is the expected user-visible result. - -## Gotchas - -- Do not treat exit 1 as "the CLI is broken." The CLI worked; the package failed the bar. -- A `ManifestError` is graded, not a `SignatureError`. Exit 2 here is a regression. -- Remediation text distinguishes a missing `validation:` block from an invalid one. This fixture is invalid (range inverted), not missing. -- `--json` still exits 1. Capture stdout anyway; the report is the proof. diff --git a/.cursor/skills/verify-openenv-validate/features/echo-env-static-validate.md b/.cursor/skills/verify-openenv-validate/features/echo-env-static-validate.md deleted file mode 100644 index d638e362fc..0000000000 --- a/.cursor/skills/verify-openenv-validate/features/echo-env-static-validate.md +++ /dev/null @@ -1,37 +0,0 @@ -# Echo env static validate - -`openenv validate` on `envs/echo_env` at the static level accepts the committed `openenv.yaml` `validation:` block, prints a PASS report, and exits 0. - -## Sub-features - -- `echo-human` prints the text report with `PASS static.manifest` and `Verdict: PASS`. -- `echo-json` prints a schema-valid `ValidationReport` whose `verdict` is `pass`. -- `echo-output` writes the same report to `--output` while still printing the human summary. - -## How to get to it (user POV) - -- From the repo root, run `openenv validate envs/echo_env --level static --skip-build`. -- Add `--json` for stdout JSON. -- Add `--output ` to write the JSON report to disk. - -## Driving it with validate-run - -Preconditions: - -- Doctor exited 0. -- `envs/echo_env/openenv.yaml` contains `name: echo_env` and a `validation:` block. -- `$SCRATCH_ROOT` exists and is writable. - -- **Human entry.** Validate echo_env as an author would. Run `bash "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/validate-run.sh" echo-env-static-validate -- validate envs/echo_env --level static --skip-build`. Exit code `0`. Stdout contains `Validation report for`, `PASS static.manifest`, `signature: openenv.yaml`, `levels run: static`, and `Verdict: PASS`. Stderr is empty or lacks `Error:`. -- **JSON entry.** Repeat with `--json`. Run `EXPECTED_EXIT=0 bash "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/validate-run.sh" echo-env-static-validate-json -- validate envs/echo_env --level static --skip-build --json`. Exit code `0`. Stdout is JSON. Run `uv run python "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/check-report.py" "$EVIDENCE_ROOT/echo-env-static-validate-json/stdout.txt"`. It prints `verdict=pass` and `static.manifest=pass`. -- **Output file.** Write a report beside the human printout. Run `EXPECTED_EXIT=0 bash "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/validate-run.sh" echo-env-static-validate-output -- validate envs/echo_env --level static --skip-build --output "$SCRATCH_ROOT/echo-env-report.json"`. Exit code `0`. Stdout is the human report (`Verdict: PASS`), not JSON. `$SCRATCH_ROOT/echo-env-report.json` exists. Copy it to `$EVIDENCE_ROOT/echo-env-static-validate-output/report.json` if the helper did not. `check-report.py` on that file prints `verdict=pass` and `static.manifest=pass`. -- **Proof.** Keep `$EVIDENCE_ROOT/echo-env-static-validate/`, `echo-env-static-validate-json/`, and `echo-env-static-validate-output/`. Each directory has `command.txt`, `stdout.txt`, `stderr.txt`, and `exit_code.txt`. - -## Gotchas - -- Default `--level` is `semantic`, not `static`. Slice 1 has no semantic graders; still pass `--level static` so the proof matches the checkpoint and `levels_run` is `[static]`. -- `--skip-build` does not skip a build today. Pass it anyway; the user path in the CLI help names it. -- `source_digest` and `duration_s` change every run. Do not pin them. -- `manifest` is present and non-null on this path. A null `manifest` means parse failed — that is a FAIL package, not echo_env. -- Do not use `PYTHONPATH=src:envs` as a substitute for `uv run`. The installed package is enough for `openenv validate`. -- A green `pytest tests/test_validation/test_checkpoints.py` is not this feature's proof. Drive the CLI. diff --git a/.cursor/skills/verify-openenv-validate/features/json-report-output.md b/.cursor/skills/verify-openenv-validate/features/json-report-output.md deleted file mode 100644 index e61b624c6a..0000000000 --- a/.cursor/skills/verify-openenv-validate/features/json-report-output.md +++ /dev/null @@ -1,33 +0,0 @@ -# JSON report output - -`--json` prints the validation report as schema-versioned JSON on stdout. `--output ` writes that JSON to disk. An author can pin `policy_version` and re-read the file as `ValidationReport`. - -## Sub-features - -- `json-stdout` replaces the human summary with JSON when `--json` is set. -- `json-file` writes JSON to `--output` even without `--json`. -- `json-schema` round-trips through `ValidationReport.model_validate_json`. - -## How to get to it (user POV) - -- Run `openenv validate envs/echo_env --level static --skip-build --json`. -- Run `openenv validate envs/echo_env --level static --skip-build --output report.json`. -- Combine `--json --output report.json` to print and write the same payload. - -## Driving it with validate-run - -Preconditions: - -- Doctor exited 0. -- Echo env static validate is reachable (same package as that feature). - -- **JSON stdout.** Run `EXPECTED_EXIT=0 bash "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/validate-run.sh" json-report-stdout -- validate envs/echo_env --level static --skip-build --json`. Exit code `0`. `check-report.py` on `$EVIDENCE_ROOT/json-report-stdout/stdout.txt` prints `verdict=pass` `static.manifest=pass` `schema=1`. -- **Output file only.** Run `EXPECTED_EXIT=0 bash "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/validate-run.sh" json-report-file -- validate envs/echo_env --level static --skip-build --output "$SCRATCH_ROOT/report.json"`. Exit code `0`. Stdout contains `Verdict: PASS` and does not start with `{`. `check-report.py "$SCRATCH_ROOT/report.json"` succeeds. Copy the file to `$EVIDENCE_ROOT/json-report-file/report.json` if needed. -- **Both.** Run `EXPECTED_EXIT=0 bash "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/validate-run.sh" json-report-both -- validate envs/echo_env --level static --skip-build --json --output "$SCRATCH_ROOT/report-both.json"`. Exit code `0`. Stdout JSON and the file both pass `check-report.py` with the same `verdict` and `check_id`s. -- **Proof.** Keep the three artifact directories. File proof is the bytes on disk, not the helper's return code alone. - -## Gotchas - -- Human mode plus `--output` still prints the text report. Do not require stdout to be JSON unless `--json` was passed. -- `--json` on a FAIL package still prints JSON and exits 1. Schema validity does not imply `verdict=pass`. -- Do not pretty-print or re-serialize the report before `check-report.py`. Validate the CLI's bytes. diff --git a/.cursor/skills/verify-openenv-validate/features/unrecognized-package.md b/.cursor/skills/verify-openenv-validate/features/unrecognized-package.md deleted file mode 100644 index 44af01583f..0000000000 --- a/.cursor/skills/verify-openenv-validate/features/unrecognized-package.md +++ /dev/null @@ -1,28 +0,0 @@ -# Unrecognized package - -`openenv validate` refuses a directory with no well-known package file. It prints an unrecognized-package error on stderr and exits 2. - -## Sub-features - -- `unrecognized-refuse` exits 2 on `tests/fixtures/validation/unrecognized_package`. -- `unrecognized-stderr` says the package is unrecognized. -- `unrecognized-no-guess` does not invent an `openenv.yaml` or emit `Verdict:`. - -## How to get to it (user POV) - -- Run `openenv validate tests/fixtures/validation/unrecognized_package --level static --skip-build`. - -## Driving it with validate-run - -Preconditions: - -- Doctor exited 0. -- The fixture directory has no `openenv.yaml`, `task.toml`, or `task.md`. - -- **Signature refusal.** Run `EXPECTED_EXIT=2 bash "$REPO_ROOT/.cursor/skills/verify-openenv-validate/bin/validate-run.sh" unrecognized-package -- validate tests/fixtures/validation/unrecognized_package --level static --skip-build`. Exit code `2`. Stderr contains `unrecognized` (case-insensitive). Stdout has no `Verdict:`. -- **Proof.** Keep `$EVIDENCE_ROOT/unrecognized-package/{command,stdout,stderr,exit_code}.txt`. - -## Gotchas - -- A Harbor-only `task.toml` package (`tests/fixtures/validation/harbor_task_min`) also exits 2 in slice 1 because no Harbor parser ships yet. That is "unsupported format", not this fixture. Do not use it as a stand-in for unrecognized. -- A missing directory (path does not exist) also exits 2 (`not a package directory`). That is a different message. Use the committed empty-ish fixture. From 3549bdaf07a390c296d62acfee696f8bad00adc1 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 26 Aug 2026 15:52:53 +0000 Subject: [PATCH 10/14] test(validation): pin echo_env on the CLI Real-package pin: openenv validate envs/echo_env exits 0 with a static.manifest PASS. Co-authored-by: Zach Wentz --- tests/test_validation/test_cli_validate.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_validation/test_cli_validate.py b/tests/test_validation/test_cli_validate.py index ba6555d3f7..47c7e8693f 100644 --- a/tests/test_validation/test_cli_validate.py +++ b/tests/test_validation/test_cli_validate.py @@ -25,6 +25,20 @@ def test_valid_package_exits_zero(): assert "Verdict: PASS" in result.stdout +def test_echo_env_exits_zero(): + result = _validate( + str(REPO_ROOT / "envs" / "echo_env"), + "--level", + "static", + "--skip-build", + "--json", + ) + assert result.returncode == 0, result.stdout + result.stderr + report = ValidationReport.model_validate_json(result.stdout) + assert report.verdict.value == "pass" + assert any(r.check_id == "static.manifest" for r in report.results) + + def test_json_report_is_schema_valid(): result = _validate( str(FIXTURES / "served_min_pass"), "--level", "static", "--skip-build", "--json" From 9379566188ea95e373eed756c52b6a41186f3f85 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 26 Aug 2026 17:01:50 +0000 Subject: [PATCH 11/14] docs(validation): restore the package docstring for the walking skeleton Slice 0a narrowed the contract docstring. This slice ships the pipeline. Co-authored-by: Zach Wentz --- src/openenv/validation/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/openenv/validation/__init__.py b/src/openenv/validation/__init__.py index eb577c78b3..135021d659 100644 --- a/src/openenv/validation/__init__.py +++ b/src/openenv/validation/__init__.py @@ -1,4 +1,9 @@ -"""RFC 008 validation contracts: types, [`~openenv.validation.manifest.NormalizedManifest`], registries, report, and severity policy.""" +"""RFC 008 environment validation: contracts and the local `openenv validate` pipeline. + +Ships signature detection, parsers, one normalized manifest, a grader registry, +a report schema, a versioned severity policy, and the local runner. Graders read +the manifest, never the signature. +""" from .graders import ENTRY_POINT_GROUP, Grader, GraderRegistry, Subject from .manifest import ( From 6c673a527c1c0ecc4a8165c013c96bf3adc1f29e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 26 Aug 2026 17:49:45 +0000 Subject: [PATCH 12/14] test(validation): pin WELL_KNOWN_FILES to the served parser The 0b pin was empty. This slice registers openenv.yaml. Co-authored-by: Zach Wentz --- tests/test_validation/test_types.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/test_validation/test_types.py b/tests/test_validation/test_types.py index 313e871df8..911eea0a23 100644 --- a/tests/test_validation/test_types.py +++ b/tests/test_validation/test_types.py @@ -1,5 +1,12 @@ from openenv.validation.signature import WELL_KNOWN_FILES -from openenv.validation.types import CheckStatus, Lane, Level, Severity, Verdict +from openenv.validation.types import ( + CheckStatus, + Lane, + Level, + Severity, + SignatureKind, + Verdict, +) def test_levels_are_ordered_by_cost(): @@ -22,5 +29,5 @@ def test_lanes(): assert {lane.value for lane in Lane} == {"local", "hub"} -def test_well_known_files_is_empty_until_a_parser_exists(): - assert WELL_KNOWN_FILES == {} +def test_well_known_files_lists_implemented_parsers(): + assert WELL_KNOWN_FILES == {SignatureKind.OPENENV_SERVED: "openenv.yaml"} From 7ac4edaf7a812e2d6c8b8d63998f42e72b7c937b Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 1 Sep 2026 12:18:56 +0200 Subject: [PATCH 13/14] fix: harden validation reporting --- src/openenv/cli/commands/validate.py | 2 +- src/openenv/validation/runner.py | 21 +++++++++---------- .../validation/served_min_pass/openenv.yaml | 1 - .../validation/unpinned_judge/openenv.yaml | 1 - tests/test_validation/test_cli_validate.py | 14 +++++++++++++ tests/test_validation/test_runner.py | 11 ++++++++++ 6 files changed, 36 insertions(+), 14 deletions(-) diff --git a/src/openenv/cli/commands/validate.py b/src/openenv/cli/commands/validate.py index a5497bc0d4..fb8741d13d 100644 --- a/src/openenv/cli/commands/validate.py +++ b/src/openenv/cli/commands/validate.py @@ -188,6 +188,7 @@ def validate( skip_build=skip_build, policy=load_policy(policy_version), ) + report_json = write_report(validation_report, output) except (SignatureError, UnsupportedPackageError) as exc: typer.echo(f"Error: {exc}", err=True) raise typer.Exit(EXIT_UNSUPPORTED) from exc @@ -198,7 +199,6 @@ def validate( typer.echo(f"Internal error: {exc}", err=True) raise typer.Exit(EXIT_INTERNAL) from exc - report_json = write_report(validation_report, output) if json_output: typer.echo(report_json) else: diff --git a/src/openenv/validation/runner.py b/src/openenv/validation/runner.py index 7b79673968..187010882e 100644 --- a/src/openenv/validation/runner.py +++ b/src/openenv/validation/runner.py @@ -31,17 +31,16 @@ def source_digest(package_root: Path) -> str: `str`: a 64-character hex digest. """ digest = hashlib.sha256() - files = sorted( - path - for path in package_root.rglob("*") - if path.is_file() - and not any( - part in _DIGEST_EXCLUDED_DIRS - for part in path.relative_to(package_root).parts - ) - ) - for path in files: - digest.update(str(path.relative_to(package_root)).encode()) + files = [] + for path in package_root.rglob("*"): + relative_path = path.relative_to(package_root) + if path.is_file() and not any( + part in _DIGEST_EXCLUDED_DIRS for part in relative_path.parts + ): + files.append((relative_path.as_posix(), path)) + + for relative_path, path in sorted(files, key=lambda item: item[0]): + digest.update(relative_path.encode()) digest.update(b"\0") digest.update(path.read_bytes()) digest.update(b"\0") diff --git a/tests/fixtures/validation/served_min_pass/openenv.yaml b/tests/fixtures/validation/served_min_pass/openenv.yaml index 3643b91a21..8ad02becb6 100644 --- a/tests/fixtures/validation/served_min_pass/openenv.yaml +++ b/tests/fixtures/validation/served_min_pass/openenv.yaml @@ -1,5 +1,4 @@ spec_version: 1 -version: 0.1.0 name: served-min-pass type: env runtime: python diff --git a/tests/fixtures/validation/unpinned_judge/openenv.yaml b/tests/fixtures/validation/unpinned_judge/openenv.yaml index a7d4cf2962..8cd7b5680f 100644 --- a/tests/fixtures/validation/unpinned_judge/openenv.yaml +++ b/tests/fixtures/validation/unpinned_judge/openenv.yaml @@ -1,5 +1,4 @@ spec_version: 1 -version: 0.1.0 name: unpinned-judge type: env runtime: python diff --git a/tests/test_validation/test_cli_validate.py b/tests/test_validation/test_cli_validate.py index 47c7e8693f..a1aee9c6ed 100644 --- a/tests/test_validation/test_cli_validate.py +++ b/tests/test_validation/test_cli_validate.py @@ -62,6 +62,20 @@ def test_output_writes_the_json_report(tmp_path): ValidationReport.model_validate_json(out.read_text()) +def test_output_write_failure_is_an_internal_error(tmp_path): + out = tmp_path / "missing" / "report.json" + result = _validate( + str(FIXTURES / "served_min_pass"), + "--level", + "static", + "--skip-build", + "--output", + str(out), + ) + assert result.returncode == 3, result.stdout + result.stderr + assert "Internal error:" in result.stderr + + def test_failing_manifest_exits_one(): result = _validate( str(FIXTURES / "broken_manifest"), "--level", "static", "--skip-build" diff --git a/tests/test_validation/test_runner.py b/tests/test_validation/test_runner.py index 6e2b1fd0bf..8117819033 100644 --- a/tests/test_validation/test_runner.py +++ b/tests/test_validation/test_runner.py @@ -1,3 +1,4 @@ +import hashlib import shutil import pytest @@ -76,3 +77,13 @@ def test_source_digest_is_deterministic_and_content_sensitive(tmp_path): assert len(first) == 64 (copy / "extra.txt").write_text("changed\n") assert source_digest(copy) != first + + +def test_source_digest_uses_portable_relative_paths(tmp_path): + package_root = tmp_path / "pkg" + nested = package_root / "nested" + nested.mkdir(parents=True) + (nested / "file.txt").write_bytes(b"contents") + + expected = hashlib.sha256(b"nested/file.txt\0contents\0").hexdigest() + assert source_digest(package_root) == expected From 01a395ed9afbff6fbcfe5eb2d49841b6bfff9895 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 1 Sep 2026 12:44:07 +0200 Subject: [PATCH 14/14] fix: handle invalid task encoding --- src/openenv/validation/signature.py | 2 +- tests/test_validation/test_signature_detection.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/openenv/validation/signature.py b/src/openenv/validation/signature.py index 4589d82302..d6b9c6d15a 100644 --- a/src/openenv/validation/signature.py +++ b/src/openenv/validation/signature.py @@ -60,7 +60,7 @@ def __init__(self, category: str, reason: str): def _has_yaml_frontmatter(path: Path) -> bool: try: return path.read_text(encoding="utf-8").lstrip().startswith("---") - except OSError: + except (OSError, UnicodeError): return False diff --git a/tests/test_validation/test_signature_detection.py b/tests/test_validation/test_signature_detection.py index 08897f97ce..3363d82f28 100644 --- a/tests/test_validation/test_signature_detection.py +++ b/tests/test_validation/test_signature_detection.py @@ -40,6 +40,12 @@ def test_task_md_without_frontmatter_is_not_a_named_format(tmp_path): assert detect_signature(tmp_path) is SignatureKind.OPENENV_SERVED +def test_non_utf8_task_md_is_not_a_named_format(tmp_path): + (tmp_path / "openenv.yaml").write_text("spec_version: 1\nname: x\n") + (tmp_path / "task.md").write_bytes(b"\xff\xfe") + assert detect_signature(tmp_path) is SignatureKind.OPENENV_SERVED + + def test_missing_directory_is_a_signature_error(tmp_path): with pytest.raises(SignatureError, match="not a package directory"): detect_signature(tmp_path / "does_not_exist")