From db3fa6708af3b50808205ccbdc4fad0c24c50a40 Mon Sep 17 00:00:00 2001 From: Ytallo Layon Date: Wed, 19 Aug 2026 17:16:52 -0300 Subject: [PATCH 1/4] Support exact manual stacks in ephemeral E2E --- .../scripts/harness_e2e_shadow_contract.py | 263 +++++++++++++++++- .github/workflows/harness-e2e-shadow.yml | 16 +- harness/tests/e2e/run-shadow-control-ci.sh | 100 ++++--- 3 files changed, 333 insertions(+), 46 deletions(-) diff --git a/.github/scripts/harness_e2e_shadow_contract.py b/.github/scripts/harness_e2e_shadow_contract.py index 22c28f5ff..9b298d416 100755 --- a/.github/scripts/harness_e2e_shadow_contract.py +++ b/.github/scripts/harness_e2e_shadow_contract.py @@ -31,6 +31,10 @@ def canonical(value: Any) -> str: return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True) +def canonical_sha256(value: Any) -> str: + return f"sha256:{hashlib.sha256(canonical(value).encode()).hexdigest()}" + + def require_uuid(value: Any, label: str) -> str: if not isinstance(value, str): raise ValueError(f"{label} must be a UUID") @@ -53,9 +57,150 @@ def require_digest(value: Any, label: str) -> str: return value +def require_version_map(value: Any, label: str) -> dict[str, str]: + if not isinstance(value, dict) or not value: + raise ValueError(f"{label} must be a non-empty object") + result: dict[str, str] = {} + for worker, version in value.items(): + if not isinstance(worker, str) or not re.fullmatch(r"[a-z0-9][a-z0-9_-]*", worker): + raise ValueError(f"{label} contains an invalid worker name") + if not isinstance(version, str) or not VERSION.fullmatch(version): + raise ValueError(f"{label} contains an invalid version for {worker}") + result[worker] = version + return result + + +def require_positive_integer(value: Any, label: str) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value < 1: + raise ValueError(f"{label} must be a positive integer") + return value + + +def v2_target_member(contract: dict[str, Any], target: dict[str, Any], name: str) -> Any: + """Accept the early top-level draft while emitting target-scoped v2 contracts.""" + return target.get(name) if name in target else contract.get(name) + + +def validate_v2_contract(contract: dict[str, Any], target: dict[str, Any], runner: dict[str, Any]) -> None: + stack = v2_target_member(contract, target, "stack") + if not isinstance(stack, dict): + raise ValueError("target.stack must be an object") + requested = require_version_map(stack.get("requested_versions"), "target.stack.requested_versions") + resolved = require_version_map(stack.get("resolved_versions"), "target.stack.resolved_versions") + if requested != resolved: + raise ValueError("target stack requested_versions must equal resolved_versions") + if resolved != target["stack_versions"]: + raise ValueError("target stack resolved_versions must match target.stack_versions") + resolution_sha256 = require_digest(stack.get("resolution_sha256"), "target.stack.resolution_sha256") + if resolution_sha256 != canonical_sha256(resolved): + raise ValueError("target stack resolution_sha256 does not match resolved_versions") + if resolution_sha256 != target["stack_digest"]: + raise ValueError("target stack resolution_sha256 must match target.stack_digest") + + origin = v2_target_member(contract, target, "origin") + if not isinstance(origin, dict): + raise ValueError("target.origin must be an object") + require_uuid(origin.get("operation_id"), "target.origin.operation_id") + require_uuid(origin.get("step_id"), "target.origin.step_id") + origin_worker = require_text(origin.get("worker"), "target.origin.worker") + origin_version = require_text(origin.get("version"), "target.origin.version") + if resolved.get(origin_worker) != origin_version: + raise ValueError("target origin worker/version must be present in the resolved stack") + origin_sha = require_text(origin.get("source_sha"), "target.origin.source_sha") + if not re.fullmatch(r"[0-9a-f]{40}", origin_sha): + raise ValueError("target.origin.source_sha must be a full lowercase git SHA") + require_positive_integer(origin.get("release_run_id"), "target.origin.release_run_id") + require_positive_integer(origin.get("release_run_attempt"), "target.origin.release_run_attempt") + + base = v2_target_member(contract, target, "base") + if not isinstance(base, dict) or base.get("kind") not in {"deployment", "snapshot"}: + raise ValueError("target.base.kind must be deployment or snapshot") + require_uuid(base.get("id"), "target.base.id") + + provenance = stack.get("provenance") + if not isinstance(provenance, list) or len(provenance) != len(resolved): + raise ValueError("target.stack.provenance must describe every resolved worker") + provenance_workers: list[str] = [] + for index, item in enumerate(provenance): + if not isinstance(item, dict): + raise ValueError(f"target.stack.provenance[{index}] must be an object") + worker = require_text(item.get("worker"), f"target.stack.provenance[{index}].worker") + version = require_text(item.get("version"), f"target.stack.provenance[{index}].version") + if resolved.get(worker) != version: + raise ValueError(f"target stack provenance does not match resolved version for {worker}") + provenance_workers.append(worker) + source_sha = item.get("source_sha") + if source_sha is not None and (not isinstance(source_sha, str) or not re.fullmatch(r"[0-9a-f]{40}", source_sha)): + raise ValueError(f"target.stack.provenance[{index}].source_sha is invalid") + for field in ("operation_id", "step_id"): + if item.get(field) is not None: + require_uuid(item[field], f"target.stack.provenance[{index}].{field}") + run_id = item.get("release_run_id") + run_attempt = item.get("release_run_attempt") + if (run_id is None) != (run_attempt is None): + raise ValueError("target stack provenance release run id and attempt must be paired") + if run_id is not None: + require_positive_integer(run_id, f"target.stack.provenance[{index}].release_run_id") + require_positive_integer(run_attempt, f"target.stack.provenance[{index}].release_run_attempt") + if provenance_workers != sorted(provenance_workers) or len(set(provenance_workers)) != len(provenance_workers): + raise ValueError("target.stack.provenance must be unique and ordered by worker") + origin_provenance = next((item for item in provenance if item.get("worker") == origin_worker), None) + if not origin_provenance or any( + origin_provenance.get(field) != origin.get(field) + for field in ( + "worker", + "version", + "source_sha", + "operation_id", + "step_id", + "release_run_id", + "release_run_attempt", + ) + ): + raise ValueError("target origin must match its stack provenance entry") + + runtime = contract.get("runtime") + if not isinstance(runtime, dict): + raise ValueError("runtime must be an object") + cli = runtime.get("cli") + if not isinstance(cli, dict): + raise ValueError("runtime.cli must be an object") + cli_version = require_text(cli.get("version"), "runtime.cli.version") + if not VERSION.fullmatch(cli_version): + raise ValueError("runtime.cli.version must be an exact version") + runtime_versions = require_version_map(runtime.get("stack_versions"), "runtime.stack_versions") + runtime_digest = require_digest(runtime.get("stack_digest"), "runtime.stack_digest") + if runtime_digest != canonical_sha256(runtime_versions): + raise ValueError("runtime.stack_digest does not match runtime.stack_versions") + conflicts = sorted( + worker for worker in runtime_versions.keys() & resolved.keys() if runtime_versions[worker] != resolved[worker] + ) + if conflicts: + raise ValueError(f"runtime and target stack pins conflict: {', '.join(conflicts)}") + + runner_ref = require_text(runner.get("registry_ref"), "runner.registry_ref") + if not VERSION.fullmatch(runner_ref): + raise ValueError("runner.registry_ref must be an exact version for schema v2") + runner_revision = runner.get("revision") + if runner_revision is not None and ( + not isinstance(runner_revision, str) or not re.fullmatch(r"[0-9a-f]{40}", runner_revision) + ): + raise ValueError("runner.revision must be a full lowercase git SHA") + if runner.get("catalog_sha256") is not None: + require_digest(runner["catalog_sha256"], "runner.catalog_sha256") + + security = contract.get("security") + if not isinstance(security, dict): + raise ValueError("security must be an object") + audience = require_text(security.get("oidc_audience"), "security.oidc_audience") + if not re.fullmatch(r"[A-Za-z0-9._:/-]+", audience): + raise ValueError("security.oidc_audience contains unsupported characters") + + def validate_contract(contract: dict[str, Any]) -> dict[str, Any]: - if contract.get("schema_version") != 1: - raise ValueError("execution contract schema_version must be 1") + schema_version = contract.get("schema_version") + if schema_version not in {1, 2}: + raise ValueError("execution contract schema_version must be 1 or 2") require_uuid(contract.get("campaign_id"), "campaign_id") require_uuid(contract.get("execution_id"), "execution_id") attempt = contract.get("attempt") @@ -73,14 +218,7 @@ def validate_contract(contract: dict[str, Any]) -> dict[str, Any]: if not re.fullmatch(r"[0-9a-f]{40}", source_sha): raise ValueError("target.source_sha must be a full lowercase git SHA") require_uuid(target.get("deployment_id"), "target.deployment_id") - stack = target.get("stack_versions") - if not isinstance(stack, dict) or not stack: - raise ValueError("target.stack_versions must be a non-empty object") - for worker, version in stack.items(): - if not isinstance(worker, str) or not re.fullmatch(r"[a-z0-9][a-z0-9_-]*", worker): - raise ValueError("target.stack_versions contains an invalid worker name") - if not isinstance(version, str) or not VERSION.fullmatch(version): - raise ValueError(f"target.stack_versions contains an invalid version for {worker}") + stack = require_version_map(target.get("stack_versions"), "target.stack_versions") if stack.get("harness") != target.get("version"): raise ValueError("target version must match stack_versions.harness") require_digest(target.get("stack_digest"), "target.stack_digest") @@ -121,6 +259,8 @@ def validate_contract(contract: dict[str, Any]) -> dict[str, Any]: runner_ref = require_text(runner.get("registry_ref"), "runner.registry_ref") if not re.fullmatch(r"[A-Za-z0-9._-]+", runner_ref): raise ValueError("runner.registry_ref is invalid") + if schema_version == 2: + validate_v2_contract(contract, target, runner) return contract @@ -134,6 +274,14 @@ def materialize_request(contract: dict[str, Any], catalog: dict[str, Any]) -> di for field in ("name", "version", "revision"): require_text(runner.get(field), f"catalog.runner.{field}") catalog_sha256 = require_digest(catalog.get("catalog_sha256"), "catalog.catalog_sha256") + if contract["schema_version"] == 2: + expected_runner = contract["runner"] + if runner.get("name") != expected_runner["registry_worker"] or runner.get("version") != expected_runner["registry_ref"]: + raise ValueError("scenario catalog runner does not match the exact runner pin") + if expected_runner.get("revision") is not None and runner.get("revision") != expected_runner["revision"]: + raise ValueError("scenario catalog runner revision does not match the contract") + if expected_runner.get("catalog_sha256") is not None and catalog_sha256 != expected_runner["catalog_sha256"]: + raise ValueError("scenario catalog digest does not match the contract") descriptors = catalog.get("scenarios") if not isinstance(descriptors, list): raise ValueError("scenario catalog scenarios must be a list") @@ -169,6 +317,12 @@ def materialize_request(contract: dict[str, Any], catalog: dict[str, Any]) -> di } ) + target_stack = contract["target"]["stack_versions"] + target_stack_digest = contract["target"]["stack_digest"] + if contract["schema_version"] == 2: + stack = v2_target_member(contract, contract["target"], "stack") + target_stack = stack["resolved_versions"] + target_stack_digest = stack["resolution_sha256"] run_contract = { "schema_version": 1, "mode": {"environment": "demonstration", "decision": "observe_only"}, @@ -177,8 +331,8 @@ def materialize_request(contract: dict[str, Any], catalog: dict[str, Any]) -> di "version": contract["target"]["version"], "stack": { "mode": "registry", - "stack_versions": contract["target"]["stack_versions"], - "stack_lock_digest": contract["target"]["stack_digest"], + "stack_versions": target_stack, + "stack_lock_digest": target_stack_digest, }, }, "plan": { @@ -214,6 +368,79 @@ def materialize_request(contract: dict[str, Any], catalog: dict[str, Any]) -> di } +def verify_lock(contract: dict[str, Any], lock_path: Path) -> dict[str, Any]: + validate_contract(contract) + try: + import yaml + except ImportError as error: # pragma: no cover - CI installs PyYAML explicitly. + raise ValueError("PyYAML is required to verify iii.lock") from error + + lock = yaml.safe_load(lock_path.read_text()) or {} + workers = lock.get("workers") if isinstance(lock, dict) else None + if not isinstance(workers, dict): + raise ValueError("iii.lock workers must be an object") + observed = { + str(worker): str(record.get("version")) + for worker, record in workers.items() + if isinstance(record, dict) and isinstance(record.get("version"), str) + } + + target = contract["target"] + if contract["schema_version"] == 2: + stack = v2_target_member(contract, target, "stack") + expected_target = stack["resolved_versions"] + target_digest = stack["resolution_sha256"] + runtime = contract["runtime"] + expected_runtime = runtime["stack_versions"] + runtime_digest = runtime["stack_digest"] + else: + expected_target = target["stack_versions"] + target_digest = target["stack_digest"] + expected_runtime = {} + runtime_digest = None + + expected = {**expected_runtime, **expected_target, contract["runner"]["registry_worker"]: contract["runner"]["registry_ref"]} + mismatches = [ + f"{worker}: expected {version}, resolved {observed.get(worker, 'missing')}" + for worker, version in sorted(expected.items()) + if observed.get(worker) != version + ] + if mismatches: + raise ValueError("stack_version_mismatch: " + "; ".join(mismatches)) + + target_stack = v2_target_member(contract, target, "stack") if contract["schema_version"] == 2 else None + return { + "schema": "e2e-stack-manifest/v1", + "contract_schema_version": contract["schema_version"], + "target": { + "application": target["application"], + "version": target["version"], + "requested_versions": target_stack["requested_versions"] if target_stack else expected_target, + "resolved_versions": expected_target, + "observed_versions": {worker: observed[worker] for worker in sorted(expected_target)}, + "resolution_sha256": target_digest, + "provenance": target_stack.get("provenance", []) if target_stack else [], + }, + "runtime": { + "cli": contract.get("runtime", {}).get("cli"), + "stack_versions": expected_runtime, + "observed_versions": {worker: observed[worker] for worker in sorted(expected_runtime)}, + "stack_digest": runtime_digest, + }, + "runner": { + **contract["runner"], + "observed_version": observed[contract["runner"]["registry_worker"]], + }, + "lock": { + "sha256": f"sha256:{hashlib.sha256(lock_path.read_bytes()).hexdigest()}", + "worker_count": len(observed), + "resolved_versions": dict(sorted(observed.items())), + }, + "origin": v2_target_member(contract, target, "origin") if contract["schema_version"] == 2 else None, + "base": v2_target_member(contract, target, "base") if contract["schema_version"] == 2 else None, + } + + def package_bundle(root: Path, contract: dict[str, Any], workflow: dict[str, Any]) -> dict[str, Any]: validate_contract(contract) files = [] @@ -236,6 +463,7 @@ def package_bundle(root: Path, contract: dict[str, Any], workflow: dict[str, Any "campaign_id": contract["campaign_id"], "execution_id": contract["execution_id"], "attempt": contract["attempt"], + "execution_contract_sha256": canonical_sha256(contract), "workflow": workflow, "terminal_payload": "results.json" if terminal.is_file() else None, "failure_payload": "failure.json" if failure.is_file() else None, @@ -248,10 +476,16 @@ def main() -> int: commands = parser.add_subparsers(dest="command", required=True) validate = commands.add_parser("validate") validate.add_argument("--contract", type=Path, required=True) + digest = commands.add_parser("digest") + digest.add_argument("--contract", type=Path, required=True) materialize = commands.add_parser("materialize") materialize.add_argument("--contract", type=Path, required=True) materialize.add_argument("--catalog", type=Path, required=True) materialize.add_argument("--output", type=Path, required=True) + lock = commands.add_parser("verify-lock") + lock.add_argument("--contract", type=Path, required=True) + lock.add_argument("--lock", type=Path, required=True) + lock.add_argument("--output", type=Path, required=True) package = commands.add_parser("package") package.add_argument("--root", type=Path, required=True) package.add_argument("--contract", type=Path, required=True) @@ -262,9 +496,14 @@ def main() -> int: contract = validate_contract(load_object(args.contract, "execution contract")) if args.command == "validate": print(canonical(contract)) + elif args.command == "digest": + print(canonical_sha256(contract)) elif args.command == "materialize": request = materialize_request(contract, load_object(args.catalog, "scenario catalog")) args.output.write_text(json.dumps(request, indent=2, sort_keys=True) + "\n") + elif args.command == "verify-lock": + manifest = verify_lock(contract, args.lock) + args.output.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") else: workflow = json.loads(args.workflow) if not isinstance(workflow, dict): diff --git a/.github/workflows/harness-e2e-shadow.yml b/.github/workflows/harness-e2e-shadow.yml index 0bc903489..c97d44fb3 100644 --- a/.github/workflows/harness-e2e-shadow.yml +++ b/.github/workflows/harness-e2e-shadow.yml @@ -52,6 +52,7 @@ jobs: --step-id '${{ inputs.execution_id }}' - name: Prepare execution contract + id: contract env: EXECUTION_CONTRACT: ${{ inputs.execution_contract }} run: | @@ -60,6 +61,13 @@ jobs: printf '%s\n' "$EXECUTION_CONTRACT" > target/harness-e2e-shadow/execution-contract.json python3 .github/scripts/harness_e2e_shadow_contract.py validate \ --contract target/harness-e2e-shadow/execution-contract.json >/dev/null + contract_sha256=$(python3 .github/scripts/harness_e2e_shadow_contract.py digest \ + --contract target/harness-e2e-shadow/execution-contract.json) + oidc_audience=$(jq -er \ + 'if .schema_version == 2 then .security.oidc_audience else "release-control-harness-e2e" end' \ + target/harness-e2e-shadow/execution-contract.json) + printf 'contract_sha256=%s\n' "$contract_sha256" >> "$GITHUB_OUTPUT" + printf 'oidc_audience=%s\n' "$oidc_audience" >> "$GITHUB_OUTPUT" jq -e \ --arg campaign '${{ inputs.campaign_id }}' \ --arg execution '${{ inputs.execution_id }}' \ @@ -75,12 +83,14 @@ jobs: continue-on-error: true env: RELEASE_CONTROL_API_URL: ${{ vars.RELEASE_CONTROL_API_URL }} + EXECUTION_CONTRACT_SHA256: ${{ steps.contract.outputs.contract_sha256 }} + OIDC_AUDIENCE: ${{ steps.contract.outputs.oidc_audience }} run: | set -euo pipefail test -n "$RELEASE_CONTROL_API_URL" token_response=$(curl -fsSL \ -H "Authorization: Bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ - "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=release-control-harness-e2e") + "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=${OIDC_AUDIENCE}") oidc_token=$(jq -er '.value' <<<"$token_response") response=$(curl -fsSL \ -H "Authorization: Bearer $oidc_token" \ @@ -91,7 +101,8 @@ jobs: --argjson run_id '${{ github.run_id }}' \ --argjson run_attempt '${{ github.run_attempt }}' \ --arg workflow_sha '${{ github.sha }}' \ - '{campaign_id:$campaign_id,run_id:$run_id,run_attempt:$run_attempt,workflow_sha:$workflow_sha}')") + --arg execution_contract_sha256 "$EXECUTION_CONTRACT_SHA256" \ + '{campaign_id:$campaign_id,run_id:$run_id,run_attempt:$run_attempt,workflow_sha:$workflow_sha,execution_contract_sha256:$execution_contract_sha256}')") printf '%s\n' "$response" > target/harness-e2e-shadow/admission.json jq -e '.admitted == true' target/harness-e2e-shadow/admission.json >/dev/null @@ -105,7 +116,6 @@ jobs: ZAI_API_KEY: ${{ secrets.ZAI_API_KEY }} DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} HARNESS_E2E_EXECUTION_CONTRACT: ${{ inputs.execution_contract }} - III_CLI_CHANNEL: latest run: harness/tests/e2e/run-shadow-control-ci.sh - name: Record admission failure diff --git a/harness/tests/e2e/run-shadow-control-ci.sh b/harness/tests/e2e/run-shadow-control-ci.sh index e147113d1..36893fef7 100755 --- a/harness/tests/e2e/run-shadow-control-ci.sh +++ b/harness/tests/e2e/run-shadow-control-ci.sh @@ -8,7 +8,6 @@ repo_root=$(cd -- "$script_dir/../../.." && pwd) contract_tool="$repo_root/.github/scripts/harness_e2e_shadow_contract.py" artifact_dir=${HARNESS_E2E_ARTIFACTS_DIR:-"$repo_root/target/harness-e2e-shadow"} install_url=${III_INSTALL_URL:-https://install.iii.dev/iii/main/install.sh} -cli_channel=${III_CLI_CHANNEL:-latest} engine_port=${HARNESS_E2E_ENGINE_PORT:-49134} wait_seconds=${HARNESS_E2E_WAIT_SECONDS:-180} run_timeout_seconds=${HARNESS_E2E_RUN_TIMEOUT_SECONDS:-10800} @@ -17,16 +16,27 @@ case "$artifact_dir" in "$repo_root"/target/*) ;; *) echo "HARNESS_E2E_ARTIFACTS_DIR must be below $repo_root/target" >&2; exit 2 ;; esac -case "$cli_channel" in latest | next) ;; *) echo "III_CLI_CHANNEL must be latest or next" >&2; exit 2 ;; esac - mkdir -p "$artifact_dir" artifact_dir=$(cd "$artifact_dir" && pwd) contract_path="$artifact_dir/execution-contract.json" printf '%s\n' "$HARNESS_E2E_EXECUTION_CONTRACT" >"$contract_path" python3 "$contract_tool" validate --contract "$contract_path" >/dev/null -stack_versions=$(jq -c '.target.stack_versions' "$contract_path") -stack_digest=$(jq -r '.target.stack_digest' "$contract_path") +contract_schema=$(jq -r '.schema_version' "$contract_path") +if [[ "$contract_schema" == 2 ]]; then + stack_versions=$(jq -c '.target.stack.resolved_versions' "$contract_path") + stack_digest=$(jq -r '.target.stack.resolution_sha256' "$contract_path") + runtime_versions=$(jq -c '.runtime.stack_versions' "$contract_path") + cli_version=$(jq -r '.runtime.cli.version' "$contract_path") + cli_channel="" +else + stack_versions=$(jq -c '.target.stack_versions' "$contract_path") + stack_digest=$(jq -r '.target.stack_digest' "$contract_path") + runtime_versions='{}' + cli_version="" + cli_channel=${III_CLI_CHANNEL:-latest} + case "$cli_channel" in latest | next) ;; *) echo "III_CLI_CHANNEL must be latest or next" >&2; exit 2 ;; esac +fi runner_worker=$(jq -r '.runner.registry_worker' "$contract_path") runner_ref=$(jq -r '.runner.registry_ref' "$contract_path") subject_provider=$(jq -r '.plan.definition.subject.provider' "$contract_path") @@ -142,17 +152,26 @@ add_with_retry() { return 1 } -log "Installing iii CLI from $cli_channel" +if [[ "$contract_schema" == 2 ]]; then + log "Installing exact iii CLI $cli_version" +else + log "Installing iii CLI from $cli_channel" +fi curl -fsSL --retry 3 --retry-all-errors --retry-delay 5 "$install_url" -o "$run_root/install.sh" -if [[ "$cli_channel" == next ]]; then +if [[ "$contract_schema" == 2 ]]; then + VERSION="$cli_version" sh "$run_root/install.sh" 2>&1 | tee "$artifact_dir/logs/install.log" +elif [[ "$cli_channel" == next ]]; then sh "$run_root/install.sh" --next 2>&1 | tee "$artifact_dir/logs/install.log" else sh "$run_root/install.sh" 2>&1 | tee "$artifact_dir/logs/install.log" fi iii_bin=$(command -v iii) -cli_version=$("$iii_bin" --version 2>&1) -printf '%s\n' "$cli_version" >"$artifact_dir/iii-version.txt" -export HARNESS_E2E_ENGINE_REVISION="$cli_version" +observed_cli_version=$("$iii_bin" --version 2>&1) +printf '%s\n' "$observed_cli_version" >"$artifact_dir/iii-version.txt" +if [[ "$contract_schema" == 2 && "$observed_cli_version" != *"$cli_version"* ]]; then + fail "iii CLI version mismatch: expected $cli_version, observed $observed_cli_version" +fi +export HARNESS_E2E_ENGINE_REVISION="$observed_cli_version" printf 'workers: []\n' >"$project_dir/config.yaml" (cd "$project_dir" && exec setsid "$iii_bin" -c config.yaml --no-update-check) \ @@ -161,15 +180,22 @@ engine_pid=$! wait_for_engine failure_phase=registry -support=("database@latest" "storage@latest" "fp@latest" "web@latest") -declare -A providers=() -for provider in "$subject_provider" "$judge_provider"; do - [[ -n "${providers[$provider]:-}" ]] && continue - support+=("provider-$provider@latest") - providers[$provider]=1 -done -log "Installing E2E support workers" -add_with_retry support "${support[@]}" +if [[ "$contract_schema" == 2 ]]; then + while IFS=$'\t' read -r worker version; do + log "Installing exact E2E runtime: $worker@$version" + add_with_retry "runtime-$worker" "$worker@$version" --force + done < <(jq -r 'to_entries | sort_by(.key)[] | [.key,.value] | @tsv' <<<"$runtime_versions") +else + support=("database@latest" "storage@latest" "fp@latest" "web@latest") + declare -A providers=() + for provider in "$subject_provider" "$judge_provider"; do + [[ -n "${providers[$provider]:-}" ]] && continue + support+=("provider-$provider@latest") + providers[$provider]=1 + done + log "Installing E2E support workers" + add_with_retry support "${support[@]}" +fi while IFS=$'\t' read -r worker version; do log "Installing exact target stack: $worker@$version" @@ -179,23 +205,35 @@ done < <(jq -r 'to_entries | sort_by(.key)[] | [.key,.value] | @tsv' <<<"$stack_ log "Installing ephemeral runner: $runner_worker@$runner_ref" add_with_retry runner "$runner_worker@$runner_ref" --force -# A runner dependency may resolve a different version of a target worker. The -# target stack is authoritative, so reapply every exact pin after the runner. +# A runner dependency may resolve a different version of a runtime or target +# worker. Reapply every exact pin after the runner, with the target last. +if [[ "$contract_schema" == 2 ]]; then + while IFS=$'\t' read -r worker version; do + add_with_retry "repin-runtime-$worker" "$worker@$version" --force + done < <(jq -r 'to_entries | sort_by(.key)[] | [.key,.value] | @tsv' <<<"$runtime_versions") +fi while IFS=$'\t' read -r worker version; do add_with_retry "repin-$worker" "$worker@$version" --force done < <(jq -r 'to_entries | sort_by(.key)[] | [.key,.value] | @tsv' <<<"$stack_versions") target_harness_version=$(jq -r '.target.version' "$contract_path") -python3 "$repo_root/.github/scripts/verify_registry_lock.py" \ - --lock "$project_dir/iii.lock" \ - --worker harness \ - --version "$target_harness_version" \ - --required harness \ - --required "$runner_worker" \ - --required database \ - --required storage \ - --expected-versions-json "$stack_versions" \ - --output "$artifact_dir/stack/lock-verification.json" >/dev/null +if [[ "$contract_schema" == 2 ]]; then + python3 "$contract_tool" verify-lock \ + --contract "$contract_path" \ + --lock "$project_dir/iii.lock" \ + --output "$artifact_dir/stack/stack-manifest.json" +else + python3 "$repo_root/.github/scripts/verify_registry_lock.py" \ + --lock "$project_dir/iii.lock" \ + --worker harness \ + --version "$target_harness_version" \ + --required harness \ + --required "$runner_worker" \ + --required database \ + --required storage \ + --expected-versions-json "$stack_versions" \ + --output "$artifact_dir/stack/lock-verification.json" >/dev/null +fi wait_for_functions \ e2e::scenarios-list e2e::run e2e::status e2e::results-get \ From a01f38aa3e874935b986c77288ce9a2ac700cb71 Mon Sep 17 00:00:00 2001 From: Ytallo Layon Date: Wed, 19 Aug 2026 17:19:47 -0300 Subject: [PATCH 2/4] Allow originless v2 E2E contracts --- .../scripts/harness_e2e_shadow_contract.py | 87 +++++++++++-------- 1 file changed, 51 insertions(+), 36 deletions(-) diff --git a/.github/scripts/harness_e2e_shadow_contract.py b/.github/scripts/harness_e2e_shadow_contract.py index 9b298d416..b76b0d9c8 100755 --- a/.github/scripts/harness_e2e_shadow_contract.py +++ b/.github/scripts/harness_e2e_shadow_contract.py @@ -98,19 +98,21 @@ def validate_v2_contract(contract: dict[str, Any], target: dict[str, Any], runne raise ValueError("target stack resolution_sha256 must match target.stack_digest") origin = v2_target_member(contract, target, "origin") - if not isinstance(origin, dict): - raise ValueError("target.origin must be an object") - require_uuid(origin.get("operation_id"), "target.origin.operation_id") - require_uuid(origin.get("step_id"), "target.origin.step_id") - origin_worker = require_text(origin.get("worker"), "target.origin.worker") - origin_version = require_text(origin.get("version"), "target.origin.version") - if resolved.get(origin_worker) != origin_version: - raise ValueError("target origin worker/version must be present in the resolved stack") - origin_sha = require_text(origin.get("source_sha"), "target.origin.source_sha") - if not re.fullmatch(r"[0-9a-f]{40}", origin_sha): - raise ValueError("target.origin.source_sha must be a full lowercase git SHA") - require_positive_integer(origin.get("release_run_id"), "target.origin.release_run_id") - require_positive_integer(origin.get("release_run_attempt"), "target.origin.release_run_attempt") + origin_worker = None + if origin is not None: + if not isinstance(origin, dict): + raise ValueError("target.origin must be an object or null") + require_uuid(origin.get("operation_id"), "target.origin.operation_id") + require_uuid(origin.get("step_id"), "target.origin.step_id") + origin_worker = require_text(origin.get("worker"), "target.origin.worker") + origin_version = require_text(origin.get("version"), "target.origin.version") + if resolved.get(origin_worker) != origin_version: + raise ValueError("target origin worker/version must be present in the resolved stack") + origin_sha = require_text(origin.get("source_sha"), "target.origin.source_sha") + if not re.fullmatch(r"[0-9a-f]{40}", origin_sha): + raise ValueError("target.origin.source_sha must be a full lowercase git SHA") + require_positive_integer(origin.get("release_run_id"), "target.origin.release_run_id") + require_positive_integer(origin.get("release_run_attempt"), "target.origin.release_run_attempt") base = v2_target_member(contract, target, "base") if not isinstance(base, dict) or base.get("kind") not in {"deployment", "snapshot"}: @@ -144,20 +146,21 @@ def validate_v2_contract(contract: dict[str, Any], target: dict[str, Any], runne require_positive_integer(run_attempt, f"target.stack.provenance[{index}].release_run_attempt") if provenance_workers != sorted(provenance_workers) or len(set(provenance_workers)) != len(provenance_workers): raise ValueError("target.stack.provenance must be unique and ordered by worker") - origin_provenance = next((item for item in provenance if item.get("worker") == origin_worker), None) - if not origin_provenance or any( - origin_provenance.get(field) != origin.get(field) - for field in ( - "worker", - "version", - "source_sha", - "operation_id", - "step_id", - "release_run_id", - "release_run_attempt", - ) - ): - raise ValueError("target origin must match its stack provenance entry") + if origin_worker is not None: + origin_provenance = next((item for item in provenance if item.get("worker") == origin_worker), None) + if not origin_provenance or any( + origin_provenance.get(field) != origin.get(field) + for field in ( + "worker", + "version", + "source_sha", + "operation_id", + "step_id", + "release_run_id", + "release_run_attempt", + ) + ): + raise ValueError("target origin must match its stack provenance entry") runtime = contract.get("runtime") if not isinstance(runtime, dict): @@ -214,14 +217,26 @@ def validate_contract(contract: dict[str, Any]) -> dict[str, Any]: if not isinstance(target, dict) or target.get("application") != "harness": raise ValueError("target application must be harness") require_text(target.get("version"), "target.version") - source_sha = require_text(target.get("source_sha"), "target.source_sha") - if not re.fullmatch(r"[0-9a-f]{40}", source_sha): - raise ValueError("target.source_sha must be a full lowercase git SHA") - require_uuid(target.get("deployment_id"), "target.deployment_id") - stack = require_version_map(target.get("stack_versions"), "target.stack_versions") - if stack.get("harness") != target.get("version"): - raise ValueError("target version must match stack_versions.harness") - require_digest(target.get("stack_digest"), "target.stack_digest") + if schema_version == 1: + source_sha = require_text(target.get("source_sha"), "target.source_sha") + if not re.fullmatch(r"[0-9a-f]{40}", source_sha): + raise ValueError("target.source_sha must be a full lowercase git SHA") + require_uuid(target.get("deployment_id"), "target.deployment_id") + stack = require_version_map(target.get("stack_versions"), "target.stack_versions") + if stack.get("harness") != target.get("version"): + raise ValueError("target version must match stack_versions.harness") + require_digest(target.get("stack_digest"), "target.stack_digest") + else: + if target.get("source_sha") is not None and not re.fullmatch(r"[0-9a-f]{40}", target["source_sha"]): + raise ValueError("target.source_sha must be a full lowercase git SHA when present") + if target.get("deployment_id") is not None: + require_uuid(target["deployment_id"], "target.deployment_id") + if target.get("stack_versions") is not None: + stack = require_version_map(target["stack_versions"], "target.stack_versions") + if stack.get("harness") != target.get("version"): + raise ValueError("target version must match stack_versions.harness") + if target.get("stack_digest") is not None: + require_digest(target["stack_digest"], "target.stack_digest") plan = contract.get("plan") if not isinstance(plan, dict): @@ -346,7 +361,7 @@ def materialize_request(contract: dict[str, Any], catalog: dict[str, Any]) -> di "selected_cases": selected_cases, "correlation": { "system": "release-control", - "deployment_id": contract["target"]["deployment_id"], + "deployment_id": contract["target"].get("deployment_id") or contract["campaign_id"], "operation_id": contract["campaign_id"], }, } From ff48408fe4d316aa6a9145be0e3196961b38b874 Mon Sep 17 00:00:00 2001 From: Ytallo Layon Date: Thu, 20 Aug 2026 09:14:06 -0300 Subject: [PATCH 3/4] Test manual E2E stack contracts --- .../tests/test_harness_e2e_shadow_contract.py | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/.github/scripts/tests/test_harness_e2e_shadow_contract.py b/.github/scripts/tests/test_harness_e2e_shadow_contract.py index 7e57d9d7c..bcda07092 100644 --- a/.github/scripts/tests/test_harness_e2e_shadow_contract.py +++ b/.github/scripts/tests/test_harness_e2e_shadow_contract.py @@ -67,6 +67,71 @@ def catalog(): } +def manual_v2_contract(): + versions = {"harness": "1.9.0", "state": "0.22.1"} + stack_digest = MODULE.canonical_sha256(versions) + return { + "schema_version": 2, + "campaign_id": "11111111-1111-4111-8111-111111111111", + "execution_id": "22222222-2222-4222-8222-222222222222", + "attempt": 1, + "idempotency_key": f"rc:d0:{'a' * 64}", + "target": { + "application": "harness", + "version": "1.9.0", + "source_sha": "b" * 40, + "deployment_id": "33333333-3333-4333-8333-333333333333", + "stack_versions": versions, + "stack_digest": stack_digest, + "origin": None, + "base": {"kind": "deployment", "id": "33333333-3333-4333-8333-333333333333"}, + "stack": { + "requested_versions": versions, + "resolved_versions": versions, + "resolution_sha256": stack_digest, + "provenance": [ + { + "worker": "harness", + "version": "1.9.0", + "source_sha": "b" * 40, + "operation_id": "33333333-3333-4333-8333-333333333333", + }, + {"worker": "state", "version": "0.22.1"}, + ], + }, + }, + "plan": { + "id": "44444444-4444-4444-8444-444444444444", + "revision": 2, + "sha256": f"sha256:{'c' * 64}", + "definition": { + "mode": "demonstrative", + "entrypoint": "e2e::run", + "label": "Manual stack", + "lane": "release-control-shadow", + "subject": {"provider": "anthropic", "model": "claude-sonnet-4-6"}, + "judge": {"provider": "anthropic", "model": "claude-sonnet-4-6"}, + "scenarios": ["direct_answer"], + "runs": 1, + "seed": 4404, + "technicalRetries": 1, + "progressIntervalSeconds": 15, + }, + }, + "runner": {"registry_worker": "harness-e2e", "registry_ref": "0.1.0-experimental"}, + "workflow": {"repository": "iii-hq/workers", "file": "harness-e2e-shadow.yml", "ref": "main"}, + "runtime": {"cli": {"version": "0.22.1"}, "stack_versions": versions, "stack_digest": stack_digest}, + "security": {"oidc_audience": "release-control-harness-e2e"}, + } + + +def manual_v2_catalog(): + return { + **catalog(), + "runner": {"name": "harness-e2e", "version": "0.1.0-experimental", "revision": "e" * 40}, + } + + class ShadowContractTest(unittest.TestCase): def test_materializes_observe_only_request(self): request = MODULE.materialize_request(contract(), catalog()) @@ -85,6 +150,23 @@ def test_accepts_exact_registry_prerelease_versions(self): changed["target"]["stack_versions"]["database"] = "0.11.0-next.5" MODULE.validate_contract(changed) + def test_materializes_the_release_control_manual_v2_contract_without_an_origin_step(self): + request = MODULE.materialize_request(manual_v2_contract(), manual_v2_catalog()) + + self.assertEqual( + request["run_contract"]["target"]["stack"]["stack_versions"], + {"harness": "1.9.0", "state": "0.22.1"}, + ) + self.assertEqual(request["run_contract"]["runner"]["version"], "0.1.0-experimental") + self.assertEqual(request["run_contract"]["selected_cases"][0]["scenario_id"], "direct_answer") + + def test_rejects_a_manual_v2_contract_when_the_resolved_stack_digest_is_tampered(self): + changed = manual_v2_contract() + changed["target"]["stack"]["resolution_sha256"] = f"sha256:{'0' * 64}" + + with self.assertRaisesRegex(ValueError, "resolution_sha256 does not match"): + MODULE.validate_contract(changed) + def test_packages_raw_file_digests(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) From e53490fd512f487db5597f3d445cf4dc168fff73 Mon Sep 17 00:00:00 2001 From: Ytallo Layon Date: Thu, 20 Aug 2026 10:03:52 -0300 Subject: [PATCH 4/4] Isolate the ephemeral E2E engine configuration --- harness/tests/e2e/run-shadow-control-ci.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/harness/tests/e2e/run-shadow-control-ci.sh b/harness/tests/e2e/run-shadow-control-ci.sh index 36893fef7..446e5cc53 100755 --- a/harness/tests/e2e/run-shadow-control-ci.sh +++ b/harness/tests/e2e/run-shadow-control-ci.sh @@ -45,6 +45,7 @@ seed=$(jq -r '.plan.definition.seed' "$contract_path") run_root=$(mktemp -d "${TMPDIR:-/tmp}/harness-e2e-shadow.XXXXXX") project_dir="$run_root/project" +project_config="$project_dir/iii.config.yaml" e2e_home="$run_root/home" mkdir -p "$project_dir" "$e2e_home" "$artifact_dir/logs" "$artifact_dir/stack" @@ -74,7 +75,7 @@ fail() { } snapshot_stack() { - for file in config.yaml iii.lock workers.json; do + for file in iii.config.yaml iii.lock workers.json; do [[ -f "$project_dir/$file" ]] && cp "$project_dir/$file" "$artifact_dir/stack/$file" done if [[ -f "$project_dir/iii.lock" ]]; then @@ -173,8 +174,8 @@ if [[ "$contract_schema" == 2 && "$observed_cli_version" != *"$cli_version"* ]]; fi export HARNESS_E2E_ENGINE_REVISION="$observed_cli_version" -printf 'workers: []\n' >"$project_dir/config.yaml" -(cd "$project_dir" && exec setsid "$iii_bin" -c config.yaml --no-update-check) \ +printf 'workers: []\n' >"$project_config" +(cd "$project_dir" && exec setsid "$iii_bin" -c iii.config.yaml --no-update-check) \ >"$artifact_dir/logs/engine.log" 2>&1 & engine_pid=$! wait_for_engine