diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index a7da769dd35b..0fda63a8dbd1 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -14,6 +14,8 @@ self-hosted-runner: # self-hosted runner. See check_no_self_hosted_fleet_runners. # Manual E2E canary only. Required CI remains routed through repo variables. - tart-canary + # Dispatch-only compile-admission producer; guarded separately from required jobs. + - cmux-persistent-macos-compile # Linux: Blacksmith primary (LINUX_RUNNER), WarpBuild overflow fallback. - blacksmith-4vcpu-ubuntu-2404 - blacksmith-8vcpu-ubuntu-2404 diff --git a/.github/workflows/ci-guards.yml b/.github/workflows/ci-guards.yml index 847f591d7e5b..7c30fc203317 100644 --- a/.github/workflows/ci-guards.yml +++ b/.github/workflows/ci-guards.yml @@ -36,6 +36,9 @@ jobs: - name: Validate macOS runner guards run: ./tests/test_ci_self_hosted_guard.sh + - name: Validate persistent Mac compile admission contract + run: python3 tests/test_ci_persistent_mac_compile.py + - name: Validate tagged reload works without the shared dev backend run: ./tests/test_reload_local_backend_mode.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 58986319a744..5179798dcb72 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,6 +55,9 @@ jobs: ghosttykit_release: ${{ steps.linux_guards.outputs.ghosttykit_release }} full_suite: ${{ steps.suite.outputs.full_suite }} compile_admitted: ${{ steps.admitted.outputs.compile_admitted }} + source_tree: ${{ steps.source-identity.outputs.tree }} + source_parent1: ${{ steps.source-identity.outputs.parent1 }} + source_identity_valid: ${{ steps.source-identity.outputs.valid }} permissions: actions: read contents: read @@ -65,6 +68,29 @@ jobs: with: fetch-depth: 2 + - name: Record GitHub-selected source identity + id: source-identity + run: | + set -u + valid=false + tree="" + parent1="" + read -r commit parent1 parent2 extra <&2 + fi + { + echo "valid=$valid" + echo "tree=$tree" + echo "parent1=${parent1:-}" + } >> "$GITHUB_OUTPUT" + - name: Detect CI change areas id: detect env: @@ -241,7 +267,7 @@ jobs: - name: Fingerprint the build inputs id: inputs continue-on-error: true - if: ${{ github.event_name == 'pull_request' && steps.detect.outputs.macos == 'true' }} + if: ${{ github.event_name == 'pull_request' && steps.source-identity.outputs.valid == 'true' && steps.detect.outputs.macos == 'true' }} env: XCODE_APP: ${{ vars.CMUX_CI_XCODE_APP_MACOS_15 }} run: | @@ -273,6 +299,56 @@ jobs: python3 scripts/ci/find_admitted_build.py --repository "$GITHUB_REPOSITORY" --branch "$BRANCH" \ --fingerprint "$FINGERPRINT" --current-run-id "$GITHUB_RUN_ID" --github-output "$GITHUB_OUTPUT" + + # The PR workflow only publishes immutable request evidence. Dispatch and + # cancellation authority live in persistent-macos-router.yml on main. + - name: Publish persistent Mac route request + id: persistent-route-request + if: ${{ github.event_name == 'pull_request' && steps.detect.outputs.macos == 'true' && steps.admitted.outputs.compile_admitted != 'true' && (vars.CI_PERSISTENT_MAC_COMPILE == 'pilot' || vars.CI_PERSISTENT_MAC_COMPILE == 'all' || vars.CI_PERSISTENT_MAC_COMPILE == 'on' || vars.CI_PERSISTENT_MAC_COMPILE == 'true' || vars.CI_PERSISTENT_MAC_COMPILE == '1') && github.event.pull_request.head.repo.full_name == github.repository && (github.event.pull_request.author_association == 'MEMBER' || github.event.pull_request.author_association == 'OWNER') }} + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name }} + HEAD_REF: ${{ github.head_ref }} + AUTHOR_ASSOCIATION: ${{ github.event.pull_request.author_association }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + SOURCE_TREE: ${{ steps.source-identity.outputs.tree }} + SOURCE_PARENT1: ${{ steps.source-identity.outputs.parent1 }} + run: | + set -euo pipefail + request="$RUNNER_TEMP/persistent-mac-route-request.json" + REQUEST="$request" python3 - <<'PY' + import json + import os + + payload = { + "schema_version": 1, + "request_id": f"{os.environ['GITHUB_RUN_ID']}-{os.environ['GITHUB_RUN_ATTEMPT']}", + "pr_number": os.environ["PR_NUMBER"], + "source_sha": os.environ["GITHUB_SHA"], + "source_tree": os.environ["SOURCE_TREE"], + "source_parent1": os.environ["SOURCE_PARENT1"], + "head_sha": os.environ["HEAD_SHA"], + "head_ref": os.environ["HEAD_REF"], + "head_repository": os.environ["HEAD_REPOSITORY"], + "author_association": os.environ["AUTHOR_ASSOCIATION"], + } + with open(os.environ["REQUEST"], "w", encoding="utf-8") as stream: + json.dump(payload, stream, sort_keys=True) + stream.write("\n") + PY + echo "path=$request" >> "$GITHUB_OUTPUT" + echo "publish=true" >> "$GITHUB_OUTPUT" + + - name: Upload persistent Mac route request + if: ${{ steps.persistent-route-request.outputs.publish == 'true' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: persistent-mac-route-request-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ steps.persistent-route-request.outputs.path }} + if-no-files-found: error + retention-days: 1 + compression-level: 0 + static-preflight: name: Fast static checks # No package installs, submodules or app build: reject malformed inputs @@ -535,6 +611,7 @@ jobs: permissions: contents: read actions: read + pull-requests: read needs: - changes - linux-preflight @@ -2476,6 +2553,14 @@ jobs: CMUX_SKIP_ZIG_BUILD: "1" CMUX_PRODUCT_RUNNER: ${{ vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15' }} steps: + - name: Start compile admission timers + run: | + now="$(python3 -c 'import time; print(time.monotonic())')" + { + echo "CMUX_MACOS_ADMISSION_STARTED=$now" + echo "CMUX_HOSTED_SOURCE_PREP_STARTED=$now" + } >> "$GITHUB_ENV" + - name: Clear stale git locks (self-hosted reused workspace) shell: bash run: | @@ -2533,6 +2618,43 @@ jobs: echo "== end network diagnostics ==" >&2 exit 1 + - name: Record hosted source preparation + id: hosted-source-prep + run: | + now="$(python3 -c 'import time; print(time.monotonic())')" + seconds="$(python3 -c 'import sys; print(round(float(sys.argv[2])-float(sys.argv[1]), 6))' "$CMUX_HOSTED_SOURCE_PREP_STARTED" "$now")" + echo "seconds=$seconds" >> "$GITHUB_OUTPUT" + + - name: Measure hosted queue-to-start + id: hosted-queue + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + python3 - <<'PY' + import datetime + import json + import os + import subprocess + + result = subprocess.run( + ["gh", "api", f"repos/{os.environ['GITHUB_REPOSITORY']}/actions/runs/{os.environ['GITHUB_RUN_ID']}/jobs?filter=latest&per_page=100"], + check=True, text=True, capture_output=True, + ) + jobs = json.loads(result.stdout).get("jobs", []) + matches = [job for job in jobs if job.get("name") == "macOS compile admission"] + if len(matches) != 1: + raise SystemExit(f"expected one admission job, found {len(matches)}") + job = matches[0] + created = datetime.datetime.fromisoformat(job["created_at"].replace("Z", "+00:00")) + started = datetime.datetime.fromisoformat(job["started_at"].replace("Z", "+00:00")) + seconds = max(0.0, (started - created).total_seconds()) + with open(os.environ["GITHUB_OUTPUT"], "a") as output: + output.write(f"seconds={seconds:.3f}\n") + print(f"hosted admission queue-to-start: {seconds:.3f}s") + PY + - name: Prepare isolated admission DerivedData run: | set -euo pipefail @@ -2586,13 +2708,217 @@ jobs: echo "- macOS runner minutes saved: $(show "$REUSE_MACOS_MINUTES_SAVED")" } >> "$GITHUB_STEP_SUMMARY" + - name: Observe persistent Mac compile candidate + id: persistent-route + if: ${{ steps.reuse-products.outputs.hit != 'true' && github.event_name == 'pull_request' && needs.changes.outputs.source_identity_valid == 'true' && (vars.CI_PERSISTENT_MAC_COMPILE == 'pilot' || vars.CI_PERSISTENT_MAC_COMPILE == 'all' || vars.CI_PERSISTENT_MAC_COMPILE == 'on' || vars.CI_PERSISTENT_MAC_COMPILE == 'true' || vars.CI_PERSISTENT_MAC_COMPILE == '1') && github.event.pull_request.head.repo.full_name == github.repository && (github.event.pull_request.author_association == 'MEMBER' || github.event.pull_request.author_association == 'OWNER') }} + env: + GH_TOKEN: ${{ github.token }} + CI_PERSISTENT_MAC_COMPILE: ${{ vars.CI_PERSISTENT_MAC_COMPILE }} + CI_PERSISTENT_MAC_COMPILE_COHORT: ${{ vars.CI_PERSISTENT_MAC_COMPILE_COHORT }} + CI_PERSISTENT_MAC_QUEUE_SECONDS: ${{ vars.CI_PERSISTENT_MAC_QUEUE_SECONDS }} + CI_PERSISTENT_MAC_EXECUTION_SECONDS: ${{ vars.CI_PERSISTENT_MAC_EXECUTION_SECONDS }} + PR_NUMBER: ${{ github.event.pull_request.number }} + HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name }} + HEAD_REF: ${{ github.head_ref }} + AUTHOR_ASSOCIATION: ${{ github.event.pull_request.author_association }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + SOURCE_TREE: ${{ needs.changes.outputs.source_tree }} + SOURCE_PARENT1: ${{ needs.changes.outputs.source_parent1 }} + run: | + set -euo pipefail + route_started="$(python3 -c 'import time; print(time.monotonic())')" + queue_seconds="${CI_PERSISTENT_MAC_QUEUE_SECONDS:-90}" + execution_seconds="${CI_PERSISTENT_MAC_EXECUTION_SECONDS:-480}" + python3 scripts/ci/persistent_mac_route.py \ + --observe-only \ + --event-name "$GITHUB_EVENT_NAME" \ + --selector "$CI_PERSISTENT_MAC_COMPILE" \ + --cohort "$CI_PERSISTENT_MAC_COMPILE_COHORT" \ + --repository "$GITHUB_REPOSITORY" \ + --pr-number "$PR_NUMBER" \ + --head-repository "$HEAD_REPOSITORY" \ + --head-ref "$HEAD_REF" \ + --author-association "$AUTHOR_ASSOCIATION" \ + --head-sha "$HEAD_SHA" \ + --source-sha "$GITHUB_SHA" \ + --source-tree "$SOURCE_TREE" \ + --source-parent1 "$SOURCE_PARENT1" \ + --run-id "$GITHUB_RUN_ID" \ + --run-attempt "$GITHUB_RUN_ATTEMPT" \ + --queue-seconds "$queue_seconds" \ + --execution-seconds "$execution_seconds" \ + --observe-seconds 60 \ + --github-output "$GITHUB_OUTPUT" + route_finished="$(python3 -c 'import time; print(time.monotonic())')" + route_wall="$(python3 -c 'import sys; print(round(float(sys.argv[2])-float(sys.argv[1]), 6))' "$route_started" "$route_finished")" + echo "route_wall_seconds=$route_wall" >> "$GITHUB_OUTPUT" + + - name: Download persistent Mac compile product + id: persistent-download + if: steps.reuse-products.outputs.hit != 'true' && steps.persistent-route.outputs.use_persistent == 'true' + continue-on-error: true + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + artifact-ids: ${{ steps.persistent-route.outputs.artifact_id }} + path: ${{ runner.temp }}/persistent-mac-compile + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ steps.persistent-route.outputs.producer_run_id }} + + - name: Revalidate persistent Mac compile product + id: persistent-restore + if: steps.reuse-products.outputs.hit != 'true' && steps.persistent-download.outcome == 'success' + continue-on-error: true + env: + EXPECTED_SOURCE_SHA: ${{ github.sha }} + EXPECTED_SOURCE_TREE: ${{ needs.changes.outputs.source_tree }} + run: | + set -euo pipefail + [ -z "$(git status --porcelain --untracked-files=all)" ] || { + echo "hosted validation checkout is dirty" >&2 + exit 1 + } + [ "$(git rev-parse 'HEAD^{tree}')" = "$EXPECTED_SOURCE_TREE" ] + + archive="$(find "$RUNNER_TEMP/persistent-mac-compile" -type f -name 'persistent-mac-compile.tar.gz' -print -quit)" + [ -n "$archive" ] && [ -f "$archive" ] + mkdir -p "$CMUX_COMPILE_ADMISSION_DERIVED_DATA" + tar -xzf "$archive" -C "$CMUX_COMPILE_ADMISSION_DERIVED_DATA" + metrics="$CMUX_COMPILE_ADMISSION_DERIVED_DATA/persistent-mac-metrics.json" + test -f "$metrics" + + python3 scripts/ci/app_host_test_products.py restore "$CMUX_COMPILE_ADMISSION_DERIVED_DATA" + + METRICS="$metrics" EXPECTED_SOURCE_SHA="$EXPECTED_SOURCE_SHA" EXPECTED_SOURCE_TREE="$EXPECTED_SOURCE_TREE" python3 - <<'PY' + import hashlib + import json + import os + import platform + import subprocess + from pathlib import Path + + metrics_path = Path(os.environ["METRICS"]) + metrics = json.load(open(metrics_path)) + semantic_path = metrics_path.with_name("cmux-workload-result.json") + semantic = json.load(open(semantic_path)) + expected_sha = os.environ["EXPECTED_SOURCE_SHA"] + expected_tree = os.environ["EXPECTED_SOURCE_TREE"] + + def output(*argv): + return subprocess.check_output(argv, text=True).strip() + + def digest(path): + value = hashlib.sha256() + with Path(path).open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + value.update(block) + return value.hexdigest() + + source = metrics.get("source", {}) + if source != {"commit": expected_sha, "tree": expected_tree}: + raise SystemExit("persistent producer source identity mismatch") + if metrics.get("semantic") != semantic: + raise SystemExit("persistent semantic receipt differs from producer metrics") + if semantic.get("document_type") != "cmux-workload-result" or semantic.get("schema_version") != 1: + raise SystemExit("persistent semantic receipt identity mismatch") + if semantic.get("result") != "passed": + raise SystemExit("persistent semantic compile result did not pass") + if semantic.get("source") != { + "repository": "manaflow-ai/cmux", + "commit": expected_sha, + "tree": expected_tree, + }: + raise SystemExit("persistent semantic source identity mismatch") + if semantic.get("profile") != { + "id": "cmux.macos.compile-admission", + "generation": 1, + }: + raise SystemExit("persistent semantic profile identity mismatch") + if semantic.get("semantic_validator") != "cmux.compile-admission/v1": + raise SystemExit("persistent semantic validator mismatch") + validation = semantic.get("validation", {}) + if validation.get("missing_required_artifact_classes") != []: + raise SystemExit("persistent semantic artifact validation incomplete") + cleanup = semantic.get("cleanup", {}) + if cleanup.get("state") != "complete" or cleanup.get("process_group_settled") is not True: + raise SystemExit("persistent semantic cleanup incomplete") + benchmark = semantic.get("benchmark", {}) + state_class = benchmark.get("state_class") + if state_class not in {"cold", "compiler-warm"}: + raise SystemExit("persistent semantic benchmark state is invalid") + for key in ("semantic_comparison_key", "comparison_context_key"): + value = benchmark.get(key, "") + if not isinstance(value, str) or len(value) != 71 or not value.startswith("sha256:"): + raise SystemExit(f"persistent semantic {key} is invalid") + try: + int(value[7:], 16) + except ValueError as error: + raise SystemExit(f"persistent semantic {key} is invalid") from error + + if output("git", "rev-parse", "HEAD") != expected_sha: + raise SystemExit("hosted checkout revision mismatch") + if output("git", "rev-parse", "HEAD^{tree}") != expected_tree: + raise SystemExit("hosted checkout tree mismatch") + if output("git", "status", "--porcelain", "--untracked-files=all"): + raise SystemExit("hosted checkout became dirty during validation") + + lockfile = "cmux.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved" + if digest(lockfile) != metrics.get("package_resolved_sha256"): + raise SystemExit("Package.resolved identity mismatch") + submodules = output("git", "submodule", "status", "--recursive").encode() + if hashlib.sha256(submodules).hexdigest() != metrics.get("submodule_identity_sha256"): + raise SystemExit("submodule identity mismatch") + + toolchain = metrics.get("toolchain", {}) + current_xcode = output("xcodebuild", "-version") + current_sdk = output("xcrun", "--sdk", "macosx", "--show-sdk-version") + if toolchain.get("xcode") != current_xcode: + raise SystemExit("Xcode identity mismatch") + if toolchain.get("sdk_version") != current_sdk: + raise SystemExit("macOS SDK version mismatch") + if toolchain.get("sdk_build") != output("xcrun", "--sdk", "macosx", "--show-sdk-build-version"): + raise SystemExit("macOS SDK build mismatch") + semantic_toolchain = semantic.get("toolchain", {}) + observations = semantic_toolchain.get("observations", {}) + if observations.get("xcode") != current_xcode: + raise SystemExit("persistent semantic Xcode identity mismatch") + if observations.get("macos_sdk") != current_sdk: + raise SystemExit("persistent semantic SDK identity mismatch") + if platform.machine() != "arm64": + raise SystemExit("persistent product validation requires Apple Silicon") + + classification = metrics.get("classification") + if classification not in {"hot", "partially-warm", "cold-reset"}: + raise SystemExit("invalid persistent cache classification") + if state_class == "cold" and classification != "cold-reset": + raise SystemExit("cold semantic state must report cold-reset") + if state_class == "compiler-warm" and classification not in {"hot", "partially-warm"}: + raise SystemExit("warm semantic state has incompatible cache classification") + glaeda = metrics.get("glaeda", {}) + for key in ("cache_key", "invocation_identity", "generation"): + if not isinstance(glaeda.get(key), str) or not glaeda[key]: + raise SystemExit(f"Glaeda receipt omitted {key}") + if not isinstance(glaeda.get("reset_reasons"), list): + raise SystemExit("Glaeda reset lineage is invalid") + print( + f"validated persistent product class={classification} " + f"cache_key={glaeda['cache_key']} invocation={glaeda['invocation_identity']}" + ) + PY + + { + echo "hit=true" + echo "classification=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["classification"])' "$metrics")" + echo "metrics=$metrics" + } >> "$GITHUB_OUTPUT" + - name: Capture Ghostty revision - if: steps.reuse-products.outputs.hit != 'true' + if: steps.reuse-products.outputs.hit != 'true' && steps.persistent-restore.outputs.hit != 'true' id: ghostty-revision run: echo "sha=$(git -C ghostty rev-parse HEAD)" >> "$GITHUB_OUTPUT" - name: Cache GhosttyKit.xcframework - if: steps.reuse-products.outputs.hit != 'true' + if: steps.reuse-products.outputs.hit != 'true' && steps.persistent-restore.outputs.hit != 'true' id: cache-ghosttykit-admission uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: @@ -2600,11 +2926,11 @@ jobs: key: ghosttykit-sentry-off-v1-${{ steps.ghostty-revision.outputs.sha }} - name: Download pre-built GhosttyKit.xcframework - if: steps.reuse-products.outputs.hit != 'true' && steps.cache-ghosttykit-admission.outputs.cache-hit != 'true' + if: steps.reuse-products.outputs.hit != 'true' && steps.persistent-restore.outputs.hit != 'true' && steps.cache-ghosttykit-admission.outputs.cache-hit != 'true' run: ./scripts/download-prebuilt-ghosttykit.sh - name: Cache Swift packages - if: steps.reuse-products.outputs.hit != 'true' + if: steps.reuse-products.outputs.hit != 'true' && steps.persistent-restore.outputs.hit != 'true' uses: ./.github/actions/cache-restore with: backend: ${{ inputs.cache_backend != 'default' && inputs.cache_backend || vars.CI_CACHE_BACKEND }} @@ -2613,19 +2939,24 @@ jobs: restore-keys: spm- - name: Sanitize Swift package cache - if: steps.reuse-products.outputs.hit != 'true' + if: steps.reuse-products.outputs.hit != 'true' && steps.persistent-restore.outputs.hit != 'true' run: python3 scripts/ci/sanitize-xcode-source-packages-cache.py .ci-source-packages - name: Resolve Swift packages - if: steps.reuse-products.outputs.hit != 'true' + id: hosted-package-readiness + if: steps.reuse-products.outputs.hit != 'true' && steps.persistent-restore.outputs.hit != 'true' run: | set -euo pipefail + started="$(python3 -c 'import time; print(time.monotonic())')" rm -rf "$CMUX_COMPILE_ADMISSION_DERIVED_DATA" scripts/ci/compile-app-host-test-product.sh resolve \ "$CMUX_COMPILE_ADMISSION_DERIVED_DATA" "$PWD/.ci-source-packages" + finished="$(python3 -c 'import time; print(time.monotonic())')" + seconds="$(python3 -c 'import sys; print(round(float(sys.argv[2])-float(sys.argv[1]), 6))' "$started" "$finished")" + echo "seconds=$seconds" >> "$GITHUB_OUTPUT" - name: Compute test compilation cache key - if: steps.reuse-products.outputs.hit != 'true' + if: steps.reuse-products.outputs.hit != 'true' && steps.persistent-restore.outputs.hit != 'true' id: compilation-cache-key run: | set -euo pipefail @@ -2635,7 +2966,7 @@ jobs: # only writer. A cache saved here would be scoped to this pull request and # would spend the cache budget that keeps the main seed alive. - name: Restore test compilation cache - if: steps.reuse-products.outputs.hit != 'true' + if: steps.reuse-products.outputs.hit != 'true' && steps.persistent-restore.outputs.hit != 'true' uses: ./.github/actions/cache-restore with: backend: ${{ inputs.cache_backend != 'default' && inputs.cache_backend || vars.CI_CACHE_BACKEND }} @@ -2645,21 +2976,33 @@ jobs: xcode-compilation-test-${{ runner.os }}-${{ runner.arch }}-${{ steps.compilation-cache-key.outputs.fingerprint }}- - name: Compile app-host test product - if: steps.reuse-products.outputs.hit != 'true' + id: hosted-compile + if: steps.reuse-products.outputs.hit != 'true' && steps.persistent-restore.outputs.hit != 'true' run: | set -euo pipefail + started="$(python3 -c 'import time; print(time.monotonic())')" scripts/ci/compile-app-host-test-product.sh build \ "$CMUX_COMPILE_ADMISSION_DERIVED_DATA" \ "$PWD/.ci-source-packages" \ "$CMUX_COMPILE_ADMISSION_CAS" \ "$RUNNER_TEMP/cmux-compile-admission.txt" + finished="$(python3 -c 'import time; print(time.monotonic())')" + seconds="$(python3 -c 'import sys; print(round(float(sys.argv[2])-float(sys.argv[1]), 6))' "$started" "$finished")" + echo "seconds=$seconds" >> "$GITHUB_OUTPUT" # A reuse hit can carry only the compiled product tree. Eligibility requires # the producer's entire compile-admission job to have succeeded, including # this warning gate, so producer-only build logs are unnecessary to restore. - name: Validate Swift warning budget + id: warning-validation if: steps.reuse-products.outputs.hit != 'true' - run: python3 scripts/swift_warning_budget.py --log "$CMUX_COMPILE_ADMISSION_DERIVED_DATA/cmux-build.log" + run: | + set -euo pipefail + started="$(python3 -c 'import time; print(time.monotonic())')" + python3 scripts/swift_warning_budget.py --log "$CMUX_COMPILE_ADMISSION_DERIVED_DATA/cmux-build.log" + finished="$(python3 -c 'import time; print(time.monotonic())')" + seconds="$(python3 -c 'import sys; print(round(float(sys.argv[2])-float(sys.argv[1]), 6))' "$started" "$finished")" + echo "seconds=$seconds" >> "$GITHUB_OUTPUT" - name: Stage compiled package frameworks run: | @@ -2695,6 +3038,9 @@ jobs: # Compile-only forks have no artifact consumers and cannot seed reuse: # both the reuse selector and broker require an in-repository producer. # Keep unknown metadata, every full suite, and in-org reuse producers. + - name: Start product publication timer + run: echo "CMUX_PRODUCT_PUBLICATION_STARTED=$(python3 -c 'import time; print(time.monotonic())')" >> "$GITHUB_ENV" + - name: Choose product artifact publication id: publish-products env: @@ -2742,6 +3088,14 @@ jobs: retention-days: 3 compression-level: 0 + - name: Finish product publication timer + id: publication-timing + if: always() && steps.publish-products.outputs.publish == 'true' + run: | + finished="$(python3 -c 'import time; print(time.monotonic())')" + seconds="$(python3 -c 'import os,sys; print(round(float(sys.argv[1])-float(os.environ["CMUX_PRODUCT_PUBLICATION_STARTED"]), 6))' "$finished")" + echo "seconds=$seconds" >> "$GITHUB_OUTPUT" + - name: Seed node-local compiled product cache if: steps.publish-products.outputs.publish == 'true' continue-on-error: true @@ -2755,6 +3109,132 @@ jobs: CMUX_PRODUCT_PRODUCER_RUN_ID: ${{ github.run_id }} run: python3 scripts/ci/node_product_cache.py seed "$RUNNER_TEMP/app-host-products.tar.gz" + + - name: Record compile admission metrics + id: admission-metrics + if: always() && !cancelled() + env: + ROUTE_USE_PERSISTENT: ${{ steps.persistent-route.outputs.use_persistent }} + ROUTE_REASON: ${{ steps.persistent-route.outputs.fallback_reason }} + QUEUE_TO_START_SECONDS: ${{ steps.persistent-route.outputs.queue_to_start_seconds }} + PRODUCER_ALLOCATED_SECONDS: ${{ steps.persistent-route.outputs.producer_allocated_seconds }} + ROUTE_WALL_SECONDS: ${{ steps.persistent-route.outputs.route_wall_seconds }} + PERSISTENT_HIT: ${{ steps.persistent-restore.outputs.hit }} + PERSISTENT_CLASS: ${{ steps.persistent-restore.outputs.classification }} + PERSISTENT_METRICS: ${{ steps.persistent-restore.outputs.metrics }} + HOSTED_QUEUE_SECONDS: ${{ steps.hosted-queue.outputs.seconds }} + HOSTED_SOURCE_PREP_SECONDS: ${{ steps.hosted-source-prep.outputs.seconds }} + HOSTED_PACKAGE_SECONDS: ${{ steps.hosted-package-readiness.outputs.seconds }} + HOSTED_COMPILE_SECONDS: ${{ steps.hosted-compile.outputs.seconds }} + WARNING_SECONDS: ${{ steps.warning-validation.outputs.seconds }} + PUBLICATION_SECONDS: ${{ steps.publication-timing.outputs.seconds }} + PRODUCT_PUBLISHED: ${{ steps.publish-products.outputs.publish }} + PRODUCT_UPLOAD_OUTCOME: ${{ steps.upload-products.outcome }} + run: | + set -euo pipefail + finished="$(python3 -c 'import time; print(time.monotonic())')" + admission_seconds="$(python3 -c 'import os,sys; print(round(float(sys.argv[1])-float(os.environ["CMUX_MACOS_ADMISSION_STARTED"]), 6))' "$finished")" + metrics="$RUNNER_TEMP/macos-compile-admission-metrics.json" + + ADMISSION_SECONDS="$admission_seconds" METRICS="$metrics" python3 - <<'PY' + import json + import os + + def number(name): + raw = os.environ.get(name, "").strip() + return float(raw) if raw else None + + persistent = {} + persistent_path = os.environ.get("PERSISTENT_METRICS", "").strip() + if persistent_path: + try: + persistent = json.load(open(persistent_path)) + except (OSError, ValueError): + persistent = {} + + persistent_hit = os.environ.get("PERSISTENT_HIT") == "true" + classification = os.environ.get("PERSISTENT_CLASS") if persistent_hit else "hosted fallback" + if persistent_hit: + fallback_reason = "" + elif os.environ.get("ROUTE_USE_PERSISTENT") == "true": + fallback_reason = "persistent_artifact_validation_failed" + else: + fallback_reason = os.environ.get("ROUTE_REASON") or "persistent_route_unused" + + route_wall = number("ROUTE_WALL_SECONDS") or 0.0 + admission_wall = number("ADMISSION_SECONDS") or 0.0 + result = { + "schema_version": 1, + "classification": classification, + "fallback_reason": fallback_reason, + "queue_to_start_seconds": ( + number("QUEUE_TO_START_SECONDS") if persistent_hit else number("HOSTED_QUEUE_SECONDS") + ), + "source_preparation_seconds": ( + persistent.get("source_preparation_seconds") if persistent_hit + else number("HOSTED_SOURCE_PREP_SECONDS") + ), + "package_readiness_seconds": ( + persistent.get("package_readiness_seconds") if persistent_hit + else number("HOSTED_PACKAGE_SECONDS") + ), + "compile_duration_seconds": ( + persistent.get("compile_duration_seconds") if persistent_hit + else number("HOSTED_COMPILE_SECONDS") + ), + "warning_validation_seconds": number("WARNING_SECONDS"), + "artifact_publication_seconds": number("PUBLICATION_SECONDS"), + "route_wall_seconds": number("ROUTE_WALL_SECONDS"), + "required_admission_runner_seconds": number("ADMISSION_SECONDS"), + "total_macos_compile_admission_seconds": route_wall + admission_wall, + "persistent_runner_allocated_seconds": number("PRODUCER_ALLOCATED_SECONDS"), + "hosted_admission_runner_allocated_seconds": number("ADMISSION_SECONDS"), + "product_published": os.environ.get("PRODUCT_PUBLISHED") == "true", + "product_upload_outcome": os.environ.get("PRODUCT_UPLOAD_OUTCOME", ""), + "persistent": persistent, + } + with open(os.environ["METRICS"], "w") as stream: + json.dump(result, stream, sort_keys=True, indent=2) + stream.write("\n") + print(json.dumps(result, sort_keys=True)) + PY + + echo "path=$metrics" >> "$GITHUB_OUTPUT" + METRICS="$metrics" python3 - <<'PY' >> "$GITHUB_STEP_SUMMARY" + import json + import os + + data = json.load(open(os.environ["METRICS"])) + print("### macOS compile admission timing") + print() + for label, key in ( + ("class", "classification"), + ("fallback reason", "fallback_reason"), + ("queue to start", "queue_to_start_seconds"), + ("source preparation", "source_preparation_seconds"), + ("package readiness", "package_readiness_seconds"), + ("compile", "compile_duration_seconds"), + ("warning validation", "warning_validation_seconds"), + ("artifact publication", "artifact_publication_seconds"), + ("route/controller wall", "route_wall_seconds"), + ("required admission runner", "required_admission_runner_seconds"), + ("total admission wall", "total_macos_compile_admission_seconds"), + ("persistent runner allocated", "persistent_runner_allocated_seconds"), + ): + print(f"- {label}: `{data.get(key)}`") + PY + + - name: Upload compile admission metrics + if: always() && !cancelled() && steps.admission-metrics.outputs.path != '' + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: macos-compile-admission-metrics-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ steps.admission-metrics.outputs.path }} + if-no-files-found: error + retention-days: 14 + compression-level: 0 + tests-build-and-lag: permissions: contents: read diff --git a/.github/workflows/persistent-macos-compile.yml b/.github/workflows/persistent-macos-compile.yml new file mode 100644 index 000000000000..f2a96d7d595c --- /dev/null +++ b/.github/workflows/persistent-macos-compile.yml @@ -0,0 +1,268 @@ +name: Persistent macOS compile producer +run-name: persistent-mac-compile-${{ inputs.request_id }} + +on: + workflow_dispatch: + inputs: + request_id: + description: Unique caller identity + required: true + type: string + pr_number: + description: Pull request number + required: true + type: string + source_sha: + description: Exact synthetic merge commit selected by CI + required: true + type: string + source_tree: + description: Exact Git tree selected by CI + required: true + type: string + source_parent1: + description: First parent of the synthetic merge commit + required: true + type: string + head_sha: + description: Trusted same-repository PR head + required: true + type: string + +permissions: {} + +concurrency: + group: persistent-mac-compile-pr-${{ inputs.pr_number }} + cancel-in-progress: true + +jobs: + authorize: + name: Authorize trusted PR + runs-on: ${{ vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }} + timeout-minutes: 3 + permissions: + contents: read + pull-requests: read + steps: + - name: Verify same-repository organization PR + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + PR_NUMBER: ${{ inputs.pr_number }} + EXPECTED_HEAD: ${{ inputs.head_sha }} + EXPECTED_MERGE: ${{ inputs.source_sha }} + EXPECTED_BASE: ${{ inputs.source_parent1 }} + run: | + set -euo pipefail + payload="$(gh api "repos/$GH_REPO/pulls/$PR_NUMBER")" + PAYLOAD="$payload" python3 - <<'PY' + import json + import os + + pr = json.loads(os.environ["PAYLOAD"]) + repository = os.environ["GH_REPO"].casefold() + expected_head = os.environ["EXPECTED_HEAD"] + expected_merge = os.environ["EXPECTED_MERGE"] + expected_base = os.environ["EXPECTED_BASE"] + association = pr.get("author_association") + head = pr.get("head") or {} + head_repo = (head.get("repo") or {}).get("full_name", "") + base = pr.get("base") or {} + checks = { + "open pull request": pr.get("state") == "open", + "same repository": head_repo.casefold() == repository, + "organization member": association in {"MEMBER", "OWNER"}, + "current head": head.get("sha") == expected_head, + "current base": base.get("sha") == expected_base, + "current synthetic merge": pr.get("merge_commit_sha") == expected_merge, + } + failed = [name for name, passed in checks.items() if not passed] + if failed: + raise SystemExit("persistent Mac authorization refused: " + ", ".join(failed)) + print(f"authorized PR #{pr['number']} at {expected_head} ({association})") + PY + + compile: + name: Persistent Apple compile + needs: authorize + runs-on: + group: cmux-persistent-compile + labels: [self-hosted, macOS, ARM64, cmux-persistent-macos-compile] + timeout-minutes: 35 + permissions: {} + env: + CMUX_CI_XCODE_APP: ${{ vars.CMUX_CI_XCODE_APP_MACOS_15 }} + CMUX_CI_REQUIRED_MACOS_SDK_MAJOR: "26" + CMUX_SKIP_ZIG_BUILD: "1" + GLAEDA_REF: 3e7001b3702459df2e6177c6beacc715215a7aa0 + steps: + - name: Prepare exact cmux source + id: source + env: + SOURCE_SHA: ${{ inputs.source_sha }} + SOURCE_TREE: ${{ inputs.source_tree }} + SOURCE_PARENT1: ${{ inputs.source_parent1 }} + HEAD_SHA: ${{ inputs.head_sha }} + PR_NUMBER: ${{ inputs.pr_number }} + run: | + set -euo pipefail + started="$(python3 -c 'import time; print(time.monotonic())')" + workspace="$GITHUB_WORKSPACE" + source_reset=false + cd "$RUNNER_TEMP" + + initialize_checkout() { + source_reset=true + rm -rf "$workspace" + mkdir -p "$workspace" + git -C "$workspace" init -q + git -C "$workspace" remote add origin https://github.com/manaflow-ai/cmux.git + } + + if [ ! -d "$workspace/.git" ]; then + initialize_checkout + else + origin="$(git -C "$workspace" remote get-url origin 2>/dev/null || true)" + case "$origin" in + https://github.com/manaflow-ai/cmux.git|https://github.com/manaflow-ai/cmux) ;; + *) initialize_checkout ;; + esac + fi + + if [ -n "$(git -C "$workspace" status --porcelain --untracked-files=all 2>/dev/null || true)" ]; then + source_reset=true + fi + git -C "$workspace" reset --hard -q || initialize_checkout + git -C "$workspace" submodule foreach --recursive 'git reset --hard -q; git clean -ffdx -q' || true + git -C "$workspace" clean -ffdx -q -e .glaeda/ -e GhosttyKit.xcframework + + if ! git -C "$workspace" fetch --no-tags --force origin "$SOURCE_SHA"; then + git -C "$workspace" fetch --no-tags --force origin "refs/pull/$PR_NUMBER/merge" + fi + fetched="$(git -C "$workspace" rev-parse FETCH_HEAD)" + [ "$fetched" = "$SOURCE_SHA" ] || { + echo "fetched source $fetched differs from caller source $SOURCE_SHA" >&2 + exit 1 + } + + git -C "$workspace" checkout --detach -f "$SOURCE_SHA" + git -C "$workspace" submodule sync --recursive + git -C "$workspace" submodule update --init --recursive --force + git -C "$workspace" submodule foreach --recursive 'git reset --hard -q; git clean -ffdx -q' + + actual_tree="$(git -C "$workspace" rev-parse 'HEAD^{tree}')" + read -r actual_commit actual_parent1 actual_parent2 extra <> "$GITHUB_OUTPUT" + echo "prepared exact source commit=$SOURCE_SHA tree=$SOURCE_TREE reset=$source_reset seconds=$seconds" + + - name: Select Xcode + working-directory: ${{ github.workspace }} + run: ./scripts/select-ci-xcode.sh + + - name: Install Rust + working-directory: ${{ github.workspace }} + run: ./scripts/install-rust-ci.sh + + - name: Fetch pinned Glaeda native Apple front door + id: glaeda + env: + GLAEDA_REF: ${{ env.GLAEDA_REF }} + run: | + set -euo pipefail + root="$RUNNER_TEMP/glaeda" + rm -rf "$root" + mkdir -p "$root" + git -C "$root" init -q + git -C "$root" remote add origin https://github.com/teamleaderleo/glaeda.git + git -C "$root" fetch --no-tags --depth=1 origin "$GLAEDA_REF" + git -C "$root" checkout --detach -q "$GLAEDA_REF" + [ "$(git -C "$root" rev-parse HEAD)" = "$GLAEDA_REF" ] + test -x "$root/scripts/apple-build" + echo "apple_build=$root/scripts/apple-build" >> "$GITHUB_OUTPUT" + + - name: Run Glaeda compile admission + id: compile + working-directory: ${{ github.workspace }} + env: + EXPECTED_COMMIT: ${{ inputs.source_sha }} + EXPECTED_TREE: ${{ inputs.source_tree }} + REQUEST_ID: ${{ inputs.request_id }} + SOURCE_PREP_SECONDS: ${{ steps.source.outputs.seconds }} + SOURCE_RESET: ${{ steps.source.outputs.source_reset }} + GLAEDA_APPLE_BUILD: ${{ steps.glaeda.outputs.apple_build }} + run: | + set -euo pipefail + metrics="$RUNNER_TEMP/persistent-mac-metrics.json" + python3 scripts/ci/run-persistent-mac-compile.py --glaeda "$GLAEDA_APPLE_BUILD" --expected-commit "$EXPECTED_COMMIT" --expected-tree "$EXPECTED_TREE" --request-id "$REQUEST_ID" --source-preparation-seconds "$SOURCE_PREP_SECONDS" --source-reset "$SOURCE_RESET" --metrics "$metrics" + + - name: Stamp relocatable app-host products + working-directory: ${{ github.workspace }} + env: + DERIVED_DATA: ${{ steps.compile.outputs.derived_data }} + run: python3 scripts/ci/app_host_test_products.py stamp "$DERIVED_DATA" + + - name: Package producer evidence + id: package + env: + DERIVED_DATA: ${{ steps.compile.outputs.derived_data }} + METRICS: ${{ steps.compile.outputs.metrics }} + run: | + set -euo pipefail + archive="$RUNNER_TEMP/persistent-mac-compile.tar.gz" + cp "$METRICS" "$DERIVED_DATA/persistent-mac-metrics.json" + test -f "$DERIVED_DATA/cmux-workload-result.json" + tar -chzf "$archive" -C "$DERIVED_DATA" \ + Build/Products \ + cmux-build.log \ + cmux-workload-result.json \ + persistent-mac-metrics.json + echo "archive=$archive" >> "$GITHUB_OUTPUT" + + - name: Upload persistent compile product + id: upload + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: persistent-mac-compile-${{ inputs.request_id }} + path: ${{ steps.package.outputs.archive }} + if-no-files-found: error + retention-days: 1 + compression-level: 0 + + - name: Summarize producer + if: always() && steps.compile.outcome == 'success' + env: + METRICS: ${{ steps.compile.outputs.metrics }} + ARTIFACT_ID: ${{ steps.upload.outputs.artifact-id }} + run: | + python3 - <<'PY' >> "$GITHUB_STEP_SUMMARY" + import json + import os + + data = json.load(open(os.environ["METRICS"])) + print("### Persistent Mac compile producer") + print() + print(f"- class: `{data['classification']}`") + print(f"- source preparation: {data['source_preparation_seconds']:.3f}s") + print(f"- package readiness: {data['package_readiness_seconds']:.3f}s") + print(f"- compile: {data['compile_duration_seconds']:.3f}s") + print(f"- semantic validation: {data['semantic_validation_seconds']:.3f}s") + print(f"- semantic profile: `{data['semantic']['profile']['id']}@{data['semantic']['profile']['generation']}`") + print(f"- semantic key: `{data['semantic']['benchmark']['semantic_comparison_key']}`") + print(f"- Glaeda generation: `{data['glaeda']['generation']}`") + print(f"- Glaeda cache key: `{data['glaeda']['cache_key']}`") + print(f"- artifact id: `{os.environ.get('ARTIFACT_ID', '')}`") + PY diff --git a/.github/workflows/persistent-macos-router.yml b/.github/workflows/persistent-macos-router.yml new file mode 100644 index 000000000000..dc1cc9136c2a --- /dev/null +++ b/.github/workflows/persistent-macos-router.yml @@ -0,0 +1,187 @@ +name: Persistent macOS compile router +run-name: persistent-mac-router-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }} + +on: + workflow_run: + workflows: [CI] + types: [in_progress] + +permissions: {} + +concurrency: + group: persistent-mac-router-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }} + cancel-in-progress: true + +jobs: + route: + name: Route persistent Mac compile + if: ${{ github.event.workflow_run.event == 'pull_request' && (vars.CI_PERSISTENT_MAC_COMPILE == 'pilot' || vars.CI_PERSISTENT_MAC_COMPILE == 'all' || vars.CI_PERSISTENT_MAC_COMPILE == 'on' || vars.CI_PERSISTENT_MAC_COMPILE == 'true' || vars.CI_PERSISTENT_MAC_COMPILE == '1') }} + runs-on: ${{ vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }} + timeout-minutes: 15 + permissions: + actions: write + contents: read + pull-requests: read + steps: + - name: Checkout trusted router + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: main + persist-credentials: false + + - name: Find exact CI route request + id: request-artifact + env: + GH_TOKEN: ${{ github.token }} + CI_RUN_ID: ${{ github.event.workflow_run.id }} + CI_RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} + run: | + set -euo pipefail + name="persistent-mac-route-request-$CI_RUN_ID-$CI_RUN_ATTEMPT" + deadline=$(( $(date +%s) + 180 )) + artifact_id="" + while [ "$(date +%s)" -lt "$deadline" ]; do + payload="$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$CI_RUN_ID/artifacts?name=$name&per_page=100")" + count="$(python3 -c 'import json,sys; print(len(json.load(sys.stdin).get("artifacts", [])))' <<<"$payload")" + if [ "$count" -gt 1 ]; then + echo "multiple persistent route request artifacts found" >&2 + exit 1 + fi + if [ "$count" -eq 1 ]; then + artifact_id="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["artifacts"][0]["id"])' <<<"$payload")" + break + fi + sleep 2 + done + if [ -z "$artifact_id" ]; then + echo "No trusted persistent route request was published; hosted admission remains authoritative." + echo "found=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + { + echo "found=true" + echo "artifact_id=$artifact_id" + } >> "$GITHUB_OUTPUT" + + - name: Download exact CI route request + if: ${{ steps.request-artifact.outputs.found == 'true' }} + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + artifact-ids: ${{ steps.request-artifact.outputs.artifact_id }} + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + path: ${{ runner.temp }}/persistent-mac-route-request + + - name: Validate route request envelope + id: request + if: ${{ steps.request-artifact.outputs.found == 'true' }} + env: + CI_RUN_ID: ${{ github.event.workflow_run.id }} + CI_RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} + run: | + set -euo pipefail + request="$(find "$RUNNER_TEMP/persistent-mac-route-request" -type f -name 'persistent-mac-route-request.json' -print -quit)" + [ -n "$request" ] && [ -f "$request" ] + REQUEST="$request" python3 - <<'PY' + import json + import os + import re + + payload = json.load(open(os.environ["REQUEST"], encoding="utf-8")) + required = { + "schema_version", "request_id", "pr_number", "source_sha", "source_tree", + "source_parent1", "head_sha", "head_ref", "head_repository", + "author_association", + } + if set(payload) != required or payload["schema_version"] != 1: + raise SystemExit("invalid persistent route request schema") + expected_id = f"{os.environ['CI_RUN_ID']}-{os.environ['CI_RUN_ATTEMPT']}" + if payload["request_id"] != expected_id: + raise SystemExit("persistent route request identity mismatch") + if not re.fullmatch(r"[1-9][0-9]*", str(payload["pr_number"])): + raise SystemExit("invalid persistent route pull request number") + for key in ("source_sha", "source_tree", "source_parent1", "head_sha"): + if not isinstance(payload[key], str) or not re.fullmatch(r"[a-f0-9]{40,64}", payload[key]): + raise SystemExit(f"invalid persistent route {key}") + if payload["head_repository"].casefold() != os.environ["GITHUB_REPOSITORY"].casefold(): + raise SystemExit("persistent route request is not same-repository") + for key in ("head_ref", "head_repository", "author_association"): + value = payload[key] + if not isinstance(value, str) or not value or len(value) > 240 or "\n" in value or "\r" in value: + raise SystemExit(f"invalid persistent route {key}") + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + for key in ( + "pr_number", "source_sha", "source_tree", "source_parent1", + "head_sha", "head_ref", "head_repository", "author_association", + ): + output.write(f"{key}={payload[key]}\n") + PY + + - name: Dispatch and bound persistent compile + id: route + if: ${{ steps.request-artifact.outputs.found == 'true' }} + env: + GH_TOKEN: ${{ github.token }} + CI_PERSISTENT_MAC_COMPILE: ${{ vars.CI_PERSISTENT_MAC_COMPILE }} + CI_PERSISTENT_MAC_COMPILE_COHORT: ${{ vars.CI_PERSISTENT_MAC_COMPILE_COHORT }} + CI_PERSISTENT_MAC_QUEUE_SECONDS: ${{ vars.CI_PERSISTENT_MAC_QUEUE_SECONDS }} + CI_PERSISTENT_MAC_EXECUTION_SECONDS: ${{ vars.CI_PERSISTENT_MAC_EXECUTION_SECONDS }} + PR_NUMBER: ${{ steps.request.outputs.pr_number }} + HEAD_REPOSITORY: ${{ steps.request.outputs.head_repository }} + HEAD_REF: ${{ steps.request.outputs.head_ref }} + AUTHOR_ASSOCIATION: ${{ steps.request.outputs.author_association }} + HEAD_SHA: ${{ steps.request.outputs.head_sha }} + SOURCE_SHA: ${{ steps.request.outputs.source_sha }} + SOURCE_TREE: ${{ steps.request.outputs.source_tree }} + SOURCE_PARENT1: ${{ steps.request.outputs.source_parent1 }} + CI_SOURCE_RUN_ID: ${{ github.event.workflow_run.id }} + CI_SOURCE_RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} + run: | + set -euo pipefail + queue_seconds="${CI_PERSISTENT_MAC_QUEUE_SECONDS:-90}" + execution_seconds="${CI_PERSISTENT_MAC_EXECUTION_SECONDS:-480}" + python3 scripts/ci/persistent_mac_route.py \ + --event-name pull_request \ + --selector "$CI_PERSISTENT_MAC_COMPILE" \ + --cohort "$CI_PERSISTENT_MAC_COMPILE_COHORT" \ + --repository "$GITHUB_REPOSITORY" \ + --pr-number "$PR_NUMBER" \ + --head-repository "$HEAD_REPOSITORY" \ + --head-ref "$HEAD_REF" \ + --author-association "$AUTHOR_ASSOCIATION" \ + --head-sha "$HEAD_SHA" \ + --source-sha "$SOURCE_SHA" \ + --source-tree "$SOURCE_TREE" \ + --source-parent1 "$SOURCE_PARENT1" \ + --run-id "$CI_SOURCE_RUN_ID" \ + --run-attempt "$CI_SOURCE_RUN_ATTEMPT" \ + --queue-seconds "$queue_seconds" \ + --execution-seconds "$execution_seconds" \ + --github-output "$GITHUB_OUTPUT" + + - name: Record router evidence + if: ${{ always() && steps.request-artifact.outputs.found == 'true' }} + env: + REQUEST_ID: ${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }} + USE_PERSISTENT: ${{ steps.route.outputs.use_persistent }} + FALLBACK_REASON: ${{ steps.route.outputs.fallback_reason }} + PRODUCER_RUN_ID: ${{ steps.route.outputs.producer_run_id }} + ARTIFACT_ID: ${{ steps.route.outputs.artifact_id }} + QUEUE_SECONDS: ${{ steps.route.outputs.queue_to_start_seconds }} + ALLOCATED_SECONDS: ${{ steps.route.outputs.producer_allocated_seconds }} + run: | + python3 - <<'PY' >> "$GITHUB_STEP_SUMMARY" + import os + print("### Persistent Mac router") + print() + for label, name in ( + ("request", "REQUEST_ID"), + ("use persistent", "USE_PERSISTENT"), + ("fallback reason", "FALLBACK_REASON"), + ("producer run", "PRODUCER_RUN_ID"), + ("artifact", "ARTIFACT_ID"), + ("queue-to-start seconds", "QUEUE_SECONDS"), + ("allocated runner seconds", "ALLOCATED_SECONDS"), + ): + print(f"- {label}: `{os.environ.get(name, '')}`") + PY diff --git a/.gitignore b/.gitignore index 1e2e490f2db1..092dc0b271b3 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ ios/Config/AppStoreConnect.local.plist .swiftpm/ .ci-source-packages/ .ci-reload-derived-data/ +.glaeda/ # GhosttyKit binary (built from ghostty submodule via scripts/setup.sh) GhosttyKit.xcframework diff --git a/CLAUDE.md b/CLAUDE.md index e1f03a83b0be..0de5ee1fa6e9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -7,10 +7,23 @@ ## Dev builds on the Mac mini fleet For team dev builds, use the controller client `~/.local/bin/cmux-ci`. The Mac -mini fleet is **dev-build-only** for now. GitHub CI/CD, required checks, merge -queue checks, nightly/release automation, and TestFlight remain on their existing -hosted runners, including Blacksmith. A successful dev build does not replace -those checks. +mini fleet is **dev-build-only** except for the bounded compile-admission pilot +in `.github/workflows/persistent-macos-compile.yml`. That dispatch-only producer +may compile Debug app-host products for trusted same-repository organization +pull requests behind `CI_PERSISTENT_MAC_COMPILE`; dispatch/cancellation live +only in the default-branch `persistent-macos-router.yml` workflow so PR CI +retains read-only Actions permission. Its owned runner must live in the +workflow-restricted `cmux-persistent-compile` runner group pinned to the +producer workflow on `refs/heads/main`. The required +`macOS compile admission` job remains the check/log/artifact owner and +revalidates the producer before adoption. Glaeda owns native cache/admission +while CMUX's canonical `cmux.macos.compile-admission@1` profile owns semantic +pass/fail; the producer refuses candidates where that profile is absent. +Release, signing, notarization, +nightly, TestFlight, merge-queue policy, generic agent execution, and every GUI +or runtime test remain on their existing lanes. The producer receives no +repository secrets, and hosted compile fallback remains live. A successful dev +build or persistent producer run never replaces the required check. Before submitting, read the current [HQ AGENTS.md](https://github.com/manaflow-ai/cmuxterm-hq/blob/main/AGENTS.md) and [agent build contract](https://github.com/manaflow-ai/cmuxterm-hq/blob/main/build-fleet/AGENT-BUILDS.md). diff --git a/docs/ci-runners.md b/docs/ci-runners.md index be441a2f464e..282ba6987bda 100644 --- a/docs/ci-runners.md +++ b/docs/ci-runners.md @@ -23,6 +23,65 @@ missing variable. Pull requests from forks never see repository variables, so the fallback is where they always run: it must be a Blacksmith label, never the paid Warp overflow. `tests/test_ci_self_hosted_guard.sh` enforces that. +## Persistent compile-admission pilot + +`macos-compile-admission` has one narrow owned-Mac producer path for trusted, +same-repository organization-member pull requests. The required +`macOS compile admission` job remains on the ordinary paid macOS runner and +remains the check, log, validation, and artifact-publication owner. It may +consume a compile product from `.github/workflows/persistent-macos-compile.yml` +after revalidating the Git revision/tree, Xcode, SDK, architecture, +`Package.resolved`, submodules, Glaeda lineage evidence, warning budget, and +early CLI probes. Any dispatch, queue, execution, download, or validation miss +falls through to the existing hosted compile in that same required job. +The required hosted macOS job is allocated without waiting for the persistent +producer. It restores any exact reusable product first, then observes the +producer with read-only Actions permission before deciding whether to consume +the persistent artifact or compile hosted. The PR workflow never receives +Actions write authority: `changes` publishes a small exact-source request +artifact, and the default-branch `persistent-macos-router.yml` workflow +validates it against the live PR and owns producer dispatch/cancellation. + +The producer is `workflow_dispatch`-only and requires the +`cmux-persistent-compile` runner group plus the dedicated +`cmux-persistent-macos-compile` label. Before rollout, the organization-owned +runner group must allow this public repository and restrict workflow access to +`manaflow-ai/cmux/.github/workflows/persistent-macos-compile.yml@refs/heads/main`. +That group policy is the external scheduling boundary: branch-modified workflow +copies cannot acquire the owned Mac. The compile job has empty GitHub-token +permissions, performs public Git fetches instead of `actions/checkout`, and +receives no repository secrets. Glaeda owns native cache placement, admission, +recovery, and exact-source locking. CMUX owns compile semantics through +`cmux.macos.compile-admission@1` and its `cmux-workload-result/v1` receipt. +The persistent wrapper refuses execution when that canonical profile/generation +is absent, so rollout begins only after #13411 is present in the candidate +source. DerivedData, SwiftPM, module-cache, and Xcode compilation-cache state +then remain resident under Glaeda while the canonical CMUX entrypoint still +resolves dependencies, compiles, validates the warning budget, records named +stage timings, and proves artifact/process cleanup. + +The current hosted admission path remains the comparison baseline and the +fallback authority. Persistent results additionally carry the canonical semantic +comparison key and context key; compare those keys only across runs produced by +the same profile/generation and toolchain context. + +Rollout is reversible through two repository variables: + +- `CI_PERSISTENT_MAC_COMPILE=off` (or unset): hosted path only; +- `CI_PERSISTENT_MAC_COMPILE=pilot` with + `CI_PERSISTENT_MAC_COMPILE_COHORT=13198,feature/name`: only matching trusted + PR numbers or head branches; +- `CI_PERSISTENT_MAC_COMPILE=all`: every trusted same-repository + organization-member PR. + +Queue and execution ceilings may be set with +`CI_PERSISTENT_MAC_QUEUE_SECONDS` and +`CI_PERSISTENT_MAC_EXECUTION_SECONDS`; defaults are 90 and 480 seconds. +Admission publishes timing evidence for source preparation, package readiness, +compile, warning validation, product publication, total wall time, runner time, +and the `hot` / `partially-warm` / `cold-reset` / `hosted fallback` +classification. + ## Tart isolation and capacity Each GitHub runner identity is sealed into a Tart template. A job runs in a @@ -139,17 +198,21 @@ The fleet-label guard allows Tart labels only as exact manual canary choices. Required jobs continue to reference repository variables, so cutover and break-glass remain configuration changes instead of workflow edits. -## No direct physical-host runners - -We do not route required CI directly to the persistent self-hosted mac-mini -fleet (`cmux-mac-mini`, `studio1`, `mac4-cmuxvnc*`, -`cmux-austin-mini-*`). Those minis carry labels that collide with cloud labels -(notably `macos-26` and `warp-macos-26-arm64-6x`), and GitHub prefers a matching -self-hosted runner, so a required job could silently land on a mini that cannot -foreground a GUI app. It stays `Running Background`, breaking key-window, -pasteboard, IME, and XCUITest behavior. Every macOS fallback therefore routes -to Blacksmith cloud, and -`check_no_self_hosted_fleet_runners` in `tests/test_ci_self_hosted_guard.sh` -fails CI if a required workflow hardcodes a fleet label. Repository variables -may point to the isolated `tart-*` pool. Legacy physical runner services remain -disabled and their GitHub records remain offline for rollback. +## Direct physical-host runner boundary + +Required GUI, test, Release, signing, and ordinary macOS jobs never route to +the persistent self-hosted mac-mini fleet (`cmux-mac-mini`, `studio1`, +`mac4-cmuxvnc*`, `cmux-austin-mini-*`). Those records can collide with cloud +labels and lack the isolated foreground GUI guarantees expected by runtime +tests. + +The sole direct-host exception is the dispatch-only +`Persistent Apple compile` producer described above, selected by its dedicated +workflow-restricted `cmux-persistent-compile` runner group and +`cmux-persistent-macos-compile` label. It performs compile-only Debug work, +carries no repository secrets, and grants its hot state zero result authority. +Every required macOS fallback still routes to the paid hosted path. +`check_no_self_hosted_fleet_runners` in +`tests/test_ci_self_hosted_guard.sh` enforces that exact exception and rejects +any second required-job or generic fleet route. Repository variables may keep +pointing at the isolated `tart-*` pool for their existing jobs. diff --git a/glaeda.apple.json b/glaeda.apple.json new file mode 100644 index 000000000000..b03025d6e07b --- /dev/null +++ b/glaeda.apple.json @@ -0,0 +1,21 @@ +{ + "schema_version": 1, + "profiles": { + "ci-compile-admission": { + "engine": "script", + "sdk": "macosx", + "executable": "scripts/ci/persistent-mac-semantic-entrypoint.sh", + "arguments": [ + "{products}/semantic-state", + "{products}/semantic-result.json" + ], + "environment": { + "CMUX_CI_MODULE_CACHE_PATH": "{module_cache}", + "CMUX_SKIP_ZIG_BUILD": "1" + } + } + }, + "cache_policies": { + "ci-compile-admission": "native" + } +} diff --git a/scripts/ci/compile-app-host-test-product.sh b/scripts/ci/compile-app-host-test-product.sh index 8c75410e633b..e0ce973b8510 100755 --- a/scripts/ci/compile-app-host-test-product.sh +++ b/scripts/ci/compile-app-host-test-product.sh @@ -66,7 +66,12 @@ resolve() { build() { local derived_data="$1" source_packages="$2" cas_path="$3" log="${4:-/dev/null}" + local -a module_cache_setting=() mkdir -p "$cas_path" "$derived_data" + if [ -n "${CMUX_CI_MODULE_CACHE_PATH:-}" ]; then + mkdir -p "$CMUX_CI_MODULE_CACHE_PATH" + module_cache_setting=("CLANG_MODULE_CACHE_PATH=$CMUX_CI_MODULE_CACHE_PATH") + fi # Build the app/UI scheme first so its warning log retains the old runtime # job warning-budget scope; subsequent schemes reuse the same app objects. @@ -82,6 +87,7 @@ build() { COMPILATION_CACHE_ENABLE_CACHING=YES \ "COMPILATION_CACHE_CAS_PATH=$cas_path" \ "COMPILATION_CACHE_LIMIT_SIZE=$cache_limit_bytes" \ + ${module_cache_setting[@]+"${module_cache_setting[@]}"} \ build-for-testing 2>&1 | tee "$derived_data/$scheme-build.log" | tee -a "$log" done } diff --git a/scripts/ci/persistent-mac-semantic-entrypoint.sh b/scripts/ci/persistent-mac-semantic-entrypoint.sh new file mode 100755 index 000000000000..1069433d1f80 --- /dev/null +++ b/scripts/ci/persistent-mac-semantic-entrypoint.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +root="$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd)" +state_root="${1:?semantic state root is required}" +result_path="${2:?semantic result path is required}" +runner="$root/scripts/ci/cmux_workload_profile.py" +registry="$root/scripts/ci/cmux-workload-profiles.json" + +if [ ! -f "$runner" ] || [ ! -f "$registry" ]; then + echo "canonical cmux.macos.compile-admission@1 is unavailable; keep persistent routing disabled until #13411 is on the exact source" >&2 + exit 78 +fi + +python3 "$runner" describe cmux.macos.compile-admission | + python3 -c 'import json,sys; value=json.load(sys.stdin); assert value["id"] == "cmux.macos.compile-admission" and value["generation"] == 1' + +state_class=cold +if [ -d "$state_root" ] && [ -n "$(find "$state_root" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)" ]; then + state_class=compiler-warm +fi + +exec python3 "$runner" run cmux.macos.compile-admission \ + --generation 1 \ + --state-class "$state_class" \ + --state-root "$state_root" \ + --result "$result_path" diff --git a/scripts/ci/persistent_mac_route.py b/scripts/ci/persistent_mac_route.py new file mode 100644 index 000000000000..db40f46d7c35 --- /dev/null +++ b/scripts/ci/persistent_mac_route.py @@ -0,0 +1,471 @@ +#!/usr/bin/env python3 +"""Dispatch the bounded persistent-Mac compile pilot, with hosted fallback.""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import json +import os +import signal +import subprocess +import sys +import threading +import time +from collections.abc import Callable +from pathlib import Path + + +WORKFLOW = "persistent-macos-compile.yml" +JOB_NAME = "Persistent Apple compile" +TERMINAL = {"completed"} + + +def now() -> float: + return time.monotonic() + + +def parse_time(value: str | None) -> datetime | None: + if not value: + return None + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +class RoutingCanceled(RuntimeError): + """Raised when GitHub cancels an in-progress routing observation.""" + + +def install_cancel_event() -> threading.Event: + """Translate process cancellation signals into one shared observation event.""" + event = threading.Event() + + def cancel_observation(_signum: int, _frame: object) -> None: + event.set() + + signal.signal(signal.SIGINT, cancel_observation) + signal.signal(signal.SIGTERM, cancel_observation) + return event + + +def observe_until( + deadline: float, + interval_seconds: float, + observe: Callable[[], object | None], + *, + cancel_event: threading.Event, + monotonic: Callable[[], float] = now, +) -> object | None: + """Observe bounded external state until ready, canceled, or the deadline passes.""" + while True: + value = observe() + if value is not None: + return value + remaining = deadline - monotonic() + if remaining <= 0: + return None + if cancel_event.wait(min(interval_seconds, remaining)): + raise RoutingCanceled("routing observation canceled") + + +class GitHub: + def __init__(self, repository: str): + self.repository = repository + + def api(self, path: str, *, method: str = "GET") -> object: + result = subprocess.run( + ["gh", "api", "--method", method, f"repos/{self.repository}/{path}"], + text=True, + capture_output=True, + check=False, + ) + if result.returncode: + raise RuntimeError(result.stderr.strip() or f"gh api exited {result.returncode}") + return json.loads(result.stdout) if result.stdout.strip() else {} + + def dispatch(self, fields: dict[str, str]) -> None: + argv = [ + "gh", + "workflow", + "run", + WORKFLOW, + "--repo", + self.repository, + "--ref", + "main", + ] + for key, value in fields.items(): + argv.extend(["--field", f"{key}={value}"]) + result = subprocess.run(argv, text=True, capture_output=True, check=False) + if result.returncode: + raise RuntimeError(result.stderr.strip() or f"workflow dispatch exited {result.returncode}") + + +def cohort_match(cohort: str, pr_number: str, head_ref: str) -> bool: + values = {value.strip() for value in cohort.split(",") if value.strip()} + return pr_number in values or head_ref in values + + +def eligibility(args: argparse.Namespace) -> tuple[bool, str]: + selector = args.selector.strip().lower() + if selector in {"", "0", "off", "false"}: + return False, "selector_off" + if args.event_name != "pull_request": + return False, "event_not_pull_request" + if args.head_repository.casefold() != args.repository.casefold(): + return False, "untrusted_repository" + if args.author_association not in {"MEMBER", "OWNER"}: + return False, "untrusted_author" + if selector == "pilot": + if cohort_match(args.cohort, args.pr_number, args.head_ref): + return True, "pilot" + return False, "outside_pilot_cohort" + if selector in {"1", "on", "true", "all"}: + return True, "all" + return False, "invalid_selector" + + +def valid_budget(queue_seconds: int, execution_seconds: int) -> bool: + return ( + 15 <= queue_seconds <= 120 + and 60 <= execution_seconds <= 480 + and queue_seconds + execution_seconds <= 600 + ) + + +def valid_observe_budget(observe_seconds: int) -> bool: + """Bound how long a hosted admission runner may wait for persistent output.""" + return 5 <= observe_seconds <= 60 + + +def verify_live_request(api: GitHub, args: argparse.Namespace) -> tuple[bool, str]: + source_run = api.api(f"actions/runs/{args.run_id}") + if not isinstance(source_run, dict): + return False, "source_run_observation_invalid" + try: + observed_attempt = int(source_run.get("run_attempt")) + expected_attempt = int(args.run_attempt) + expected_pr = int(args.pr_number) + except (TypeError, ValueError): + return False, "source_run_identity_invalid" + source_pull_requests = source_run.get("pull_requests") + if not isinstance(source_pull_requests, list): + return False, "source_run_pr_observation_invalid" + matching_prs = [ + item for item in source_pull_requests + if isinstance(item, dict) and item.get("number") == expected_pr + ] + run_repository = (source_run.get("head_repository") or {}).get("full_name", "") + source_checks = { + "source_run_event_mismatch": source_run.get("event") == "pull_request", + "source_run_attempt_mismatch": observed_attempt == expected_attempt, + "source_run_workflow_mismatch": source_run.get("path") == ".github/workflows/ci.yml", + "source_run_source_mismatch": source_run.get("head_sha") == args.source_sha, + "source_run_repository_mismatch": str(run_repository).casefold() == args.repository.casefold(), + "source_run_pr_mismatch": len(matching_prs) == 1, + "source_run_pr_head_mismatch": ( + len(matching_prs) == 1 + and (matching_prs[0].get("head") or {}).get("sha") == args.head_sha + ), + "source_run_pr_base_mismatch": ( + len(matching_prs) == 1 + and (matching_prs[0].get("base") or {}).get("sha") == args.source_parent1 + ), + } + for reason, passed in source_checks.items(): + if not passed: + return False, reason + + pr = api.api(f"pulls/{args.pr_number}") + if not isinstance(pr, dict): + return False, "pr_observation_invalid" + head = pr.get("head") or {} + base = pr.get("base") or {} + head_repo = (head.get("repo") or {}).get("full_name", "") + checks = { + "pr_closed": pr.get("state") == "open", + "untrusted_repository": str(head_repo).casefold() == args.repository.casefold(), + "untrusted_author": pr.get("author_association") in {"MEMBER", "OWNER"}, + "head_changed": head.get("sha") == args.head_sha, + "base_changed": base.get("sha") == args.source_parent1, + "merge_changed": pr.get("merge_commit_sha") == args.source_sha, + } + for reason, passed in checks.items(): + if not passed: + return False, reason + commit = api.api(f"git/commits/{args.source_sha}") + tree = (commit.get("tree") or {}).get("sha") if isinstance(commit, dict) else None + if tree != args.source_tree: + return False, "tree_changed" + return True, "verified" + + +def find_run( + api: GitHub, + request_id: str, + deadline: float, + cancel_event: threading.Event, +) -> dict[str, object]: + """Find the exact dispatched producer run within a bounded observation window.""" + title = f"persistent-mac-compile-{request_id}" + + def observe() -> dict[str, object] | None: + payload = api.api(f"actions/workflows/{WORKFLOW}/runs?event=workflow_dispatch&per_page=50") + runs = payload.get("workflow_runs", []) if isinstance(payload, dict) else [] + matches = [ + run for run in runs + if isinstance(run, dict) + and run.get("display_title") == title + and run.get("head_branch") == "main" + ] + if not matches: + return None + matches.sort(key=lambda run: int(run.get("id", 0)), reverse=True) + return matches[0] + + found = observe_until( + deadline, + 1.0, + observe, + cancel_event=cancel_event, + ) + if not isinstance(found, dict): + raise RuntimeError("dispatched producer workflow did not become observable") + return found + + +def jobs(api: GitHub, run_id: int) -> list[dict[str, object]]: + payload = api.api(f"actions/runs/{run_id}/jobs?filter=latest&per_page=100") + if not isinstance(payload, dict) or not isinstance(payload.get("jobs"), list): + raise RuntimeError("producer job list is malformed") + return [job for job in payload["jobs"] if isinstance(job, dict)] + + +def compile_job(api: GitHub, run_id: int) -> dict[str, object] | None: + matched = [job for job in jobs(api, run_id) if job.get("name") == JOB_NAME] + if len(matched) > 1: + raise RuntimeError("producer run has multiple compile jobs") + return matched[0] if matched else None + + +def cancel(api: GitHub, run_id: int) -> None: + try: + api.api(f"actions/runs/{run_id}/cancel", method="POST") + except RuntimeError as error: + print(f"warning: producer cancellation failed: {error}", file=sys.stderr) + + +def write_outputs(path: Path, values: dict[str, object]) -> None: + with path.open("a", encoding="utf-8") as stream: + for key, value in values.items(): + stream.write(f"{key}={value}\n") + + +def fallback(output: Path, reason: str, **extra: object) -> int: + values = { + "use_persistent": "false", + "fallback_reason": reason, + "producer_run_id": "", + "artifact_id": "", + "queue_to_start_seconds": "", + "producer_allocated_seconds": "", + **extra, + } + write_outputs(output, values) + print(json.dumps(values, sort_keys=True)) + return 0 + + +def success( + output: Path, + *, + run_id: int, + artifact_id: int, + queue_seconds: float, + allocated_seconds: float, +) -> int: + values = { + "use_persistent": "true", + "fallback_reason": "", + "producer_run_id": run_id, + "artifact_id": artifact_id, + "queue_to_start_seconds": round(queue_seconds, 3), + "producer_allocated_seconds": round(allocated_seconds, 3), + } + write_outputs(output, values) + print(json.dumps(values, sort_keys=True)) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--event-name", required=True) + parser.add_argument("--selector", default="") + parser.add_argument("--cohort", default="") + parser.add_argument("--repository", required=True) + parser.add_argument("--pr-number", default="") + parser.add_argument("--head-repository", default="") + parser.add_argument("--head-ref", default="") + parser.add_argument("--author-association", default="") + parser.add_argument("--head-sha", default="") + parser.add_argument("--source-sha", required=True) + parser.add_argument("--source-tree", required=True) + parser.add_argument("--source-parent1", required=True) + parser.add_argument("--run-id", required=True) + parser.add_argument("--run-attempt", required=True) + parser.add_argument("--queue-seconds", type=int, default=90) + parser.add_argument("--execution-seconds", type=int, default=480) + parser.add_argument("--observe-seconds", type=int, default=60) + parser.add_argument("--github-output", type=Path, required=True) + parser.add_argument("--observe-only", action="store_true") + args = parser.parse_args() + + eligible, reason = eligibility(args) + if not eligible: + return fallback(args.github_output, reason) + if not valid_budget(args.queue_seconds, args.execution_seconds): + return fallback(args.github_output, "invalid_budget") + if args.observe_only and not valid_observe_budget(args.observe_seconds): + return fallback(args.github_output, "invalid_observe_budget") + + request_id = f"{args.run_id}-{args.run_attempt}" + api = GitHub(args.repository) + cancel_event = install_cancel_event() + try: + verified, live_reason = verify_live_request(api, args) + if not verified: + return fallback(args.github_output, live_reason) + + discovery_started = now() + observation_deadline = ( + discovery_started + args.observe_seconds if args.observe_only else None + ) + if args.observe_only: + try: + run = find_run(api, request_id, observation_deadline, cancel_event) + except RuntimeError as error: + if "did not become observable" in str(error): + return fallback(args.github_output, "observe_timeout") + raise + else: + api.dispatch( + { + "request_id": request_id, + "pr_number": args.pr_number, + "source_sha": args.source_sha, + "source_tree": args.source_tree, + "source_parent1": args.source_parent1, + "head_sha": args.head_sha, + } + ) + run = find_run(api, request_id, discovery_started + 30, cancel_event) + run_id = int(run["id"]) + queue_deadline = now() + args.queue_seconds + if observation_deadline is not None: + queue_deadline = min(queue_deadline, observation_deadline) + + def observe_queue() -> dict[str, object] | None: + selected = compile_job(api, run_id) + if selected and ( + selected.get("started_at") + or selected.get("status") in TERMINAL + ): + return selected + return None + + selected = observe_until( + queue_deadline, + 2.0, + observe_queue, + cancel_event=cancel_event, + ) + if isinstance(selected, dict) and selected.get("status") in TERMINAL and not selected.get("started_at"): + conclusion = str(selected.get("conclusion") or "unknown") + return fallback(args.github_output, f"producer_{conclusion}", producer_run_id=run_id) + if not isinstance(selected, dict) or not selected.get("started_at"): + if not args.observe_only: + cancel(api, run_id) + return fallback( + args.github_output, + "observe_timeout" if args.observe_only else "queue_timeout", + producer_run_id=run_id, + ) + + created = parse_time(str(selected.get("created_at") or "")) + started = parse_time(str(selected.get("started_at") or "")) + if created is None or started is None: + if not args.observe_only: + cancel(api, run_id) + return fallback(args.github_output, "producer_timing_unavailable", producer_run_id=run_id) + queue_seconds = max(0.0, (started - created).total_seconds()) + + execution_deadline = now() + args.execution_seconds + if observation_deadline is not None: + execution_deadline = min(execution_deadline, observation_deadline) + + def observe_completion() -> dict[str, object] | None: + current = compile_job(api, run_id) + if current and current.get("status") == "completed": + return current + return None + + completed = observe_until( + execution_deadline, + 3.0, + observe_completion, + cancel_event=cancel_event, + ) + if not isinstance(completed, dict): + if not args.observe_only: + cancel(api, run_id) + return fallback( + args.github_output, + "observe_timeout" if args.observe_only else "execution_budget_exceeded", + producer_run_id=run_id, + queue_to_start_seconds=round(queue_seconds, 3), + ) + if completed.get("conclusion") != "success": + return fallback( + args.github_output, + f"producer_{completed.get('conclusion') or 'failed'}", + producer_run_id=run_id, + queue_to_start_seconds=round(queue_seconds, 3), + ) + + completed_at = parse_time(str(completed.get("completed_at") or "")) + started_at = parse_time(str(completed.get("started_at") or "")) + allocated_seconds = ( + max(0.0, (completed_at - started_at).total_seconds()) + if completed_at is not None and started_at is not None + else 0.0 + ) + artifacts = api.api(f"actions/runs/{run_id}/artifacts?per_page=100") + candidates = artifacts.get("artifacts", []) if isinstance(artifacts, dict) else [] + name = f"persistent-mac-compile-{request_id}" + matches = [ + artifact for artifact in candidates + if isinstance(artifact, dict) + and artifact.get("name") == name + and artifact.get("expired") is False + ] + if len(matches) != 1: + return fallback( + args.github_output, + "producer_artifact_missing", + producer_run_id=run_id, + queue_to_start_seconds=round(queue_seconds, 3), + producer_allocated_seconds=round(allocated_seconds, 3), + ) + return success( + args.github_output, + run_id=run_id, + artifact_id=int(matches[0]["id"]), + queue_seconds=queue_seconds, + allocated_seconds=allocated_seconds, + ) + except (RuntimeError, KeyError, TypeError, ValueError, subprocess.SubprocessError) as error: + print(f"persistent Mac routing fell back to hosted: {error}", file=sys.stderr) + return fallback(args.github_output, "routing_error") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/run-persistent-mac-compile.py b/scripts/ci/run-persistent-mac-compile.py new file mode 100644 index 000000000000..aea82e207a49 --- /dev/null +++ b/scripts/ci/run-persistent-mac-compile.py @@ -0,0 +1,657 @@ +#!/usr/bin/env python3 +"""Run cmux compile admission through Glaeda's native Apple cache contract.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import stat +import subprocess +import sys +import time + + +PROFILE = "ci-compile-admission" +BASE_GENERATION = "cmux-ci-v1" +QUARANTINE_RETAINED_STORES = 1 +STATE_RESET_REASONS = ( + "state belongs to another checkout", + "existing Apple state is incomplete", + "cache generation identity mismatch", + "existing cache generation is unmarked", + "checkout identity changed before state access", +) + + +class Refusal(RuntimeError): + pass + + +def output(*argv: str, cwd: Path | None = None) -> str: + result = subprocess.run(argv, cwd=cwd, text=True, capture_output=True, check=False) + if result.returncode: + raise Refusal(f"command failed ({result.returncode}): {' '.join(argv)}\n{result.stderr.strip()}") + return result.stdout.strip() + + +def json_command(argv: list[str]) -> tuple[dict[str, object], int]: + result = subprocess.run(argv, text=True, capture_output=True, check=False) + payload: dict[str, object] | None = None + for raw in reversed(result.stdout.splitlines()): + raw = raw.strip() + if not raw: + continue + try: + candidate = json.loads(raw) + except json.JSONDecodeError: + continue + if isinstance(candidate, dict): + payload = candidate + break + if payload is None: + reason = result.stderr.strip() or result.stdout.strip() or f"exit {result.returncode}" + raise Refusal(reason) + if result.stderr.strip(): + print(result.stderr.rstrip(), file=sys.stderr) + return payload, result.returncode + + +def glaeda( + executable: Path, + action: str, + project: Path, + generation: str, + *, + expected_commit: str | None = None, + expected_tree: str | None = None, + require_clean: bool = False, + run_id: str | None = None, +) -> tuple[dict[str, object], int]: + argv = [ + str(executable), + action, + "--project", + str(project), + "--profile", + PROFILE, + "--generation", + generation, + ] + if run_id is not None: + argv.extend(["--run-id", run_id]) + if expected_commit is not None: + argv.extend(["--expected-commit", expected_commit]) + if expected_tree is not None: + argv.extend(["--expected-tree", expected_tree]) + if require_clean: + argv.append("--require-clean-source") + return json_command(argv) + + +def apple_state(project: Path) -> Path: + return project / ".glaeda" / "apple-build" + + +def plan(executable: Path, project: Path) -> dict[str, object]: + result, code = glaeda(executable, "plan", project, BASE_GENERATION) + if code: + raise Refusal(f"Glaeda plan exited {code}") + return result + + +def prune_quarantine_stores(project: Path, keep: Path | None = None) -> None: + parent = project / ".glaeda" + if not parent.is_dir(): + return + candidates: list[tuple[int, Path]] = [] + with os.scandir(parent) as entries: + for entry in entries: + if not entry.name.startswith("apple-build-quarantine-"): + continue + if not (entry.is_dir(follow_symlinks=False) or entry.is_symlink()): + continue + info = entry.stat(follow_symlinks=False) + candidates.append((info.st_mtime_ns, Path(entry.path))) + candidates.sort(reverse=True) + retained: set[Path] = set() + if keep is not None: + retained.add(keep) + for _, path in candidates: + if path in retained: + continue + if len(retained) < QUARANTINE_RETAINED_STORES: + retained.add(path) + continue + if path.parent != parent or not path.name.startswith("apple-build-quarantine-"): + raise Refusal("refusing to prune a path outside the cmux Glaeda quarantine") + if path.is_symlink(): + path.unlink() + else: + shutil.rmtree(path) + print(f"Pruned obsolete Glaeda quarantine {path.name}") + + +def quarantine_state(project: Path, request_id: str) -> Path | None: + state = apple_state(project) + if not os.path.lexists(state): + return None + suffix = re.sub(r"[^a-zA-Z0-9_.-]+", "-", request_id).strip("-") or "run" + destination = state.with_name( + f"apple-build-quarantine-{suffix}-{time.time_ns()}" + ) + state.rename(destination) + print(f"Quarantined incompatible Glaeda state as {destination.name}") + prune_quarantine_stores(project, keep=destination) + return destination + + +def reset_to_cold( + executable: Path, + project: Path, + request_id: str, + reset_reasons: list[str], + reason: str, +) -> dict[str, object]: + reset_reasons.append(reason) + quarantine_state(project, request_id) + return plan(executable, project) + + +def plan_or_reset( + executable: Path, + project: Path, + request_id: str, + source_reset: bool, +) -> tuple[dict[str, object], str, list[str]]: + reset_reasons: list[str] = [] + generation = BASE_GENERATION + state_existed = apple_state(project).exists() + try: + current = plan(executable, project) + except Refusal as error: + reason = str(error) + if "cache was interrupted" in reason: + current = reset_to_cold( + executable, project, request_id, reset_reasons, "quarantined_generation" + ) + elif any(fragment in reason for fragment in STATE_RESET_REASONS): + current = reset_to_cold( + executable, project, request_id, reset_reasons, "state_ownership_reset" + ) + else: + raise + + active = current.get("active_run") + if current.get("state") == "interrupted_or_running" and isinstance(active, str) and active: + recovered, code = glaeda( + executable, "recover", project, generation, run_id=active + ) + if code: + raise Refusal( + f"Glaeda recovery exited {code}: {json.dumps(recovered, sort_keys=True)}" + ) + current = reset_to_cold( + executable, + project, + request_id, + reset_reasons, + "interrupted_generation_recovered", + ) + + if source_reset: + reset_reasons.append("dirty_source_reset") + if apple_state(project).exists(): + quarantine_state(project, request_id) + current = plan(executable, project) + elif current.get("state") == "cold" and state_existed and not reset_reasons: + current = reset_to_cold( + executable, + project, + request_id, + reset_reasons, + "incompatible_cache_reset", + ) + + prune_quarantine_stores(project) + return current, generation, reset_reasons + + +def publish_native_log(project: Path, receipt: dict[str, object], label: str) -> None: + run_id = receipt.get("run_id") + if not isinstance(run_id, str) or not re.fullmatch(r"[0-9a-f]{32}", run_id): + return + path = project / ".glaeda" / "apple-build" / f"run-{run_id}.log" + if not path.is_file(): + return + print(f"===== Glaeda {label} native log =====") + with path.open("r", encoding="utf-8", errors="replace") as stream: + for line in stream: + print(line, end="") + print(f"===== end Glaeda {label} native log =====") + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def directory_sha256(root: Path) -> str: + """Canonical byte/type/mode digest for a cached native input tree.""" + digest = hashlib.sha256() + + def visit(directory: Path) -> None: + with os.scandir(directory) as stream: + entries = sorted(stream, key=lambda entry: entry.name.encode()) + for entry in entries: + path = Path(entry.path) + relative = path.relative_to(root).as_posix() + info = entry.stat(follow_symlinks=False) + digest.update(relative.encode() + b"\0") + digest.update(f"{info.st_mode & 0o777:o}".encode() + b"\0") + if stat.S_ISLNK(info.st_mode): + digest.update(b"L\0" + os.readlink(path).encode() + b"\0") + elif stat.S_ISDIR(info.st_mode): + digest.update(b"D\0") + visit(path) + elif stat.S_ISREG(info.st_mode): + digest.update(b"F\0") + with path.open("rb") as source: + for block in iter(lambda: source.read(1024 * 1024), b""): + digest.update(block) + digest.update(b"\0") + else: + raise Refusal(f"unsupported cached GhosttyKit entry: {relative}") + + visit(root) + return digest.hexdigest() + + +def ensure_ghostty(project: Path, cmux_state: Path) -> str: + expected = output("git", "-C", str(project / "ghostty"), "rev-parse", "HEAD") + marker = cmux_state / "ghosttykit-identity.json" + framework = project / "GhosttyKit.xcframework" + + identity = None + if marker.is_file(): + try: + identity = json.loads(marker.read_text()) + except (OSError, ValueError): + identity = None + if (framework.is_dir() and not framework.is_symlink() and isinstance(identity, dict) + and identity.get("schema_version") == 1 + and identity.get("ghostty_commit") == expected + and isinstance(identity.get("tree_sha256"), str) + and re.fullmatch(r"[a-f0-9]{64}", identity["tree_sha256"]) + and directory_sha256(framework) == identity["tree_sha256"]): + return "reused" + + if framework.exists() or framework.is_symlink(): + if framework.is_dir() and not framework.is_symlink(): + shutil.rmtree(framework) + else: + framework.unlink() + marker.unlink(missing_ok=True) + subprocess.run([str(project / "scripts" / "download-prebuilt-ghosttykit.sh")], cwd=project, check=True) + if not framework.is_dir() or framework.is_symlink(): + raise Refusal("GhosttyKit download completed without the expected framework") + verified_identity = { + "schema_version": 1, + "ghostty_commit": expected, + "tree_sha256": directory_sha256(framework), + } + temporary = marker.with_suffix(".tmp") + temporary.write_text(json.dumps(verified_identity, sort_keys=True) + "\n") + temporary.replace(marker) + return "downloaded" + + +def require_exact_source(receipt: dict[str, object], commit: str, tree: str, label: str) -> None: + if receipt.get("source_validation") != "exact_commit_tree_clean": + raise Refusal(f"{label} receipt did not record exact source validation") + for key in ("source_before", "source_after"): + observed = receipt.get(key) + if not isinstance(observed, dict): + raise Refusal(f"{label} receipt omitted {key}") + if observed.get("commit") != commit or observed.get("tree") != tree or observed.get("clean") is not True: + raise Refusal(f"{label} receipt source identity drifted") + + +def write_output(name: str, value: object) -> None: + path = os.environ.get("GITHUB_OUTPUT") + if not path: + return + with Path(path).open("a", encoding="utf-8") as stream: + stream.write(f"{name}={value}\n") + + +def semantic_stage_seconds(result: dict[str, object], stage: str) -> float: + timings = result.get("stage_timings") + if not isinstance(timings, list): + raise Refusal("canonical workload result omitted stage timings") + matched = [] + for item in timings: + if not isinstance(item, dict): + raise Refusal("canonical workload stage timing is invalid") + name = item.get("stage") + seconds = item.get("seconds") + if not isinstance(name, str) or type(seconds) not in (int, float) or seconds < 0: + raise Refusal("canonical workload stage timing is invalid") + if name == stage: + matched.append(float(seconds)) + if len(matched) != 1: + raise Refusal(f"canonical workload result must report one {stage} stage") + return matched[0] + + +def validate_semantic_result( + result: object, expected_commit: str, expected_tree: str +) -> dict[str, object]: + if not isinstance(result, dict): + raise Refusal("canonical workload result is not an object") + if result.get("document_type") != "cmux-workload-result" or result.get("schema_version") != 1: + raise Refusal("canonical workload result identity is invalid") + if result.get("result") != "passed": + raise Refusal("canonical compile-admission workload did not pass") + if result.get("source") != { + "repository": "manaflow-ai/cmux", + "commit": expected_commit, + "tree": expected_tree, + }: + raise Refusal("canonical workload source identity mismatch") + if result.get("profile") != { + "id": "cmux.macos.compile-admission", + "generation": 1, + }: + raise Refusal("canonical workload profile identity mismatch") + if result.get("semantic_validator") != "cmux.compile-admission/v1": + raise Refusal("canonical workload semantic validator mismatch") + + validation = result.get("validation") + if ( + not isinstance(validation, dict) + or validation.get("missing_required_artifact_classes") != [] + ): + raise Refusal("canonical workload artifact validation is incomplete") + cleanup = result.get("cleanup") + if ( + not isinstance(cleanup, dict) + or cleanup.get("state") != "complete" + or cleanup.get("process_group_settled") is not True + ): + raise Refusal("canonical workload cleanup is incomplete") + + benchmark = result.get("benchmark") + if not isinstance(benchmark, dict): + raise Refusal("canonical workload benchmark identity is missing") + state_class = benchmark.get("state_class") + if state_class not in {"cold", "compiler-warm"}: + raise Refusal("canonical workload used an unexpected benchmark state class") + for key in ("semantic_comparison_key", "comparison_context_key"): + value = benchmark.get(key) + if not isinstance(value, str) or re.fullmatch(r"sha256:[a-f0-9]{64}", value) is None: + raise Refusal(f"canonical workload {key} is invalid") + + toolchain = result.get("toolchain") + if not isinstance(toolchain, dict): + raise Refusal("canonical workload toolchain identity is missing") + identity = toolchain.get("identity") + observations = toolchain.get("observations") + if ( + not isinstance(identity, str) + or re.fullmatch(r"sha256:[a-f0-9]{64}", identity) is None + or not isinstance(observations, dict) + ): + raise Refusal("canonical workload toolchain identity is invalid") + + semantic_stage_seconds(result, "setup") + semantic_stage_seconds(result, "dependency_preparation") + semantic_stage_seconds(result, "compile") + semantic_stage_seconds(result, "validation") + return result + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--glaeda", type=Path, required=True) + parser.add_argument("--expected-commit", required=True) + parser.add_argument("--expected-tree", required=True) + parser.add_argument("--request-id", required=True) + parser.add_argument("--source-preparation-seconds", type=float, required=True) + parser.add_argument("--source-reset", choices=("true", "false"), required=True) + parser.add_argument("--metrics", type=Path, required=True) + args = parser.parse_args() + + project = Path.cwd().resolve() + semantic_runner = project / "scripts/ci/cmux_workload_profile.py" + semantic_registry = project / "scripts/ci/cmux-workload-profiles.json" + semantic_entrypoint = project / "scripts/ci/persistent-mac-semantic-entrypoint.sh" + if not semantic_runner.is_file() or not semantic_registry.is_file(): + raise Refusal( + "canonical cmux.macos.compile-admission@1 is unavailable; " + "persistent routing remains disabled until #13411 is on the exact source" + ) + if not semantic_entrypoint.is_file() or not os.access(semantic_entrypoint, os.X_OK): + raise Refusal("persistent semantic entrypoint is unavailable or not executable") + + lockfile = project / "cmux.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved" + if not lockfile.is_file(): + raise Refusal(f"missing Package.resolved: {lockfile.relative_to(project)}") + initial_package_identity = sha256(lockfile) + initial_submodules = output( + "git", "-C", str(project), "submodule", "status", "--recursive" + ) + submodule_identity = hashlib.sha256(initial_submodules.encode()).hexdigest() + + plan_receipt, generation, reset_reasons = plan_or_reset( + args.glaeda.resolve(), project, args.request_id, args.source_reset == "true" + ) + initial_state = str(plan_receipt.get("state")) + if initial_state not in {"cold", "prepared"}: + raise Refusal(f"unexpected Glaeda cache state after admission: {initial_state}") + + cache_key = plan_receipt.get("cache_key") + invocation_identity = plan_receipt.get("invocation_identity") + cache_root = plan_receipt.get("cache_root") + if ( + not isinstance(cache_key, str) + or re.fullmatch(r"[a-f0-9]{64}", cache_key) is None + or not isinstance(invocation_identity, str) + or re.fullmatch(r"[a-f0-9]{64}", invocation_identity) is None + ): + raise Refusal("Glaeda plan omitted valid cache lineage identities") + expected_cache_root = f".glaeda/apple-build/cache/{cache_key}" + if cache_root != expected_cache_root: + raise Refusal("Glaeda plan returned an unexpected cache locator") + + cmux_state = project / ".glaeda" / "cmux-ci" + cmux_state.mkdir(parents=True, exist_ok=True, mode=0o700) + os.chmod(cmux_state, 0o700) + build_marker = cmux_state / "last-successful-build.json" + prior_build = None + if build_marker.is_file(): + try: + prior_build = json.loads(build_marker.read_text()) + except (OSError, ValueError): + prior_build = None + invocation_cache_state = ( + "matched" + if prior_build == { + "schema_version": 1, + "cache_key": cache_key, + "invocation_identity": invocation_identity, + } + else "changed-or-unseeded" + ) + + native_started = time.monotonic() + native, native_code = glaeda( + args.glaeda.resolve(), + "run", + project, + generation, + expected_commit=args.expected_commit, + expected_tree=args.expected_tree, + require_clean=True, + ) + outer_native_seconds = time.monotonic() - native_started + publish_native_log(project, native, "canonical compile admission") + require_exact_source(native, args.expected_commit, args.expected_tree, "build") + if native_code or native.get("exit_code") != 0: + quarantine_state(project, args.request_id + "-semantic-failed") + raise Refusal( + "canonical compile admission failed with exit " + f"{native.get('exit_code', native_code)}" + ) + + cache_directory = project / expected_cache_root + if cache_directory.is_symlink() or not cache_directory.is_dir(): + raise Refusal("Glaeda native cache omitted a safe cache directory") + canonical_cache_parent = (project / ".glaeda/apple-build/cache").resolve(strict=True) + resolved_cache = cache_directory.resolve(strict=True) + if resolved_cache.parent != canonical_cache_parent: + raise Refusal("Glaeda cache locator escaped the project cache root") + + products_root = cache_directory / "products" + semantic_state = products_root / "semantic-state" + semantic_result_path = products_root / "semantic-result.json" + derived_data = semantic_state / "derived-data" + for label, path in ( + ("products", products_root), + ("semantic state", semantic_state), + ("DerivedData", derived_data), + ): + if path.is_symlink() or not path.is_dir(): + raise Refusal(f"Glaeda native cache omitted a safe {label} directory") + resolved = path.resolve(strict=True) + if resolved_cache not in resolved.parents: + raise Refusal(f"{label} escaped the admitted Glaeda cache generation") + if semantic_result_path.is_symlink() or not semantic_result_path.is_file(): + raise Refusal("canonical workload result is missing or unsafe") + + try: + raw_semantic = json.loads(semantic_result_path.read_text()) + except (OSError, ValueError) as error: + raise Refusal("canonical workload result cannot be read") from error + semantic = validate_semantic_result( + raw_semantic, args.expected_commit, args.expected_tree + ) + + final_package_identity = sha256(lockfile) + if final_package_identity != initial_package_identity: + quarantine_state(project, args.request_id + "-package-drift") + raise Refusal("Package.resolved changed during canonical compile admission") + final_submodules = output( + "git", "-C", str(project), "submodule", "status", "--recursive" + ) + if final_submodules != initial_submodules: + quarantine_state(project, args.request_id + "-submodule-drift") + raise Refusal("submodule identity changed during canonical compile admission") + + build_log = derived_data / "cmux-build.log" + debug_products = derived_data / "Build" / "Products" / "Debug" + if not build_log.is_file() or not debug_products.is_dir(): + raise Refusal("canonical compile completed without admission log/products") + semantic_copy = derived_data / "cmux-workload-result.json" + shutil.copy2(semantic_result_path, semantic_copy) + + semantic_state_class = semantic["benchmark"]["state_class"] + classification = ( + "cold-reset" + if reset_reasons or initial_state == "cold" or semantic_state_class == "cold" + else "hot" + if semantic_state_class == "compiler-warm" and invocation_cache_state == "matched" + else "partially-warm" + ) + build_marker_tmp = build_marker.with_suffix(".tmp") + build_marker_tmp.write_text( + json.dumps( + { + "schema_version": 1, + "cache_key": cache_key, + "invocation_identity": invocation_identity, + }, + sort_keys=True, + ) + + "\n" + ) + build_marker_tmp.replace(build_marker) + + toolchain = { + "xcode": output("xcodebuild", "-version"), + "sdk_version": output("xcrun", "--sdk", "macosx", "--show-sdk-version"), + "sdk_build": output("xcrun", "--sdk", "macosx", "--show-sdk-build-version"), + "developer_dir": os.environ.get("DEVELOPER_DIR", ""), + } + semantic_toolchain = semantic.get("toolchain", {}) + observations = semantic_toolchain.get("observations", {}) + if isinstance(observations, dict): + if observations.get("xcode") not in (None, toolchain["xcode"]): + raise Refusal("canonical workload Xcode observation differs after execution") + if observations.get("macos_sdk") not in (None, toolchain["sdk_version"]): + raise Refusal("canonical workload SDK observation differs after execution") + + native_timings = native.get("timings_seconds") + native_work = native.get("native_work") + package_seconds = semantic_stage_seconds(semantic, "dependency_preparation") + compile_seconds = semantic_stage_seconds(semantic, "compile") + semantic_validation_seconds = semantic_stage_seconds(semantic, "validation") + setup_seconds = semantic_stage_seconds(semantic, "setup") + + metrics = { + "schema_version": 1, + "classification": classification, + "source": {"commit": args.expected_commit, "tree": args.expected_tree}, + "source_preparation_seconds": round(args.source_preparation_seconds, 6), + "setup_seconds": setup_seconds, + "package_readiness_seconds": package_seconds, + "compile_duration_seconds": compile_seconds, + "semantic_validation_seconds": semantic_validation_seconds, + "outer_native_seconds": round(outer_native_seconds, 6), + "package_resolved_sha256": final_package_identity, + "submodule_identity_sha256": submodule_identity, + "invocation_cache_state": invocation_cache_state, + "semantic": semantic, + "glaeda": { + "generation": generation, + "initial_state": initial_state, + "reset_reasons": reset_reasons, + "cache_key": cache_key, + "invocation_identity": invocation_identity, + "native_timings_seconds": ( + native_timings if isinstance(native_timings, dict) else {} + ), + "native_work": native_work if isinstance(native_work, dict) else {}, + }, + "toolchain": toolchain, + } + args.metrics.parent.mkdir(parents=True, exist_ok=True) + args.metrics.write_text(json.dumps(metrics, sort_keys=True, indent=2) + "\n") + + write_output("derived_data", derived_data) + write_output("classification", classification) + write_output("metrics", args.metrics) + print(json.dumps(metrics, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except ( + Refusal, + OSError, + subprocess.SubprocessError, + ValueError, + KeyError, + TypeError, + ) as error: + print(f"persistent Mac compile refused: {error}", file=sys.stderr) + raise SystemExit(1) diff --git a/tests/test_ci_persistent_mac_compile.py b/tests/test_ci_persistent_mac_compile.py new file mode 100644 index 000000000000..d6730b442a5f --- /dev/null +++ b/tests/test_ci_persistent_mac_compile.py @@ -0,0 +1,426 @@ +#!/usr/bin/env python3 +"""Regression coverage for the persistent-Mac compile-admission pilot.""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +ROUTE = ROOT / "scripts/ci/persistent_mac_route.py" +CI = ROOT / ".github/workflows/ci.yml" +PRODUCER = ROOT / ".github/workflows/persistent-macos-compile.yml" +ROUTER = ROOT / ".github/workflows/persistent-macos-router.yml" +PROFILE = ROOT / "glaeda.apple.json" +DRIVER = ROOT / "scripts/ci/run-persistent-mac-compile.py" +SEMANTIC_ENTRYPOINT = ROOT / "scripts/ci/persistent-mac-semantic-entrypoint.sh" + + +spec = importlib.util.spec_from_file_location("persistent_mac_route", ROUTE) +route = importlib.util.module_from_spec(spec) +assert spec.loader is not None +spec.loader.exec_module(route) + + +driver_spec = importlib.util.spec_from_file_location("persistent_mac_driver", DRIVER) +driver = importlib.util.module_from_spec(driver_spec) +assert driver_spec.loader is not None +driver_spec.loader.exec_module(driver) + + +def args(**overrides): + values = { + "selector": "pilot", + "event_name": "pull_request", + "head_repository": "manaflow-ai/cmux", + "repository": "manaflow-ai/cmux", + "author_association": "MEMBER", + "cohort": "13198,feature/persistent", + "pr_number": "13198", + "head_ref": "feature/persistent", + } + values.update(overrides) + return argparse.Namespace(**values) + + +class RoutingTests(unittest.TestCase): + def test_bounded_observation_handles_ready_timeout_and_cancel(self): + class FakeEvent: + def __init__(self, clock, *, cancel_on_wait=False): + self.clock = clock + self.cancel_on_wait = cancel_on_wait + + def wait(self, seconds): + self.clock[0] += seconds + return self.cancel_on_wait + + clock = [0.0] + attempts = [] + ready = route.observe_until( + 10.0, + 2.0, + lambda: "ready" if len(attempts) >= 2 else attempts.append("try"), + cancel_event=FakeEvent(clock), + monotonic=lambda: clock[0], + ) + self.assertEqual(ready, "ready") + self.assertEqual(len(attempts), 2) + + clock = [0.0] + timed_out = route.observe_until( + 5.0, + 2.0, + lambda: None, + cancel_event=FakeEvent(clock), + monotonic=lambda: clock[0], + ) + self.assertIsNone(timed_out) + self.assertEqual(clock[0], 5.0) + + clock = [0.0] + with self.assertRaisesRegex(route.RoutingCanceled, "canceled"): + route.observe_until( + 5.0, + 1.0, + lambda: None, + cancel_event=FakeEvent(clock, cancel_on_wait=True), + monotonic=lambda: clock[0], + ) + + def test_only_trusted_same_repository_members_are_eligible(self): + self.assertEqual(route.eligibility(args()), (True, "pilot")) + self.assertEqual( + route.eligibility(args(head_repository="someone/cmux")), + (False, "untrusted_repository"), + ) + self.assertEqual( + route.eligibility(args(author_association="CONTRIBUTOR")), + (False, "untrusted_author"), + ) + self.assertEqual( + route.eligibility(args(event_name="merge_group")), + (False, "event_not_pull_request"), + ) + + def test_selector_and_cohort_are_reversible(self): + for selector in ("", "0", "off", "false"): + self.assertEqual(route.eligibility(args(selector=selector)), (False, "selector_off")) + self.assertEqual( + route.eligibility(args(pr_number="99", head_ref="other")), + (False, "outside_pilot_cohort"), + ) + self.assertEqual(route.eligibility(args(selector="all", cohort="")), (True, "all")) + self.assertEqual( + route.eligibility(args(selector="unexpected")), + (False, "invalid_selector"), + ) + + def test_live_verification_binds_the_originating_ci_run(self): + class API: + def __init__(self, run_overrides=None): + self.run_overrides = run_overrides or {} + + def api(self, path): + if path.startswith("actions/runs/"): + value = { + "id": 123, + "event": "pull_request", + "run_attempt": 2, + "path": ".github/workflows/ci.yml", + "head_sha": "a" * 40, + "head_repository": {"full_name": "manaflow-ai/cmux"}, + "pull_requests": [{ + "number": 13198, + "head": {"sha": "b" * 40}, + "base": {"sha": "c" * 40}, + }], + } + value.update(self.run_overrides) + return value + if path == "pulls/13198": + return { + "state": "open", + "author_association": "MEMBER", + "head": {"sha": "b" * 40, "repo": {"full_name": "manaflow-ai/cmux"}}, + "base": {"sha": "c" * 40}, + "merge_commit_sha": "a" * 40, + } + if path == "git/commits/" + "a" * 40: + return {"tree": {"sha": "d" * 40}} + raise AssertionError(path) + + live = args( + run_id="123", + run_attempt="2", + source_sha="a" * 40, + source_tree="d" * 40, + source_parent1="c" * 40, + head_sha="b" * 40, + ) + self.assertEqual(route.verify_live_request(API(), live), (True, "verified")) + self.assertEqual( + route.verify_live_request(API({"event": "push"}), live), + (False, "source_run_event_mismatch"), + ) + self.assertEqual( + route.verify_live_request(API({"head_sha": "e" * 40}), live), + (False, "source_run_source_mismatch"), + ) + self.assertEqual( + route.verify_live_request(API({"pull_requests": [{"number": 99}]}), live), + (False, "source_run_pr_mismatch"), + ) + + def test_route_budget_always_fits_controller_window(self): + self.assertTrue(route.valid_budget(90, 480)) + self.assertTrue(route.valid_budget(120, 480)) + self.assertFalse(route.valid_budget(121, 480)) + self.assertFalse(route.valid_budget(120, 481)) + self.assertFalse(route.valid_budget(120, 500)) + + def test_observe_only_budget_is_short_and_bounded(self): + self.assertTrue(route.valid_observe_budget(5)) + self.assertTrue(route.valid_observe_budget(60)) + self.assertFalse(route.valid_observe_budget(4)) + self.assertFalse(route.valid_observe_budget(61)) + + def test_output_helpers_record_hosted_fallback_and_persistent_success(self): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "output" + output.touch() + self.assertEqual(route.fallback(output, "queue_timeout", producer_run_id=42), 0) + values = dict(line.split("=", 1) for line in output.read_text().splitlines()) + self.assertEqual(values["use_persistent"], "false") + self.assertEqual(values["fallback_reason"], "queue_timeout") + self.assertEqual(values["producer_run_id"], "42") + + output.write_text("") + self.assertEqual( + route.success( + output, + run_id=43, + artifact_id=99, + queue_seconds=1.25, + allocated_seconds=31.5, + ), + 0, + ) + values = dict(line.split("=", 1) for line in output.read_text().splitlines()) + self.assertEqual(values["use_persistent"], "true") + self.assertEqual(values["producer_run_id"], "43") + self.assertEqual(values["artifact_id"], "99") + self.assertEqual(values["queue_to_start_seconds"], "1.25") + self.assertEqual(values["producer_allocated_seconds"], "31.5") + + +class StateRetentionTests(unittest.TestCase): + def test_quarantine_pruning_keeps_only_newest_owned_store(self): + with tempfile.TemporaryDirectory() as directory: + project = Path(directory) + glaeda = project / ".glaeda" + glaeda.mkdir() + old = [] + for index in range(3): + path = glaeda / f"apple-build-quarantine-run-{index}" + path.mkdir() + (path / "marker").write_text(str(index)) + os_time = 1_000_000_000 + index + path.touch() + import os + os.utime(path, ns=(os_time, os_time)) + old.append(path) + unrelated = glaeda / "unrelated" + unrelated.mkdir() + + driver.prune_quarantine_stores(project) + + remaining = sorted(glaeda.glob("apple-build-quarantine-*")) + self.assertEqual(len(remaining), driver.QUARANTINE_RETAINED_STORES) + self.assertEqual(remaining[0].name, old[-1].name) + self.assertTrue(unrelated.is_dir()) + + +class WorkflowContractTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.ci = CI.read_text() + cls.producer = PRODUCER.read_text() + cls.router = ROUTER.read_text() + cls.driver = DRIVER.read_text() + cls.semantic_entrypoint = SEMANTIC_ENTRYPOINT.read_text() + cls.profile = json.loads(PROFILE.read_text()) + + def test_producer_is_manual_dedicated_and_credential_minimized(self): + self.assertIn(" workflow_dispatch:", self.producer) + for trigger in ("pull_request:", "pull_request_target:", "push:", "schedule:", "merge_group:"): + self.assertNotIn(f" {trigger}", self.producer) + self.assertEqual( + self.producer.count(" group: cmux-persistent-compile"), + 1, + ) + self.assertEqual(self.producer.split("on:", 1)[1].split("permissions:", 1)[0].count(" workflow_dispatch:"), 1) + self.assertIn("\npermissions: {}\n", self.producer) + self.assertIn(" labels: [self-hosted, macOS, ARM64, cmux-persistent-macos-compile]", self.producer) + self.assertIn(" compile:", self.producer) + compile_block = self.producer.split(" compile:", 1)[1] + self.assertIn(" permissions: {}", compile_block) + self.assertNotIn("secrets.", self.producer) + self.assertNotIn("actions/checkout@", self.producer) + self.assertRegex(self.producer, r"(?m)^ GLAEDA_REF: [a-f0-9]{40}$") + self.assertIn("cmux-workload-result.json", self.producer) + self.assertIn("semantic profile:", self.producer) + + def test_dispatch_authority_is_default_branch_only(self): + self.assertIn(" workflow_run:", self.router) + self.assertIn(" workflows: [CI]", self.router) + self.assertIn(" types: [in_progress]", self.router) + self.assertIn("\npermissions: {}\n", self.router) + self.assertIn(" actions: write", self.router) + self.assertIn(" ref: main", self.router) + self.assertIn("persistent-mac-route-request-", self.router) + route_text = ROUTE.read_text() + self.assertIn('actions/runs/{args.run_id}', route_text) + self.assertIn('"source_run_source_mismatch"', route_text) + self.assertIn('"source_run_pr_mismatch"', route_text) + self.assertNotIn("actions: write", self.ci) + admission = self.ci.split(" macos-compile-admission:", 1)[1].split( + " app-host-unit-tests:", 1 + )[0] + self.assertNotIn(" persistent-mac-compile-route:", self.ci) + self.assertIn(" actions: read", admission) + self.assertIn(" pull-requests: read", admission) + self.assertIn("--observe-only", admission) + self.assertIn("--observe-seconds 5", admission) + resolve_index = admission.index("- name: Resolve Swift packages") + route_index = admission.index("- name: Observe persistent Mac compile candidate") + compile_index = admission.index("- name: Compile app-host test product") + self.assertLess(resolve_index, route_index) + self.assertLess(route_index, compile_index) + + def test_ci_routes_only_trusted_prs_and_preserves_hosted_fallback(self): + admission = self.ci.split(" macos-compile-admission:", 1)[1].split( + " app-host-unit-tests:", 1 + )[0] + self.assertIn("vars.CI_PERSISTENT_MAC_COMPILE", admission) + self.assertIn("persistent-mac-route-request-", self.ci) + self.assertIn("steps.source-identity.outputs.valid == 'true'", self.ci) + self.assertIn("source_identity_valid: ${{ steps.source-identity.outputs.valid }}", self.ci) + self.assertIn("needs.changes.outputs.source_identity_valid == 'true'", admission) + self.assertIn("github.event.pull_request.head.repo.full_name == github.repository", admission) + self.assertIn("github.event.pull_request.author_association == 'MEMBER'", admission) + self.assertIn("github.event.pull_request.author_association == 'OWNER'", admission) + self.assertNotIn("- persistent-mac-compile-route", admission) + self.assertIn("steps.persistent-restore.outputs.hit != 'true'", admission) + self.assertIn("actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131", admission) + self.assertIn("run-id: ${{ steps.persistent-route.outputs.producer_run_id }}", admission) + + def test_persistent_product_revalidation_retains_admission_checks(self): + admission = self.ci.split(" macos-compile-admission:", 1)[1].split( + " app-host-unit-tests:", 1 + )[0] + self.assertIn("persistent producer source identity mismatch", admission) + self.assertIn("persistent semantic profile identity mismatch", admission) + self.assertIn("persistent semantic validator mismatch", admission) + self.assertIn("persistent semantic cleanup incomplete", admission) + self.assertIn("persistent semantic Xcode identity mismatch", admission) + self.assertIn("Package.resolved identity mismatch", admission) + self.assertIn("submodule identity mismatch", admission) + self.assertIn("Xcode identity mismatch", admission) + self.assertIn("macOS SDK build mismatch", admission) + self.assertIn("python3 scripts/swift_warning_budget.py", admission) + self.assertIn("python3 tests/test_cli_version_memory_guard.py", admission) + self.assertIn("python3 tests/test_cli_contract_help.py", admission) + self.assertIn("python3 tests/test_cli_config_doctor.py", admission) + self.assertIn("macos-compile-admission-metrics-", admission) + self.assertIn('"classification": classification', admission) + + def test_glaeda_owns_native_state_while_cmux_owns_compile_semantics(self): + profile = self.profile["profiles"]["ci-compile-admission"] + self.assertEqual(profile["engine"], "script") + self.assertEqual( + self.profile["cache_policies"]["ci-compile-admission"], + "native", + ) + self.assertNotIn("preparations", self.profile) + self.assertEqual( + profile["executable"], + "scripts/ci/persistent-mac-semantic-entrypoint.sh", + ) + self.assertEqual( + profile["arguments"], + ["{products}/semantic-state", "{products}/semantic-result.json"], + ) + self.assertIn( + "{module_cache}", + profile["environment"]["CMUX_CI_MODULE_CACHE_PATH"], + ) + self.assertIn("cmux.macos.compile-admission", self.semantic_entrypoint) + self.assertIn("--generation 1", self.semantic_entrypoint) + self.assertIn("state_class=cold", self.semantic_entrypoint) + self.assertIn("state_class=compiler-warm", self.semantic_entrypoint) + self.assertIn("cmux_workload_profile.py", self.semantic_entrypoint) + self.assertIn("#13411", self.semantic_entrypoint) + self.assertTrue(SEMANTIC_ENTRYPOINT.stat().st_mode & 0o111) + + self.assertIn("--expected-commit", self.driver) + self.assertIn("--expected-tree", self.driver) + self.assertIn("require_clean=True", self.driver) + self.assertIn('"cmux-workload-result"', self.driver) + self.assertIn('"cmux.macos.compile-admission"', self.driver) + self.assertIn('"cmux.compile-admission/v1"', self.driver) + self.assertIn("Package.resolved changed during canonical compile admission", self.driver) + self.assertIn("quarantine_state(project, args.request_id + \"-semantic-failed\")", self.driver) + self.assertIn('"cold-reset"', self.driver) + self.assertIn('"partially-warm"', self.driver) + self.assertIn('"hot"', self.driver) + + def test_canonical_semantic_receipt_validation_is_fail_closed(self): + commit = "a" * 40 + tree = "b" * 40 + valid = { + "document_type": "cmux-workload-result", + "schema_version": 1, + "source": { + "repository": "manaflow-ai/cmux", + "commit": commit, + "tree": tree, + }, + "profile": {"id": "cmux.macos.compile-admission", "generation": 1}, + "semantic_validator": "cmux.compile-admission/v1", + "result": "passed", + "validation": {"missing_required_artifact_classes": []}, + "cleanup": {"state": "complete", "process_group_settled": True}, + "benchmark": { + "state_class": "compiler-warm", + "semantic_comparison_key": "sha256:" + "c" * 64, + "comparison_context_key": "sha256:" + "d" * 64, + }, + "toolchain": { + "identity": "sha256:" + "e" * 64, + "observations": {}, + }, + "stage_timings": [ + {"stage": "setup", "seconds": 1.0}, + {"stage": "dependency_preparation", "seconds": 2.0}, + {"stage": "compile", "seconds": 3.0}, + {"stage": "validation", "seconds": 4.0}, + ], + } + self.assertIs(driver.validate_semantic_result(valid, commit, tree), valid) + bad = json.loads(json.dumps(valid)) + bad["cleanup"]["state"] = "forced" + with self.assertRaisesRegex(driver.Refusal, "cleanup"): + driver.validate_semantic_result(bad, commit, tree) + bad = json.loads(json.dumps(valid)) + bad["profile"]["generation"] = 2 + with self.assertRaisesRegex(driver.Refusal, "profile"): + driver.validate_semantic_result(bad, commit, tree) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_ci_self_hosted_guard.sh b/tests/test_ci_self_hosted_guard.sh index d51086dd3526..8b480ef4e2d8 100755 --- a/tests/test_ci_self_hosted_guard.sh +++ b/tests/test_ci_self_hosted_guard.sh @@ -10,6 +10,8 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" CI_FILE="$ROOT_DIR/.github/workflows/ci.yml" +PERSISTENT_COMPILE_FILE="$ROOT_DIR/.github/workflows/persistent-macos-compile.yml" +PERSISTENT_ROUTER_FILE="$ROOT_DIR/.github/workflows/persistent-macos-router.yml" GHOSTTYKIT_FILE="$ROOT_DIR/.github/workflows/build-ghosttykit.yml" COMPAT_FILE="$ROOT_DIR/.github/workflows/ci-macos-compat.yml" E2E_FILE="$ROOT_DIR/.github/workflows/test-e2e.yml" @@ -1125,7 +1127,9 @@ check_no_self_hosted_fleet_runners() { for probe in 'runs-on: macfleet' '- tart-canary' '- tart-dual' '- tart-small' '- tart-macos-26' '- tart-ios' '- mac4' '- mac-mini' '- slot-3' '- xcode-26-3' '- cmux' \ "runs-on: \${{ vars.X || 'macos-26' }}" '- warp-macos-26-arm64-6x' \ '- cmux-aws-macos-15' '- cmux-macos-26' '- self-hosted' '- macOS' '- ARM64' \ - 'runs-on: [self-hosted, macOS, ARM64]'; do + 'runs-on: [self-hosted, macOS, ARM64]' \ + ' labels: [self-hosted, macOS, ARM64, cmux-persistent-macos-compile]' \ + ' group: cmux-persistent-compile'; do if ! printf '%s\n' "$probe" | grep -Eq "($forbidden)"; then echo "FAIL: fleet-runner guard self-test missed a known fleet/self-hosted label: $probe" exit 1 @@ -1188,6 +1192,11 @@ check_no_self_hosted_fleet_runners() { while IFS= read -r line; do content="${line#*:*:}" content_without_allowed="$(printf '%s\n' "$content" | sed -E "s/($allowed)//g")" + if [[ "$line" == "$PERSISTENT_COMPILE_FILE:"* ]] && \ + { [[ "$content" == ' group: cmux-persistent-compile' ]] || \ + [[ "$content" == ' labels: [self-hosted, macOS, ARM64, cmux-persistent-macos-compile]' ]]; }; then + continue + fi printf '%s\n' "$content_without_allowed" | grep -Eq "($forbidden)" || continue if [[ -n "$e2e_tart_option_line" ]] && [[ "$line" == "$E2E_FILE:$e2e_tart_option_line:"* ]]; then continue @@ -1202,7 +1211,7 @@ check_no_self_hosted_fleet_runners() { continue fi hits+="$line"$'\n' - done < <(grep -rnE "(runs-on:|[[:space:]]os:[[:space:]]|^[[:space:]]*-[[:space:]]+[A-Za-z0-9._-]+[[:space:]]*$)" "$ROOT_DIR/.github/workflows") + done < <(grep -rnE "(runs-on:|^[[:space:]]+(labels|group):|[[:space:]]os:[[:space:]]|^[[:space:]]*-[[:space:]]+[A-Za-z0-9._-]+[[:space:]]*$)" "$ROOT_DIR/.github/workflows") if [[ -n "$hits" ]]; then echo "FAIL: workflow references a self-hosted mac fleet label or bare self-hosted runner in a runner-selection position." echo " Use a cloud label so required jobs never land on a mini that can't foreground a GUI app:" @@ -1210,7 +1219,144 @@ check_no_self_hosted_fleet_runners() { echo "$hits" exit 1 fi - echo "PASS: no workflow can route a required job to a self-hosted mac fleet runner (cloud only)" + echo "PASS: required jobs stay on cloud runners; only the isolated persistent compile producer may target the owned Mac" +} + +check_persistent_compile_lane() { + if [ ! -f "$PERSISTENT_COMPILE_FILE" ]; then + echo "FAIL: persistent macOS compile workflow is missing" + exit 1 + fi + local triggers + triggers="$(awk ' + /^on:$/ { in_on=1; next } + in_on && /^[^[:space:]#]/ { in_on=0 } + in_on && /^ [A-Za-z0-9_-]+:/ { + key=$1 + sub(/:$/, "", key) + print key + } + ' "$PERSISTENT_COMPILE_FILE")" + if [ "$triggers" != "workflow_dispatch" ]; then + echo "FAIL: persistent macOS compile workflow must have workflow_dispatch as its only trigger" + printf 'triggers=%s\n' "$triggers" + exit 1 + fi + if ! grep -Fqx 'permissions: {}' "$PERSISTENT_COMPILE_FILE"; then + echo "FAIL: persistent macOS compile workflow must default to empty GitHub token permissions" + exit 1 + fi + if [ "$(grep -Fxc ' group: cmux-persistent-compile' "$PERSISTENT_COMPILE_FILE")" -ne 1 ] || \ + [ "$(grep -Fxc ' labels: [self-hosted, macOS, ARM64, cmux-persistent-macos-compile]' "$PERSISTENT_COMPILE_FILE")" -ne 1 ]; then + echo "FAIL: persistent compile producer must use the dedicated workflow-restricted runner group and label" + exit 1 + fi + if grep -Eq 'secrets\.|secrets\[' "$PERSISTENT_COMPILE_FILE"; then + echo "FAIL: persistent compile producer must not reference repository secrets" + exit 1 + fi + if grep -Fq 'actions/checkout@' "$PERSISTENT_COMPILE_FILE"; then + echo "FAIL: persistent compile producer must fetch public source explicitly instead of receiving checkout credentials" + exit 1 + fi + if ! awk ' + /^ compile:$/ { in_job=1; next } + in_job && /^ [A-Za-z0-9_-]+:$/ { in_job=0 } + in_job && /^ permissions: \{\}$/ { permissions=1 } + in_job && /^ group: cmux-persistent-compile$/ { group=1 } + in_job && /^ labels: \[self-hosted, macOS, ARM64, cmux-persistent-macos-compile\]$/ { runner=1 } + END { exit !(permissions && group && runner) } + ' "$PERSISTENT_COMPILE_FILE"; then + echo "FAIL: owned-Mac compile job must have empty GitHub token permissions and the dedicated runner group/label" + exit 1 + fi + if ! grep -Eq '^ GLAEDA_REF: [a-f0-9]{40}$' "$PERSISTENT_COMPILE_FILE"; then + echo "FAIL: persistent compile producer must pin Glaeda to an exact commit" + exit 1 + fi + if ! grep -Fq 'CI_PERSISTENT_MAC_COMPILE' "$CI_FILE" || \ + ! grep -Fq 'AUTHOR_ASSOCIATION:' "$CI_FILE" || \ + ! grep -Fq 'HEAD_REPOSITORY:' "$CI_FILE"; then + echo "FAIL: CI must retain the reversible selector and trust/repository routing inputs" + exit 1 + fi + if grep -Fq "needs.persistent-mac-compile-route.result == 'success'" "$CI_FILE"; then + echo "FAIL: macOS compile admission must run hosted fallback when the persistent route job itself fails" + exit 1 + fi + echo "PASS: persistent compile producer is dispatch-only, credential-minimized, pinned, and cohort-gated" +} + +check_persistent_compile_router() { + if [ ! -f "$PERSISTENT_ROUTER_FILE" ]; then + echo "FAIL: default-branch persistent Mac router workflow is missing" + exit 1 + fi + + local trigger_block expected_trigger + trigger_block="$(awk ' + /^on:$/ { in_on=1; next } + in_on && /^[^[:space:]]/ { exit } + in_on && NF { print } + ' "$PERSISTENT_ROUTER_FILE")" + expected_trigger=$' workflow_run:\n workflows: [CI]\n types: [in_progress]' + if [ "$trigger_block" != "$expected_trigger" ]; then + echo "FAIL: persistent Mac router must contain only workflow_run(in_progress) for CI" + exit 1 + fi + + if [ "$(grep -Fxc 'permissions: {}' "$PERSISTENT_ROUTER_FILE")" -ne 1 ]; then + echo "FAIL: persistent Mac router must have exactly one empty top-level permissions mapping" + exit 1 + fi + + local route_permissions expected_permissions + route_permissions="$(awk ' + /^ route:$/ { in_route=1; next } + in_route && /^ [A-Za-z0-9_-]+:$/ { exit } + in_route && /^ permissions:$/ { in_permissions=1; next } + in_permissions && /^ [A-Za-z0-9_-]+:/ { + line=$0 + sub(/^ /, "", line) + print line + next + } + in_permissions { exit } + ' "$PERSISTENT_ROUTER_FILE")" + expected_permissions=$'actions: write\ncontents: read\npull-requests: read' + if [ "$route_permissions" != "$expected_permissions" ]; then + echo "FAIL: default-branch router permissions must be exactly Actions write, contents read, and pull-requests read" + exit 1 + fi + + local checkout_with expected_checkout_with + checkout_with="$(awk ' + /^ - name: Checkout trusted router$/ { in_step=1; next } + in_step && /^ - name:/ { exit } + in_step && /^ with:$/ { in_with=1; next } + in_with && /^ [A-Za-z0-9_-]+:/ { + line=$0 + sub(/^ /, "", line) + print line + next + } + in_with && /^ [A-Za-z0-9_-]+:/ { exit } + ' "$PERSISTENT_ROUTER_FILE")" + expected_checkout_with=$'ref: main\npersist-credentials: false' + if [ "$checkout_with" != "$expected_checkout_with" ]; then + echo "FAIL: trusted router checkout must pin main and disable persisted credentials" + exit 1 + fi + + if ! grep -Fq -- '--observe-only' "$CI_FILE"; then + echo "FAIL: PR CI must remain read-only observation" + exit 1 + fi + if grep -Fq 'actions: write' "$CI_FILE"; then + echo "FAIL: PR-editable CI must not receive Actions write authority" + exit 1 + fi + echo "PASS: persistent dispatch/cancel authority is isolated to the exact default-branch router contract" } check_cla_guard_runner @@ -1218,6 +1364,8 @@ check_cla_guard_runner # ci.yml jobs check_no_bare_github_hosted_runners check_no_self_hosted_fleet_runners +check_persistent_compile_lane +check_persistent_compile_router check_macos_runner "$CI_FILE" "app-host-unit-tests" check_macos_runner "$CI_FILE" "macos-compile-admission" check_macos_runner "$CI_FILE" "tests-build-and-lag"