diff --git a/.agents/skills/manage-ci/references/current-inventory.md b/.agents/skills/manage-ci/references/current-inventory.md index ea6274bf33..48ac8c130d 100644 --- a/.agents/skills/manage-ci/references/current-inventory.md +++ b/.agents/skills/manage-ci/references/current-inventory.md @@ -59,7 +59,7 @@ removable after this branch's runner contract is active on protected main. | `ci-linux-sdk-slice.yml`, `ci-macos-sdk-slice.yml` | Platform-local Rust/Kotlin/Swift smoke consumers; SDK producers are independent top-level calls | | `ci-runner-contract-slice.yml` | Provider/cache/plan trust and main runner-image checks | | `native-sdk-artifact.yml` | Typed native SDK producer | -| `swift-sdk-artifact.yml` | Fixed `macos-15` host-only/full XCFramework producer | +| `swift-sdk-artifact.yml` | Host-only/full XCFramework producer; trusted main remains `macos-15`, while eligible same-repository PRs follow the protected Depot macOS 15 gate | | `smoke.yml` | Artifact-based inference/OpenAI/split smoke | | `scripted-binary-smoke.yml` | Artifact-based scripted product smoke | | `sdk-smoke.yml` | Artifact-based SDK consumers | @@ -109,6 +109,15 @@ from that same catalog. - `configure-sccache-gha`: event/provider-derived compiler-cache setup. - `capture-sccache-stats`: machine-readable cache evidence. +`scripts/collect-ci-metrics.py` is the read-only timing evidence collector. Its +schema-v3 report keeps workflow wall/queue, job runner queue, measured +dependency wait, job execution, runner-minutes, cancelled runner-minutes and +peak workers separate. It groups observations by provider, operating system, +architecture, semantic runner role and Depot size, and emits deterministic +queue/capacity heuristics plus an optional provider-cohort comparison. Raw +inputs and dated reports belong under `/tmp` or a tracking issue/artifact, not +under `ci/` or this inventory. + Artifacts are correctness boundaries; caches only accelerate regeneration. PR artifacts generally retain for one day. Protected same-repository and fork lanes cannot publish shared trusted-main caches, and Depot cache access is @@ -129,15 +138,32 @@ non-fail-fast, failed producers suppress only declared consumers through GitHub-hosted labels are `ubuntu-24.04`, `ubuntu-24.04-arm`, `macos-15`, and `windows-2022`. Depot labels are selected only by `select-ci-runners` for trusted main Linux when `DEPOT_RUNNERS_ENABLED` is exactly `true`; no workflow -accepts a raw provider label. The documented `gpu-nvidia` ephemeral scale set -is the sole uncredentialed, hardware-qualified same-repository PR exception. +accepts a raw provider label. Current ordinary PR execution remains hosted +until the protected PR gate is separately activated. The intended final gate +may cover eligible build/test rows across Linux, Depot macOS 15 and Windows +2022 when equivalent images/architectures exist; planning/required summaries, +credential-bearing smokes, `gpu-nvidia` hardware and uncertified Intel macOS +rows remain exceptions. The documented `gpu-nvidia` ephemeral scale set is +the sole current uncredentialed, hardware-qualified same-repository PR +exception. The future Depot PR gate is documented in `ci/DEPOT_MIGRATION.md`. It requires cache isolation, no PR cache/registry tokens, exact protected workflow refs, ephemeral runners, a sentinel canary, and a tested GitHub rollback. No Depot settings or runner groups are changed by this workflow refactor. +External administrative posture is now verified as follows: automatic Depot +Cache connectivity is disabled, automatic Registry Actions authentication is +disabled, and the Depot runner group is restricted to `Mesh-LLM/mesh-llm` and +the exact protected workflow refs. The repository token cannot independently +inspect organization runner-group settings through the API (403), so these +remain external facts rather than checked-in evidence. The protected +branch/main canary, same-repository PR canary, fork PR canary, provider-parity +comparison, and rollback run are still pending; `DEPOT_RUNNERS_ENABLED` or a +successful manual canary does not prove PR isolation. + Relevant repository variable names include `DEPOT_RUNNERS_ENABLED`, +`DEPOT_PR_RUNNERS_ENABLED` (absent/false until protected PR activation), `CUDA_VERSION`, `VULKAN_SDK_VERSION`, smoke configuration variables, and release/deployment variables. Secret values never belong in this inventory; known names include `HF_TOKEN`, release-attestation keys, `CARGO_REGISTRY_TOKEN` diff --git a/.github/actions/audit-depot-pr-isolation/action.yml b/.github/actions/audit-depot-pr-isolation/action.yml new file mode 100644 index 0000000000..bf2fffb9ee --- /dev/null +++ b/.github/actions/audit-depot-pr-isolation/action.yml @@ -0,0 +1,70 @@ +name: Audit Depot PR isolation +description: Fail closed when a PR build receives Depot cache, registry, or compiler-cache authority. + +inputs: + original_event_name: + description: Original event used to derive the trust boundary. + required: false + default: '' + +runs: + using: composite + steps: + - name: Verify PR runner has no privileged Depot credentials + shell: bash + env: + INPUT_ORIGINAL_EVENT_NAME: ${{ inputs.original_event_name }} + run: | + set -euo pipefail + original_event="${INPUT_ORIGINAL_EVENT_NAME:-${GITHUB_EVENT_NAME:-}}" + if [[ "$original_event" != "pull_request" && + "$original_event" != "pull_request_target" ]]; then + exit 0 + fi + + forbidden_names=( + DEPOT_CACHE_TOKEN + DEPOT_TOKEN + DEPOT_REGISTRY_PULL_TOKEN + DEPOT_REGISTRY_TOKEN + DEPOT_CACHE_URL + DEPOT_CACHE_API_URL + DEPOT_REGISTRY_URL + DEPOT_REGISTRY_HOST + SCCACHE_WEBDAV_ENDPOINT + SCCACHE_WEBDAV_TOKEN + SCCACHE_WEBDAV_USERNAME + SCCACHE_WEBDAV_PASSWORD + TURBO_TOKEN + TURBO_API + TURBO_TEAM + GOCACHEPROG + ) + for name in "${forbidden_names[@]}"; do + if [[ -n "${!name:-}" ]]; then + echo "PR runner received forbidden credential/cache authority: $name" >&2 + exit 1 + fi + done + + for endpoint_name in ACTIONS_CACHE_URL ACTIONS_RESULTS_URL; do + endpoint="${!endpoint_name:-}" + endpoint_lower="$(printf '%s' "$endpoint" | tr '[:upper:]' '[:lower:]')" + if [[ ! "$endpoint_lower" =~ ^https://([a-z0-9-]+\.)*actions\.githubusercontent\.com(:[0-9]+)?([/?#]|$) ]]; then + echo "PR runner received a non-GitHub Actions endpoint: $endpoint_name" >&2 + exit 1 + fi + done + + docker_auth_config="${DOCKER_AUTH_CONFIG:-}" + if [[ -n "$docker_auth_config" ]] && + printf '%s' "$docker_auth_config" | grep -Eiq 'depot\.dev'; then + echo "PR runner has Depot registry authentication configured" >&2 + exit 1 + fi + + docker_config="${DOCKER_CONFIG:-${HOME:-}/.docker}/config.json" + if [[ -f "$docker_config" ]] && grep -Eiq 'depot\.dev' "$docker_config"; then + echo "PR runner has Depot registry authentication configured" >&2 + exit 1 + fi diff --git a/.github/actions/select-ci-runners/action.yml b/.github/actions/select-ci-runners/action.yml index 20fcc238d4..b7c49cbf85 100644 --- a/.github/actions/select-ci-runners/action.yml +++ b/.github/actions/select-ci-runners/action.yml @@ -1,10 +1,20 @@ name: Select trusted CI runners -description: Select Depot or GitHub-hosted Linux runners without routing pull-request events to Depot. +description: Select Depot or GitHub-hosted CI runners from the real event and trust context. inputs: event_name: description: GitHub event name. required: true + original_event_name: + description: Source event declared by a workflow-dispatch controller, if any. + required: false + default: "" + repository: + description: Exact owner/name from the GitHub repository context. + required: true + head_repository: + description: Exact owner/name of the pull-request head repository, or repository outside PR events. + required: true ref: description: Fully qualified Git ref used to enforce the trusted-main boundary. required: true @@ -12,6 +22,14 @@ inputs: description: Repository-wide Depot gate for trusted push and dispatch events. required: false default: "false" + depot_pr_enabled: + description: Exact-string repository gate for direct pull-request Depot placement. + required: false + default: "false" + force_hosted: + description: Fail-closed planner signal for CI-control or runner-policy changes. + required: false + default: "false" manual_use_depot: description: Typed workflow-dispatch canary selection. required: false @@ -48,6 +66,12 @@ outputs: runner_arm_16: description: Sixteen-vCPU Linux ARM64 runner label for measured high-parallelism builds. value: ${{ steps.select.outputs.runner_arm_16 }} + runner_macos: + description: macOS runner label selected by the centralized provider policy. + value: ${{ steps.select.outputs.runner_macos }} + runner_windows: + description: Windows runner label selected by the centralized provider policy. + value: ${{ steps.select.outputs.runner_windows }} runs: using: composite @@ -57,15 +81,22 @@ runs: shell: bash env: INPUT_EVENT_NAME: ${{ inputs.event_name }} + INPUT_ORIGINAL_EVENT_NAME: ${{ inputs.original_event_name }} DISPATCH_ORIGINAL_EVENT_NAME: ${{ github.event.inputs.original_event_name || '' }} + INPUT_REPOSITORY: ${{ inputs.repository }} + INPUT_HEAD_REPOSITORY: ${{ inputs.head_repository }} INPUT_REF: ${{ inputs.ref }} INPUT_DEPOT_MAIN_ENABLED: ${{ inputs.depot_main_enabled }} + INPUT_DEPOT_PR_ENABLED: ${{ inputs.depot_pr_enabled }} + INPUT_FORCE_HOSTED: ${{ inputs.force_hosted }} INPUT_MANUAL_USE_DEPOT: ${{ inputs.manual_use_depot }} run: | set -euo pipefail for value_name in \ INPUT_DEPOT_MAIN_ENABLED \ + INPUT_DEPOT_PR_ENABLED \ + INPUT_FORCE_HOSTED \ INPUT_MANUAL_USE_DEPOT do value="${!value_name}" @@ -75,35 +106,70 @@ runs: fi done + # A direct PR may opt into Depot only when this protected repository + # explicitly enables the feature. Both the base and PR-head repository + # comparisons remain exact so a fork, mirror, or similarly named + # repository cannot opt in. + trusted_repository="Mesh-LLM/mesh-llm" depot_enabled=false - effective_event_name="$INPUT_EVENT_NAME" - if [[ "$INPUT_EVENT_NAME" == "workflow_dispatch" ]]; then - case "${DISPATCH_ORIGINAL_EVENT_NAME:-}" in - pull_request|pull_request_target) effective_event_name=pull_request ;; - push) effective_event_name=push ;; - esac + allow_depot_remote_cache=false + is_direct_pull_request=false + is_dispatched_pull_request=false + dispatch_original_event="${INPUT_ORIGINAL_EVENT_NAME:-}" + if [[ -z "$dispatch_original_event" ]]; then + dispatch_original_event="${DISPATCH_ORIGINAL_EVENT_NAME:-}" fi - case "$effective_event_name" in - pull_request|pull_request_target) - # Current PR workflows and local actions are evaluated from - # PR-controlled code. A repository variable or same-repo check - # therefore cannot establish a runner trust boundary. + + case "$INPUT_EVENT_NAME" in + pull_request) + # The caller must use the protected default-branch reusable lane; + # only the canonical PR merge ref is eligible for this path. + if [[ "$INPUT_REPOSITORY" == "$trusted_repository" && \ + "$INPUT_HEAD_REPOSITORY" == "$trusted_repository" && \ + "$INPUT_REF" =~ ^refs/pull/[0-9]+/merge$ && \ + "$INPUT_FORCE_HOSTED" == "false" && \ + "$INPUT_DEPOT_PR_ENABLED" == "true" ]]; then + is_direct_pull_request=true + depot_enabled=true + fi + ;; + pull_request_target) + # Never execute pull_request_target content on Depot, even when a + # caller declares a matching repository, ref, or feature gate. depot_enabled=false ;; workflow_dispatch) - if [[ "$INPUT_REF" == "refs/heads/main" && ( \ - "$INPUT_DEPOT_MAIN_ENABLED" == "true" || \ - "$INPUT_MANUAL_USE_DEPOT" == "true" \ - ) ]]; then - depot_enabled=true - fi + # A dispatch controller can label its source event for cache and + # cancellation policy. A declared PR source is still untrusted and + # must remain on GitHub; it must not become a Depot opt-in. + case "$dispatch_original_event" in + pull_request|pull_request_target) + is_dispatched_pull_request=true + ;; + push) + if [[ "$INPUT_REPOSITORY" == "$trusted_repository" && \ + "$INPUT_REF" == "refs/heads/main" && \ + "$INPUT_DEPOT_MAIN_ENABLED" == "true" ]]; then + depot_enabled=true + fi + ;; + workflow_dispatch|"") + if [[ "$INPUT_REPOSITORY" == "$trusted_repository" && \ + "$INPUT_REF" == "refs/heads/main" && ( \ + "$INPUT_DEPOT_MAIN_ENABLED" == "true" || \ + "$INPUT_MANUAL_USE_DEPOT" == "true" \ + ) ]]; then + depot_enabled=true + fi + ;; + esac ;; push) - if [[ \ + if [[ "$INPUT_REPOSITORY" == "$trusted_repository" && \ "$INPUT_REF" == "refs/heads/main" && \ - "$INPUT_DEPOT_MAIN_ENABLED" == "true" \ - ]]; then + "$INPUT_DEPOT_MAIN_ENABLED" == "true" ]]; then depot_enabled=true + allow_depot_remote_cache=true fi ;; *) @@ -111,6 +177,16 @@ runs: ;; esac + # Depot's repository-scoped cache is never authorized by a PR gate. + # Trusted main push/dispatch paths are the only paths that may retain + # the existing remote-cache behavior. + if [[ "$depot_enabled" == "true" && \ + "$is_direct_pull_request" != "true" && \ + "$is_dispatched_pull_request" != "true" && \ + "$INPUT_EVENT_NAME" == "workflow_dispatch" ]]; then + allow_depot_remote_cache=true + fi + if [[ "$depot_enabled" == "true" ]]; then runner=depot-ubuntu-24.04 runner_4=depot-ubuntu-24.04-4 @@ -120,6 +196,8 @@ runs: runner_arm_4=depot-ubuntu-24.04-arm-4 runner_arm_8=depot-ubuntu-24.04-arm-8 runner_arm_16=depot-ubuntu-24.04-arm-16 + runner_macos=depot-macos-15 + runner_windows=depot-windows-2022 else runner=ubuntu-24.04 runner_4=ubuntu-24.04 @@ -129,11 +207,13 @@ runs: runner_arm_4=ubuntu-24.04-arm runner_arm_8=ubuntu-24.04-arm runner_arm_16=ubuntu-24.04-arm + runner_macos=macos-15 + runner_windows=windows-2022 fi { echo "depot_enabled=$depot_enabled" - echo "allow_depot_remote_cache=$depot_enabled" + echo "allow_depot_remote_cache=$allow_depot_remote_cache" echo "runner=$runner" echo "runner_4=$runner_4" echo "runner_8=$runner_8" @@ -142,4 +222,6 @@ runs: echo "runner_arm_4=$runner_arm_4" echo "runner_arm_8=$runner_arm_8" echo "runner_arm_16=$runner_arm_16" + echo "runner_macos=$runner_macos" + echo "runner_windows=$runner_windows" } >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/ci-linux-host-slice.yml b/.github/workflows/ci-linux-host-slice.yml index c787f2ee10..2d8862217b 100644 --- a/.github/workflows/ci-linux-host-slice.yml +++ b/.github/workflows/ci-linux-host-slice.yml @@ -7,6 +7,16 @@ on: description: Immutable product source revision. required: true type: string + original_event_name: + description: Original event used for protected runner and cache policy. + required: false + default: '' + type: string + force_hosted: + description: Force GitHub-hosted placement for runner-contract changes. + required: false + default: false + type: boolean hosts_matrix: description: JSON Linux host rows from the versioned CI plan. required: true @@ -63,8 +73,13 @@ jobs: uses: ./.github/actions/select-ci-runners with: event_name: ${{ github.event_name }} + original_event_name: ${{ inputs.original_event_name }} + repository: ${{ github.repository }} + head_repository: ${{ github.event.pull_request.head.repo.full_name }} ref: ${{ github.ref }} depot_main_enabled: ${{ vars.DEPOT_RUNNERS_ENABLED == 'true' }} + depot_pr_enabled: ${{ vars.DEPOT_PR_RUNNERS_ENABLED == 'true' }} + force_hosted: ${{ inputs.force_hosted }} manual_use_depot: ${{ inputs.use_depot }} linux_host: @@ -80,9 +95,6 @@ jobs: host: ${{ fromJson(inputs.hosts_matrix) }} container: image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d - credentials: - username: ${{ github.actor }} - password: ${{ github.token }} defaults: run: shell: bash @@ -92,6 +104,9 @@ jobs: RUSTC_WRAPPER: sccache RUSTFLAGS: "-C link-arg=-fuse-ld=lld" steps: + - uses: Mesh-LLM/mesh-llm/.github/actions/audit-depot-pr-isolation@4f9899c0c381b8cc225c9be73980ffa8db525c57 + with: + original_event_name: ${{ inputs.original_event_name }} - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: ref: ${{ inputs.source_sha || github.sha }} diff --git a/.github/workflows/ci-linux-lane.yml b/.github/workflows/ci-linux-lane.yml index 9021dc062f..0dbf89feb5 100644 --- a/.github/workflows/ci-linux-lane.yml +++ b/.github/workflows/ci-linux-lane.yml @@ -61,6 +61,8 @@ jobs: uses: ./.github/workflows/ci-ui-artifact-slice.yml with: source_sha: ${{ inputs.source_sha }} + original_event_name: ${{ inputs.original_event_name }} + force_hosted: ${{ fromJson(inputs.lane_plan_json).signals.runner_contract_required }} artifact_name: ci-ui-dist-linux-${{ github.run_id }} retention_days: 1 @@ -70,6 +72,8 @@ jobs: uses: ./.github/workflows/static-abi-artifact.yml with: source_sha: ${{ inputs.source_sha }} + original_event_name: ${{ inputs.original_event_name }} + force_hosted: ${{ fromJson(inputs.lane_plan_json).signals.runner_contract_required }} backend: cpu target: x86_64-unknown-linux-gnu artifact_name: ci-static-abi-${{ github.run_id }} @@ -84,6 +88,8 @@ jobs: uses: ./.github/workflows/ci-rust-tests-slice.yml with: source_sha: ${{ inputs.source_sha }} + original_event_name: ${{ inputs.original_event_name }} + force_hosted: ${{ fromJson(inputs.lane_plan_json).signals.runner_contract_required }} rust_tests_matrix: ${{ toJson(fromJson(inputs.lane_plan_json).matrices.rust_tests) }} static_abi_artifact_name: ci-static-abi-${{ github.run_id }} static_abi_toolchain_epoch: ${{ needs.static_abi.outputs.toolchain_epoch }} @@ -97,6 +103,8 @@ jobs: uses: ./.github/workflows/ci-linux-host-slice.yml with: source_sha: ${{ inputs.source_sha }} + original_event_name: ${{ inputs.original_event_name }} + force_hosted: ${{ fromJson(inputs.lane_plan_json).signals.runner_contract_required }} hosts_matrix: ${{ toJson(fromJson(inputs.lane_plan_json).matrices.hosts) }} ui_artifact_name: ci-ui-dist-linux-${{ github.run_id }} profile: release @@ -110,6 +118,8 @@ jobs: uses: ./.github/workflows/ci-linux-runtime-slice.yml with: source_sha: ${{ inputs.source_sha }} + original_event_name: ${{ inputs.original_event_name }} + force_hosted: ${{ fromJson(inputs.lane_plan_json).signals.runner_contract_required }} runtime_matrix: ${{ toJson(fromJson(inputs.lane_plan_json).matrices.runtime_products) }} max_parallel: ${{ fromJson(inputs.lane_plan_json).budgets.linux_max_parallel }} fail_fast: ${{ inputs.original_event_name == 'pull_request' }} @@ -122,6 +132,8 @@ jobs: uses: ./.github/workflows/ci-linux-product-slice.yml with: source_sha: ${{ inputs.source_sha }} + original_event_name: ${{ inputs.original_event_name }} + force_hosted: ${{ fromJson(inputs.lane_plan_json).signals.runner_contract_required }} runtime_matrix: ${{ toJson(fromJson(inputs.lane_plan_json).matrices.runtime_products) }} max_parallel: ${{ fromJson(inputs.lane_plan_json).budgets.linux_max_parallel }} fail_fast: ${{ inputs.original_event_name == 'pull_request' }} @@ -133,6 +145,8 @@ jobs: uses: ./.github/workflows/native-sdk-artifact.yml with: source_sha: ${{ inputs.source_sha }} + original_event_name: ${{ inputs.original_event_name }} + force_hosted: ${{ fromJson(inputs.lane_plan_json).signals.runner_contract_required }} backend: cpu target: x86_64-unknown-linux-gnu profile: release diff --git a/.github/workflows/ci-linux-product-slice.yml b/.github/workflows/ci-linux-product-slice.yml index 0186973254..aa413df59a 100644 --- a/.github/workflows/ci-linux-product-slice.yml +++ b/.github/workflows/ci-linux-product-slice.yml @@ -7,6 +7,16 @@ on: description: Immutable product source revision. required: true type: string + original_event_name: + description: Original event used for protected runner and cache policy. + required: false + default: '' + type: string + force_hosted: + description: Force GitHub-hosted placement for runner-contract changes. + required: false + default: false + type: boolean runtime_matrix: description: JSON Linux product rows from the versioned CI plan. required: true @@ -27,10 +37,38 @@ permissions: packages: read jobs: + runner_policy: + name: Select Linux product runner + if: ${{ inputs.runtime_matrix != '[]' }} + runs-on: ubuntu-24.04 + permissions: + contents: read + outputs: + runner: ${{ steps.policy.outputs.runner }} + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - name: Select provider for the Linux product role + id: policy + uses: ./.github/actions/select-ci-runners + with: + event_name: ${{ github.event_name }} + original_event_name: ${{ inputs.original_event_name }} + repository: ${{ github.repository }} + head_repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.ref }} + # Product composition is not in the trusted-main Depot allowlist. + depot_main_enabled: 'false' + depot_pr_enabled: ${{ vars.DEPOT_PR_RUNNERS_ENABLED == 'true' }} + force_hosted: ${{ inputs.force_hosted }} + linux_product: name: Linux product (${{ matrix.runtime.backend }}) + needs: runner_policy if: ${{ inputs.runtime_matrix != '[]' }} - runs-on: ubuntu-24.04 + runs-on: ${{ needs.runner_policy.outputs.runner }} timeout-minutes: 15 strategy: fail-fast: ${{ inputs.fail_fast }} @@ -39,13 +77,13 @@ jobs: runtime: ${{ fromJson(inputs.runtime_matrix) }} container: image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d - credentials: - username: ${{ github.actor }} - password: ${{ github.token }} defaults: run: shell: bash steps: + - uses: Mesh-LLM/mesh-llm/.github/actions/audit-depot-pr-isolation@4f9899c0c381b8cc225c9be73980ffa8db525c57 + with: + original_event_name: ${{ inputs.original_event_name }} - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: ref: ${{ inputs.source_sha || github.sha }} diff --git a/.github/workflows/ci-linux-runtime-slice.yml b/.github/workflows/ci-linux-runtime-slice.yml index 3e7e355880..e31d111493 100644 --- a/.github/workflows/ci-linux-runtime-slice.yml +++ b/.github/workflows/ci-linux-runtime-slice.yml @@ -7,6 +7,16 @@ on: description: Immutable product source revision. required: true type: string + original_event_name: + description: Original event used for protected runner and cache policy. + required: false + default: '' + type: string + force_hosted: + description: Force GitHub-hosted placement for runner-contract changes. + required: false + default: false + type: boolean runtime_matrix: description: JSON Linux runtime rows from the versioned CI plan. required: true @@ -56,8 +66,13 @@ jobs: uses: ./.github/actions/select-ci-runners with: event_name: ${{ github.event_name }} + original_event_name: ${{ inputs.original_event_name }} + repository: ${{ github.repository }} + head_repository: ${{ github.event.pull_request.head.repo.full_name }} ref: ${{ github.ref }} depot_main_enabled: ${{ vars.DEPOT_RUNNERS_ENABLED == 'true' }} + depot_pr_enabled: ${{ vars.DEPOT_PR_RUNNERS_ENABLED == 'true' }} + force_hosted: ${{ inputs.force_hosted }} manual_use_depot: ${{ inputs.use_depot }} linux_runtime: @@ -73,9 +88,6 @@ jobs: runtime: ${{ fromJson(inputs.runtime_matrix) }} container: image: ${{ matrix.runtime.container_image }} - credentials: - username: ${{ github.actor }} - password: ${{ github.token }} defaults: run: shell: bash @@ -90,6 +102,9 @@ jobs: LLAMA_STAGE_SKIP_NCCL: "1" RUSTC_WRAPPER: sccache steps: + - uses: Mesh-LLM/mesh-llm/.github/actions/audit-depot-pr-isolation@4f9899c0c381b8cc225c9be73980ffa8db525c57 + with: + original_event_name: ${{ inputs.original_event_name }} - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: ref: ${{ inputs.source_sha || github.sha }} diff --git a/.github/workflows/ci-macos-host-slice.yml b/.github/workflows/ci-macos-host-slice.yml index da2a896771..735acab19c 100644 --- a/.github/workflows/ci-macos-host-slice.yml +++ b/.github/workflows/ci-macos-host-slice.yml @@ -7,6 +7,16 @@ on: description: Immutable product source revision. required: true type: string + original_event_name: + description: Original event used for protected runner and cache policy. + required: false + default: '' + type: string + force_hosted: + description: Force GitHub-hosted placement for runner-contract changes. + required: false + default: false + type: boolean hosts_matrix: description: JSON macOS host rows from the versioned CI plan. required: true @@ -40,10 +50,37 @@ env: SCCACHE_GHA_ENABLED: "true" jobs: + runner_policy: + name: Select macOS host runner + if: ${{ inputs.hosts_matrix != '[]' }} + runs-on: ubuntu-24.04 + permissions: + contents: read + outputs: + runner_macos: ${{ steps.policy.outputs.runner_macos }} + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - name: Select provider for the macOS host role + id: policy + uses: ./.github/actions/select-ci-runners + with: + event_name: ${{ github.event_name }} + original_event_name: ${{ inputs.original_event_name }} + repository: ${{ github.repository }} + head_repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.ref }} + depot_main_enabled: 'false' + depot_pr_enabled: ${{ vars.DEPOT_PR_RUNNERS_ENABLED == 'true' }} + force_hosted: ${{ inputs.force_hosted }} + macos_host: name: macOS host (${{ matrix.host.architecture }}) + needs: runner_policy if: ${{ inputs.hosts_matrix != '[]' }} - runs-on: macos-15 + runs-on: ${{ needs.runner_policy.outputs.runner_macos }} timeout-minutes: 45 strategy: fail-fast: ${{ inputs.fail_fast }} @@ -51,6 +88,9 @@ jobs: matrix: host: ${{ fromJson(inputs.hosts_matrix) }} steps: + - uses: Mesh-LLM/mesh-llm/.github/actions/audit-depot-pr-isolation@4f9899c0c381b8cc225c9be73980ffa8db525c57 + with: + original_event_name: ${{ inputs.original_event_name }} - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: ref: ${{ inputs.source_sha || github.sha }} diff --git a/.github/workflows/ci-macos-lane.yml b/.github/workflows/ci-macos-lane.yml index 0f51a8cace..d0b18af9b4 100644 --- a/.github/workflows/ci-macos-lane.yml +++ b/.github/workflows/ci-macos-lane.yml @@ -72,6 +72,8 @@ jobs: uses: ./.github/workflows/ci-ui-artifact-slice.yml with: source_sha: ${{ inputs.source_sha }} + original_event_name: ${{ inputs.original_event_name }} + force_hosted: ${{ fromJson(inputs.lane_plan_json).signals.runner_contract_required }} artifact_name: ci-ui-dist-macos-${{ github.run_id }} retention_days: 1 @@ -82,6 +84,8 @@ jobs: uses: ./.github/workflows/ci-macos-host-slice.yml with: source_sha: ${{ inputs.source_sha }} + original_event_name: ${{ inputs.original_event_name }} + force_hosted: ${{ fromJson(inputs.lane_plan_json).signals.runner_contract_required }} hosts_matrix: ${{ toJson(fromJson(inputs.lane_plan_json).matrices.hosts) }} ui_artifact_name: ci-ui-dist-macos-${{ github.run_id }} profile: release @@ -94,6 +98,8 @@ jobs: uses: ./.github/workflows/ci-macos-runtime-slice.yml with: source_sha: ${{ inputs.source_sha }} + original_event_name: ${{ inputs.original_event_name }} + force_hosted: ${{ fromJson(inputs.lane_plan_json).signals.runner_contract_required }} runtime_matrix: ${{ toJson(fromJson(inputs.lane_plan_json).matrices.runtime_products) }} max_parallel: ${{ fromJson(inputs.lane_plan_json).budgets.macos_max_parallel }} fail_fast: ${{ inputs.original_event_name == 'pull_request' }} @@ -105,6 +111,8 @@ jobs: uses: ./.github/workflows/ci-macos-product-slice.yml with: source_sha: ${{ inputs.source_sha }} + original_event_name: ${{ inputs.original_event_name }} + force_hosted: ${{ fromJson(inputs.lane_plan_json).signals.runner_contract_required }} runtime_matrix: ${{ toJson(fromJson(inputs.lane_plan_json).matrices.runtime_products) }} max_parallel: ${{ fromJson(inputs.lane_plan_json).budgets.macos_max_parallel }} fail_fast: ${{ inputs.original_event_name == 'pull_request' }} @@ -115,9 +123,10 @@ jobs: uses: ./.github/workflows/ci-platform-checks-slice.yml with: source_sha: ${{ inputs.source_sha }} + original_event_name: ${{ inputs.original_event_name }} + force_hosted: ${{ fromJson(inputs.lane_plan_json).signals.runner_contract_required }} platform_checks_matrix: ${{ toJson(fromJson(inputs.lane_plan_json).matrices.platform_checks) }} profile: ${{ fromJson(inputs.lane_plan_json).profile }} - original_event_name: ${{ inputs.original_event_name }} max_parallel: ${{ fromJson(inputs.lane_plan_json).budgets.macos_max_parallel }} fail_fast: ${{ inputs.original_event_name == 'pull_request' }} @@ -127,6 +136,8 @@ jobs: uses: ./.github/workflows/swift-sdk-artifact.yml with: source_sha: ${{ inputs.source_sha }} + original_event_name: ${{ inputs.original_event_name }} + force_hosted: ${{ fromJson(inputs.lane_plan_json).signals.runner_contract_required }} mode: ${{ (fromJson(inputs.lane_plan_json).profile == 'main' || fromJson(inputs.lane_plan_json).profile == 'manual-full') && 'full' || 'host-only' }} artifact_name: ci-sdk-swift retention_days: 1 diff --git a/.github/workflows/ci-macos-product-slice.yml b/.github/workflows/ci-macos-product-slice.yml index 5525c0beff..b2b33bd2e4 100644 --- a/.github/workflows/ci-macos-product-slice.yml +++ b/.github/workflows/ci-macos-product-slice.yml @@ -7,6 +7,16 @@ on: description: Immutable product source revision. required: true type: string + original_event_name: + description: Original event used for protected runner and cache policy. + required: false + default: '' + type: string + force_hosted: + description: Force GitHub-hosted placement for runner-contract changes. + required: false + default: false + type: boolean runtime_matrix: description: JSON macOS product rows from the versioned CI plan. required: true @@ -26,10 +36,37 @@ permissions: contents: read jobs: + runner_policy: + name: Select macOS product runner + if: ${{ inputs.runtime_matrix != '[]' }} + runs-on: ubuntu-24.04 + permissions: + contents: read + outputs: + runner_macos: ${{ steps.policy.outputs.runner_macos }} + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - name: Select provider for the macOS product role + id: policy + uses: ./.github/actions/select-ci-runners + with: + event_name: ${{ github.event_name }} + original_event_name: ${{ inputs.original_event_name }} + repository: ${{ github.repository }} + head_repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.ref }} + depot_main_enabled: 'false' + depot_pr_enabled: ${{ vars.DEPOT_PR_RUNNERS_ENABLED == 'true' }} + force_hosted: ${{ inputs.force_hosted }} + macos_product: name: macOS product (${{ matrix.runtime.backend }}) + needs: runner_policy if: ${{ inputs.runtime_matrix != '[]' }} - runs-on: macos-15 + runs-on: ${{ needs.runner_policy.outputs.runner_macos }} timeout-minutes: 15 strategy: fail-fast: ${{ inputs.fail_fast }} @@ -37,6 +74,9 @@ jobs: matrix: runtime: ${{ fromJson(inputs.runtime_matrix) }} steps: + - uses: Mesh-LLM/mesh-llm/.github/actions/audit-depot-pr-isolation@4f9899c0c381b8cc225c9be73980ffa8db525c57 + with: + original_event_name: ${{ inputs.original_event_name }} - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: ref: ${{ inputs.source_sha || github.sha }} diff --git a/.github/workflows/ci-macos-runtime-slice.yml b/.github/workflows/ci-macos-runtime-slice.yml index 943985d0e8..392687b7c4 100644 --- a/.github/workflows/ci-macos-runtime-slice.yml +++ b/.github/workflows/ci-macos-runtime-slice.yml @@ -7,6 +7,16 @@ on: description: Immutable product source revision. required: true type: string + original_event_name: + description: Original event used for protected runner and cache policy. + required: false + default: '' + type: string + force_hosted: + description: Force GitHub-hosted placement for runner-contract changes. + required: false + default: false + type: boolean runtime_matrix: description: JSON macOS runtime rows from the versioned CI plan. required: true @@ -31,10 +41,37 @@ env: SCCACHE_GHA_ENABLED: "true" jobs: + runner_policy: + name: Select macOS runtime runner + if: ${{ inputs.runtime_matrix != '[]' }} + runs-on: ubuntu-24.04 + permissions: + contents: read + outputs: + runner_macos: ${{ steps.policy.outputs.runner_macos }} + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - name: Select provider for the macOS runtime role + id: policy + uses: ./.github/actions/select-ci-runners + with: + event_name: ${{ github.event_name }} + original_event_name: ${{ inputs.original_event_name }} + repository: ${{ github.repository }} + head_repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.ref }} + depot_main_enabled: 'false' + depot_pr_enabled: ${{ vars.DEPOT_PR_RUNNERS_ENABLED == 'true' }} + force_hosted: ${{ inputs.force_hosted }} + macos_runtime: name: macOS Metal runtime + needs: runner_policy if: ${{ inputs.runtime_matrix != '[]' }} - runs-on: macos-15 + runs-on: ${{ needs.runner_policy.outputs.runner_macos }} timeout-minutes: 75 strategy: fail-fast: ${{ inputs.fail_fast }} @@ -45,6 +82,9 @@ jobs: LLAMA_STAGE_BACKEND: ${{ matrix.runtime.backend }} LLAMA_STAGE_BUILD_DIR: ${{ matrix.runtime.build_dir }} steps: + - uses: Mesh-LLM/mesh-llm/.github/actions/audit-depot-pr-isolation@4f9899c0c381b8cc225c9be73980ffa8db525c57 + with: + original_event_name: ${{ inputs.original_event_name }} - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: ref: ${{ inputs.source_sha || github.sha }} diff --git a/.github/workflows/ci-platform-checks-slice.yml b/.github/workflows/ci-platform-checks-slice.yml index 2202091a97..7d9f8a36ef 100644 --- a/.github/workflows/ci-platform-checks-slice.yml +++ b/.github/workflows/ci-platform-checks-slice.yml @@ -20,6 +20,11 @@ on: description: Original event used to derive isolated cache-write policy. required: true type: string + force_hosted: + description: Force GitHub-hosted placement for runner-contract changes. + required: false + default: false + type: boolean max_parallel: description: Platform worker budget. required: false @@ -41,10 +46,48 @@ env: MESH_LLM_SKIP_UI: "1" jobs: + runner_policy: + name: Select platform-check runner + if: ${{ inputs.platform_checks_matrix != '[]' }} + runs-on: ubuntu-24.04 + permissions: + contents: read + outputs: + runner_macos: ${{ steps.policy.outputs.runner_macos }} + runner_windows: ${{ steps.policy.outputs.runner_windows }} + runner_by_platform: ${{ steps.platform_runners.outputs.runner_by_platform }} + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - name: Select provider for platform checks + id: policy + uses: ./.github/actions/select-ci-runners + with: + event_name: ${{ github.event_name }} + original_event_name: ${{ inputs.original_event_name }} + repository: ${{ github.repository }} + head_repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.ref }} + depot_main_enabled: 'false' + depot_pr_enabled: ${{ vars.DEPOT_PR_RUNNERS_ENABLED == 'true' }} + force_hosted: ${{ inputs.force_hosted }} + - name: Map platform runners explicitly + id: platform_runners + env: + RUNNER_MACOS: ${{ steps.policy.outputs.runner_macos }} + RUNNER_WINDOWS: ${{ steps.policy.outputs.runner_windows }} + run: | + set -euo pipefail + printf 'runner_by_platform={"macos":"%s","windows":"%s"}\n' \ + "$RUNNER_MACOS" "$RUNNER_WINDOWS" >> "$GITHUB_OUTPUT" + platform_checks: name: Platform checks (${{ matrix.check.id }}) + needs: runner_policy if: ${{ inputs.platform_checks_matrix != '[]' }} - runs-on: ${{ matrix.check.platform == 'macos' && 'macos-15' || 'windows-2022' }} + runs-on: ${{ fromJSON(needs.runner_policy.outputs.runner_by_platform)[matrix.check.platform] }} timeout-minutes: 75 strategy: fail-fast: ${{ inputs.fail_fast }} @@ -52,6 +95,9 @@ jobs: matrix: check: ${{ fromJson(inputs.platform_checks_matrix) }} steps: + - uses: Mesh-LLM/mesh-llm/.github/actions/audit-depot-pr-isolation@4f9899c0c381b8cc225c9be73980ffa8db525c57 + with: + original_event_name: ${{ inputs.original_event_name }} - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: ref: ${{ inputs.source_sha || github.sha }} diff --git a/.github/workflows/ci-quality-lane.yml b/.github/workflows/ci-quality-lane.yml index b5f0ddd62b..c482055034 100644 --- a/.github/workflows/ci-quality-lane.yml +++ b/.github/workflows/ci-quality-lane.yml @@ -66,6 +66,8 @@ jobs: cli_surface_changed: ${{ fromJson(inputs.lane_plan_json).signals.cli_surface_changed }} website_docs_changed: ${{ fromJson(inputs.lane_plan_json).signals.website_docs_changed }} clippy_max_parallel: ${{ fromJson(inputs.lane_plan_json).budgets.linux_max_parallel }} + original_event_name: ${{ inputs.original_event_name }} + force_hosted: ${{ fromJson(inputs.lane_plan_json).signals.runner_contract_required }} use_depot: ${{ inputs.use_depot }} runner_contract: diff --git a/.github/workflows/ci-quality-slice.yml b/.github/workflows/ci-quality-slice.yml index 6ba42c921f..45d7eaf9b9 100644 --- a/.github/workflows/ci-quality-slice.yml +++ b/.github/workflows/ci-quality-slice.yml @@ -8,6 +8,16 @@ on: required: false default: '' type: string + original_event_name: + description: Original event used for protected runner and cache policy. + required: false + default: '' + type: string + force_hosted: + description: Force GitHub-hosted placement for runner-contract changes. + required: false + default: false + type: boolean clippy_matrix: description: Validated JSON array of Clippy batches. required: false @@ -65,8 +75,13 @@ jobs: uses: ./.github/actions/select-ci-runners with: event_name: ${{ github.event_name }} + original_event_name: ${{ inputs.original_event_name }} + repository: ${{ github.repository }} + head_repository: ${{ github.event.pull_request.head.repo.full_name }} ref: ${{ github.ref }} depot_main_enabled: ${{ vars.DEPOT_RUNNERS_ENABLED == 'true' }} + depot_pr_enabled: ${{ vars.DEPOT_PR_RUNNERS_ENABLED == 'true' }} + force_hosted: ${{ inputs.force_hosted }} manual_use_depot: ${{ inputs.use_depot }} quality_contracts: @@ -75,6 +90,9 @@ jobs: runs-on: ${{ needs.runner_policy.outputs.runner_4 }} timeout-minutes: 20 steps: + - uses: Mesh-LLM/mesh-llm/.github/actions/audit-depot-pr-isolation@4f9899c0c381b8cc225c9be73980ffa8db525c57 + with: + original_event_name: ${{ inputs.original_event_name }} - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: ref: ${{ inputs.source_sha || github.sha }} @@ -106,6 +124,9 @@ jobs: runs-on: ${{ needs.runner_policy.outputs.runner_4 }} timeout-minutes: 15 steps: + - uses: Mesh-LLM/mesh-llm/.github/actions/audit-depot-pr-isolation@4f9899c0c381b8cc225c9be73980ffa8db525c57 + with: + original_event_name: ${{ inputs.original_event_name }} - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: ref: ${{ inputs.source_sha || github.sha }} @@ -129,9 +150,6 @@ jobs: batch: ${{ fromJson(inputs.clippy_matrix) }} container: image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d - credentials: - username: ${{ github.actor }} - password: ${{ github.token }} defaults: run: shell: bash @@ -139,6 +157,9 @@ jobs: RUSTFLAGS: "-C link-arg=-fuse-ld=lld" RUSTC_WRAPPER: sccache steps: + - uses: Mesh-LLM/mesh-llm/.github/actions/audit-depot-pr-isolation@4f9899c0c381b8cc225c9be73980ffa8db525c57 + with: + original_event_name: ${{ inputs.original_event_name }} - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: ref: ${{ inputs.source_sha || github.sha }} @@ -185,6 +206,9 @@ jobs: runs-on: ${{ needs.runner_policy.outputs.runner_4 }} timeout-minutes: 5 steps: + - uses: Mesh-LLM/mesh-llm/.github/actions/audit-depot-pr-isolation@4f9899c0c381b8cc225c9be73980ffa8db525c57 + with: + original_event_name: ${{ inputs.original_event_name }} - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: ref: ${{ inputs.source_sha || github.sha }} diff --git a/.github/workflows/ci-rust-tests-slice.yml b/.github/workflows/ci-rust-tests-slice.yml index b980f9a7e9..23dc048eed 100644 --- a/.github/workflows/ci-rust-tests-slice.yml +++ b/.github/workflows/ci-rust-tests-slice.yml @@ -8,6 +8,16 @@ on: required: false default: '' type: string + original_event_name: + description: Original event used for protected runner and cache policy. + required: false + default: '' + type: string + force_hosted: + description: Force GitHub-hosted placement for runner-contract changes. + required: false + default: false + type: boolean rust_tests_matrix: description: Validated JSON array of deterministic Rust test batches. required: true @@ -41,10 +51,38 @@ env: SCCACHE_GHA_ENABLED: "true" jobs: + runner_policy: + name: Select Rust test runner + if: ${{ inputs.rust_tests_matrix != '[]' }} + runs-on: ubuntu-24.04 + permissions: + contents: read + outputs: + runner: ${{ steps.policy.outputs.runner }} + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - name: Select provider for the Rust test role + id: policy + uses: ./.github/actions/select-ci-runners + with: + event_name: ${{ github.event_name }} + original_event_name: ${{ inputs.original_event_name }} + repository: ${{ github.repository }} + head_repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.ref }} + # Rust tests are not in the existing trusted-main Depot allowlist. + depot_main_enabled: 'false' + depot_pr_enabled: ${{ vars.DEPOT_PR_RUNNERS_ENABLED == 'true' }} + force_hosted: ${{ inputs.force_hosted }} + rust_tests: name: Rust tests (${{ matrix.batch.id }}) + needs: runner_policy if: ${{ inputs.rust_tests_matrix != '[]' }} - runs-on: ubuntu-24.04 + runs-on: ${{ needs.runner_policy.outputs.runner }} timeout-minutes: 60 strategy: fail-fast: ${{ inputs.fail_fast }} @@ -53,9 +91,6 @@ jobs: batch: ${{ fromJson(inputs.rust_tests_matrix) }} container: image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d - credentials: - username: ${{ github.actor }} - password: ${{ github.token }} defaults: run: shell: bash @@ -67,6 +102,9 @@ jobs: RUSTFLAGS: "-C link-arg=-fuse-ld=lld" RUSTC_WRAPPER: sccache steps: + - uses: Mesh-LLM/mesh-llm/.github/actions/audit-depot-pr-isolation@4f9899c0c381b8cc225c9be73980ffa8db525c57 + with: + original_event_name: ${{ inputs.original_event_name }} - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: ref: ${{ inputs.source_sha || github.sha }} diff --git a/.github/workflows/ci-ui-artifact-slice.yml b/.github/workflows/ci-ui-artifact-slice.yml index 907f54ad04..a29e3d8ca9 100644 --- a/.github/workflows/ci-ui-artifact-slice.yml +++ b/.github/workflows/ci-ui-artifact-slice.yml @@ -8,6 +8,16 @@ on: required: false default: '' type: string + original_event_name: + description: Original event used for protected runner and cache policy. + required: false + default: '' + type: string + force_hosted: + description: Force GitHub-hosted placement for runner-contract changes. + required: false + default: false + type: boolean artifact_name: description: Immutable UI distribution artifact consumed by host builders. required: true @@ -25,14 +35,45 @@ env: CACHE_NAMESPACE: mesh-llm jobs: + runner_policy: + name: Select console artifact runner + runs-on: ubuntu-24.04 + permissions: + contents: read + outputs: + runner_4: ${{ steps.policy.outputs.runner_4 }} + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - name: Select provider for the console artifact role + id: policy + uses: ./.github/actions/select-ci-runners + with: + event_name: ${{ github.event_name }} + original_event_name: ${{ inputs.original_event_name }} + repository: ${{ github.repository }} + head_repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.ref }} + # The pre-existing trusted-main allowlist does not include this + # producer. PR placement is independently gated and cache-isolated. + depot_main_enabled: 'false' + depot_pr_enabled: ${{ vars.DEPOT_PR_RUNNERS_ENABLED == 'true' }} + force_hosted: ${{ inputs.force_hosted }} + ui_artifact: name: Build immutable console UI - runs-on: ubuntu-24.04 + needs: runner_policy + runs-on: ${{ needs.runner_policy.outputs.runner_4 }} timeout-minutes: 20 defaults: run: working-directory: crates/mesh-llm-ui steps: + - uses: Mesh-LLM/mesh-llm/.github/actions/audit-depot-pr-isolation@4f9899c0c381b8cc225c9be73980ffa8db525c57 + with: + original_event_name: ${{ inputs.original_event_name }} - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: ref: ${{ inputs.source_sha || github.sha }} diff --git a/.github/workflows/ci-web-slice.yml b/.github/workflows/ci-web-slice.yml index e299363d4f..6daa91ff6a 100644 --- a/.github/workflows/ci-web-slice.yml +++ b/.github/workflows/ci-web-slice.yml @@ -8,6 +8,16 @@ on: required: false default: '' type: string + original_event_name: + description: Original event used for protected runner and cache policy. + required: false + default: '' + type: string + force_hosted: + description: Force GitHub-hosted placement for runner-contract changes. + required: false + default: false + type: boolean ui_changed: description: Whether the console UI source changed. required: true @@ -24,15 +34,45 @@ env: CACHE_NAMESPACE: mesh-llm jobs: + runner_policy: + name: Select website runner + runs-on: ubuntu-24.04 + permissions: + contents: read + outputs: + runner_4: ${{ steps.policy.outputs.runner_4 }} + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - name: Select provider for the website role + id: policy + uses: ./.github/actions/select-ci-runners + with: + event_name: ${{ github.event_name }} + original_event_name: ${{ inputs.original_event_name }} + repository: ${{ github.repository }} + head_repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.ref }} + # Website is not in the existing trusted-main Depot allowlist. + depot_main_enabled: 'false' + depot_pr_enabled: ${{ vars.DEPOT_PR_RUNNERS_ENABLED == 'true' }} + force_hosted: ${{ inputs.force_hosted }} + ui_quality: name: Console UI quality + needs: runner_policy if: ${{ inputs.ui_changed }} - runs-on: ubuntu-24.04 + runs-on: ${{ needs.runner_policy.outputs.runner_4 }} timeout-minutes: 20 defaults: run: working-directory: crates/mesh-llm-ui steps: + - uses: Mesh-LLM/mesh-llm/.github/actions/audit-depot-pr-isolation@4f9899c0c381b8cc225c9be73980ffa8db525c57 + with: + original_event_name: ${{ inputs.original_event_name }} - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: ref: ${{ inputs.source_sha || github.sha }} @@ -70,10 +110,14 @@ jobs: website: name: Public website build + needs: runner_policy if: ${{ inputs.website_changed }} - runs-on: ubuntu-24.04 + runs-on: ${{ needs.runner_policy.outputs.runner_4 }} timeout-minutes: 20 steps: + - uses: Mesh-LLM/mesh-llm/.github/actions/audit-depot-pr-isolation@4f9899c0c381b8cc225c9be73980ffa8db525c57 + with: + original_event_name: ${{ inputs.original_event_name }} - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: ref: ${{ inputs.source_sha || github.sha }} diff --git a/.github/workflows/ci-website-lane.yml b/.github/workflows/ci-website-lane.yml index 575d48ce04..8442fd9a04 100644 --- a/.github/workflows/ci-website-lane.yml +++ b/.github/workflows/ci-website-lane.yml @@ -52,6 +52,8 @@ jobs: uses: ./.github/workflows/ci-web-slice.yml with: source_sha: ${{ inputs.source_sha }} + original_event_name: ${{ inputs.original_event_name }} + force_hosted: ${{ fromJson(inputs.lane_plan_json).signals.runner_contract_required }} ui_changed: ${{ fromJson(inputs.lane_plan_json).signals.ui_changed || contains(fromJson(inputs.lane_plan_json).domains, 'ci-control') }} website_changed: ${{ fromJson(inputs.lane_plan_json).signals.website_changed || contains(fromJson(inputs.lane_plan_json).domains, 'ci-control') }} diff --git a/.github/workflows/ci-windows-host-slice.yml b/.github/workflows/ci-windows-host-slice.yml index 86cf35392b..082594256b 100644 --- a/.github/workflows/ci-windows-host-slice.yml +++ b/.github/workflows/ci-windows-host-slice.yml @@ -7,6 +7,16 @@ on: description: Immutable product source revision. required: true type: string + original_event_name: + description: Original event used for protected runner and cache policy. + required: false + default: '' + type: string + force_hosted: + description: Force GitHub-hosted placement for runner-contract changes. + required: false + default: false + type: boolean hosts_matrix: description: JSON Windows host rows from the versioned CI plan. required: true @@ -40,10 +50,37 @@ env: SCCACHE_GHA_ENABLED: "true" jobs: + runner_policy: + name: Select Windows host runner + if: ${{ inputs.hosts_matrix != '[]' }} + runs-on: ubuntu-24.04 + permissions: + contents: read + outputs: + runner_windows: ${{ steps.policy.outputs.runner_windows }} + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - name: Select provider for the Windows host role + id: policy + uses: ./.github/actions/select-ci-runners + with: + event_name: ${{ github.event_name }} + original_event_name: ${{ inputs.original_event_name }} + repository: ${{ github.repository }} + head_repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.ref }} + depot_main_enabled: 'false' + depot_pr_enabled: ${{ vars.DEPOT_PR_RUNNERS_ENABLED == 'true' }} + force_hosted: ${{ inputs.force_hosted }} + windows_host: name: Windows host (${{ matrix.host.architecture }}) + needs: runner_policy if: ${{ inputs.hosts_matrix != '[]' }} - runs-on: windows-2022 + runs-on: ${{ needs.runner_policy.outputs.runner_windows }} timeout-minutes: 50 strategy: fail-fast: ${{ inputs.fail_fast }} @@ -54,6 +91,9 @@ jobs: MESH_LLM_REQUIRE_SCCACHE: "1" RUSTC_WRAPPER: sccache steps: + - uses: Mesh-LLM/mesh-llm/.github/actions/audit-depot-pr-isolation@4f9899c0c381b8cc225c9be73980ffa8db525c57 + with: + original_event_name: ${{ inputs.original_event_name }} - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: ref: ${{ inputs.source_sha || github.sha }} diff --git a/.github/workflows/ci-windows-lane.yml b/.github/workflows/ci-windows-lane.yml index a2458f8593..e94a2938a7 100644 --- a/.github/workflows/ci-windows-lane.yml +++ b/.github/workflows/ci-windows-lane.yml @@ -53,6 +53,8 @@ jobs: uses: ./.github/workflows/ci-ui-artifact-slice.yml with: source_sha: ${{ inputs.source_sha }} + original_event_name: ${{ inputs.original_event_name }} + force_hosted: ${{ fromJson(inputs.lane_plan_json).signals.runner_contract_required }} artifact_name: ci-ui-dist-windows-${{ github.run_id }} retention_days: 1 @@ -63,6 +65,8 @@ jobs: uses: ./.github/workflows/ci-windows-host-slice.yml with: source_sha: ${{ inputs.source_sha }} + original_event_name: ${{ inputs.original_event_name }} + force_hosted: ${{ fromJson(inputs.lane_plan_json).signals.runner_contract_required }} hosts_matrix: ${{ toJson(fromJson(inputs.lane_plan_json).matrices.hosts) }} ui_artifact_name: ci-ui-dist-windows-${{ github.run_id }} profile: release @@ -76,6 +80,7 @@ jobs: with: source_sha: ${{ inputs.source_sha }} original_event_name: ${{ inputs.original_event_name }} + force_hosted: ${{ fromJson(inputs.lane_plan_json).signals.runner_contract_required }} runtime_matrix: ${{ toJson(fromJson(inputs.lane_plan_json).matrices.runtime_products) }} max_parallel: ${{ fromJson(inputs.lane_plan_json).budgets.windows_max_parallel }} fail_fast: ${{ inputs.original_event_name == 'pull_request' }} @@ -87,6 +92,8 @@ jobs: uses: ./.github/workflows/ci-windows-product-slice.yml with: source_sha: ${{ inputs.source_sha }} + original_event_name: ${{ inputs.original_event_name }} + force_hosted: ${{ fromJson(inputs.lane_plan_json).signals.runner_contract_required }} runtime_matrix: ${{ toJson(fromJson(inputs.lane_plan_json).matrices.runtime_products) }} max_parallel: ${{ fromJson(inputs.lane_plan_json).budgets.windows_max_parallel }} fail_fast: ${{ inputs.original_event_name == 'pull_request' }} @@ -97,9 +104,10 @@ jobs: uses: ./.github/workflows/ci-platform-checks-slice.yml with: source_sha: ${{ inputs.source_sha }} + original_event_name: ${{ inputs.original_event_name }} + force_hosted: ${{ fromJson(inputs.lane_plan_json).signals.runner_contract_required }} platform_checks_matrix: ${{ toJson(fromJson(inputs.lane_plan_json).matrices.platform_checks) }} profile: ${{ fromJson(inputs.lane_plan_json).profile }} - original_event_name: ${{ inputs.original_event_name }} max_parallel: ${{ fromJson(inputs.lane_plan_json).budgets.windows_max_parallel }} fail_fast: ${{ inputs.original_event_name == 'pull_request' }} diff --git a/.github/workflows/ci-windows-product-slice.yml b/.github/workflows/ci-windows-product-slice.yml index e950ea358c..14cc5687ba 100644 --- a/.github/workflows/ci-windows-product-slice.yml +++ b/.github/workflows/ci-windows-product-slice.yml @@ -7,6 +7,16 @@ on: description: Immutable product source revision. required: true type: string + original_event_name: + description: Original event used for protected runner and cache policy. + required: false + default: '' + type: string + force_hosted: + description: Force GitHub-hosted placement for runner-contract changes. + required: false + default: false + type: boolean runtime_matrix: description: JSON Windows product rows from the versioned CI plan. required: true @@ -26,10 +36,37 @@ permissions: contents: read jobs: + runner_policy: + name: Select Windows product runner + if: ${{ inputs.runtime_matrix != '[]' }} + runs-on: ubuntu-24.04 + permissions: + contents: read + outputs: + runner_windows: ${{ steps.policy.outputs.runner_windows }} + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - name: Select provider for the Windows product role + id: policy + uses: ./.github/actions/select-ci-runners + with: + event_name: ${{ github.event_name }} + original_event_name: ${{ inputs.original_event_name }} + repository: ${{ github.repository }} + head_repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.ref }} + depot_main_enabled: 'false' + depot_pr_enabled: ${{ vars.DEPOT_PR_RUNNERS_ENABLED == 'true' }} + force_hosted: ${{ inputs.force_hosted }} + windows_product: name: Windows product (${{ matrix.runtime.backend }}) + needs: runner_policy if: ${{ inputs.runtime_matrix != '[]' }} - runs-on: windows-2022 + runs-on: ${{ needs.runner_policy.outputs.runner_windows }} timeout-minutes: 20 strategy: fail-fast: ${{ inputs.fail_fast }} @@ -37,6 +74,9 @@ jobs: matrix: runtime: ${{ fromJson(inputs.runtime_matrix) }} steps: + - uses: Mesh-LLM/mesh-llm/.github/actions/audit-depot-pr-isolation@4f9899c0c381b8cc225c9be73980ffa8db525c57 + with: + original_event_name: ${{ inputs.original_event_name }} - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: ref: ${{ inputs.source_sha || github.sha }} diff --git a/.github/workflows/ci-windows-runtime-slice.yml b/.github/workflows/ci-windows-runtime-slice.yml index e27c3ee0e9..b7769759aa 100644 --- a/.github/workflows/ci-windows-runtime-slice.yml +++ b/.github/workflows/ci-windows-runtime-slice.yml @@ -11,6 +11,11 @@ on: description: Original event used to derive trusted cache-write policy. required: true type: string + force_hosted: + description: Force GitHub-hosted placement for runner-contract changes. + required: false + default: false + type: boolean runtime_matrix: description: JSON Windows runtime rows from the versioned CI plan. required: true @@ -35,10 +40,37 @@ env: SCCACHE_GHA_ENABLED: "true" jobs: + runner_policy: + name: Select Windows runtime runner + if: ${{ inputs.runtime_matrix != '[]' }} + runs-on: ubuntu-24.04 + permissions: + contents: read + outputs: + runner_windows: ${{ steps.policy.outputs.runner_windows }} + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - name: Select provider for the Windows runtime role + id: policy + uses: ./.github/actions/select-ci-runners + with: + event_name: ${{ github.event_name }} + original_event_name: ${{ inputs.original_event_name }} + repository: ${{ github.repository }} + head_repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.ref }} + depot_main_enabled: 'false' + depot_pr_enabled: ${{ vars.DEPOT_PR_RUNNERS_ENABLED == 'true' }} + force_hosted: ${{ inputs.force_hosted }} + windows_runtime: name: Windows runtime (${{ matrix.runtime.backend }}) + needs: runner_policy if: ${{ inputs.runtime_matrix != '[]' }} - runs-on: windows-2022 + runs-on: ${{ needs.runner_policy.outputs.runner_windows }} timeout-minutes: 90 strategy: fail-fast: ${{ inputs.fail_fast }} @@ -53,6 +85,9 @@ jobs: WINDOWS_VULKAN_SDK_VERSION: ${{ vars.VULKAN_SDK_VERSION || '1.4.328.1' }} ROCM_HIP_SDK_FILENAME: AMD-Software-PRO-Edition-25.Q3-WinSvr2022-For-HIP.exe steps: + - uses: Mesh-LLM/mesh-llm/.github/actions/audit-depot-pr-isolation@4f9899c0c381b8cc225c9be73980ffa8db525c57 + with: + original_event_name: ${{ inputs.original_event_name }} - name: Validate immutable source SHA shell: pwsh env: diff --git a/.github/workflows/depot-canary.yml b/.github/workflows/depot-canary.yml index 5285f3b22d..e8609c3b9f 100644 --- a/.github/workflows/depot-canary.yml +++ b/.github/workflows/depot-canary.yml @@ -2,12 +2,6 @@ name: Depot Runner Canary on: workflow_dispatch: - inputs: - expect_cache_hit: - description: Fail unless this is a warm Depot Cache run. - required: false - default: false - type: boolean permissions: {} @@ -51,17 +45,61 @@ jobs: echo "runner architecture mismatch: expected $expected_arch, got $actual_arch" >&2 exit 1 fi - if [[ -z "${SCCACHE_WEBDAV_ENDPOINT:-}" ]]; then - echo "Depot Cache WebDAV endpoint was not injected" >&2 - exit 1 - fi if [[ -z "${ImageOS:-}" || -z "${ImageVersion:-}" ]]; then echo "Depot runner image identity was not injected" >&2 exit 1 fi - if [[ -z "${DEPOT_CACHE_TOKEN:-}" && - -z "${SCCACHE_WEBDAV_TOKEN:-}" ]]; then - echo "Depot Cache authentication was not injected" >&2 + + # Automatic Depot Cache and Registry Actions connectivity is + # intentionally disabled. Fail closed if the runner still receives + # any Depot cache/registry authority or transparently redirects the + # GitHub cache service. Names are checked without printing values. + forbidden_names=( + DEPOT_CACHE_TOKEN + DEPOT_CACHE_URL + DEPOT_CACHE_API_URL + DEPOT_TOKEN + DEPOT_REGISTRY_HOST + DEPOT_REGISTRY_URL + DEPOT_REGISTRY_PULL_TOKEN + DEPOT_REGISTRY_TOKEN + SCCACHE_WEBDAV_ENDPOINT + SCCACHE_WEBDAV_TOKEN + SCCACHE_WEBDAV_USERNAME + SCCACHE_WEBDAV_PASSWORD + ) + for name in "${forbidden_names[@]}"; do + if [[ -n "${!name:-}" ]]; then + echo "forbidden Depot cache/registry setting was injected: $name" >&2 + exit 1 + fi + done + actions_cache_url="${ACTIONS_CACHE_URL:-}" + actions_results_url="${ACTIONS_RESULTS_URL:-}" + if [[ "${actions_cache_url,,}" == *depot.dev* || + "${actions_results_url,,}" == *depot.dev* ]]; then + echo "GitHub Actions cache was transparently redirected to Depot" >&2 + exit 1 + fi + for endpoint_name in ACTIONS_CACHE_URL ACTIONS_RESULTS_URL; do + endpoint="${!endpoint_name:-}" + endpoint_lower="${endpoint,,}" + if [[ ! "$endpoint_lower" =~ ^https://([a-z0-9-]+\.)*actions\.githubusercontent\.com(:[0-9]+)?([/?#]|$) ]]; then + echo "GitHub Actions endpoint is missing or not GitHub-owned: $endpoint_name" >&2 + exit 1 + fi + done + docker_auth_config="${DOCKER_AUTH_CONFIG:-}" + if [[ "${docker_auth_config,,}" == *depot.dev* ]]; then + echo "Docker registry authentication points at Depot" >&2 + exit 1 + fi + docker_home="${HOME:-}" + docker_config="${DOCKER_CONFIG:-$docker_home/.docker}/config.json" + if [[ -f "$docker_config" ]] && + grep -Eiq '(^|[^[:alnum:]])(registry\.)?depot\.dev([^[:alnum:]]|$)' \ + "$docker_config"; then + echo "Docker config contains Depot registry authentication" >&2 exit 1 fi @@ -75,98 +113,7 @@ jobs: printf '| memory KiB | %s |\n' "$(awk '/MemTotal/ { print $2 }' /proc/meminfo)" printf '| workspace free KiB | %s |\n' "$(df -Pk "$GITHUB_WORKSPACE" | awk 'NR == 2 { print $4 }')" printf '| runner image | %s %s |\n' "$ImageOS" "$ImageVersion" - echo "| Depot cache endpoint injected | yes |" - echo "| Depot cache authentication injected | yes |" + echo "| Depot cache credentials/endpoints injected | no |" + echo "| Depot registry credentials/config injected | no |" + echo "| GitHub cache transparently redirected to Depot | no |" } >> "$GITHUB_STEP_SUMMARY" - - - name: Install sccache - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 - - - name: Exercise authenticated Depot sccache - shell: bash - env: - EXPECT_CACHE_HIT: ${{ inputs.expect_cache_hit }} - run: | - set -euo pipefail - - if [[ -z "${SCCACHE_WEBDAV_TOKEN:-}" ]]; then - export SCCACHE_WEBDAV_TOKEN="$DEPOT_CACHE_TOKEN" - fi - if sccache --stop-server >/dev/null 2>&1; then - : - fi - sccache --zero-stats - mkdir -p .depot-sccache-canary - printf 'int depot_sccache_probe(void) { return 42; }\n' \ - > .depot-sccache-canary/probe.c - sccache cc \ - -c .depot-sccache-canary/probe.c \ - -o .depot-sccache-canary/probe.o - sccache --show-stats --stats-format json \ - > "$RUNNER_TEMP/depot-sccache-stats.json" - - python3 - \ - "$RUNNER_TEMP/depot-sccache-stats.json" \ - "$EXPECT_CACHE_HIT" \ - "$GITHUB_STEP_SUMMARY" <<'PY' - import json - import sys - - stats_path, expect_cache_hit, summary_path = sys.argv[1:] - with open(stats_path, encoding="utf-8") as handle: - payload = json.load(handle) - - location = str(payload.get("cache_location", "")).lower() - if "webdav" not in location and "depot" not in location: - raise SystemExit("sccache did not select the Depot WebDAV backend") - - stats = payload["stats"] - hits = sum(stats["cache_hits"]["counts"].values()) - misses = sum(stats["cache_misses"]["counts"].values()) - cache_errors = sum(stats["cache_errors"]["counts"].values()) - cache_errors += stats["cache_read_errors"] - cache_errors += stats["cache_write_errors"] - writes = stats["cache_writes"] - - if cache_errors: - raise SystemExit( - f"Depot sccache reported {cache_errors} cache errors" - ) - if not hits and not writes: - raise SystemExit( - "Depot sccache produced neither a cache hit nor a cache write" - ) - if expect_cache_hit == "true" and not hits: - raise SystemExit("expected a warm Depot sccache hit") - - with open(summary_path, "a", encoding="utf-8") as summary: - summary.write(f"| sccache hits | {hits} |\n") - summary.write(f"| sccache misses | {misses} |\n") - summary.write(f"| sccache writes | {writes} |\n") - PY - - - name: Restore Depot cache probe - id: cache - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 - with: - path: .depot-canary-cache - # Keep the complete label between fixed delimiters so a shorter - # runner label cannot prefix-match a sibling size. - key: depot-runner-canary-v2-exact-${{ matrix.runner }}-probe - - - name: Record cache probe - shell: bash - env: - CACHE_HIT: ${{ steps.cache.outputs.cache-hit }} - EXPECT_CACHE_HIT: ${{ inputs.expect_cache_hit }} - run: | - set -euo pipefail - - if [[ "$EXPECT_CACHE_HIT" == "true" && "$CACHE_HIT" != "true" ]]; then - echo "expected a warm Depot Cache hit for ${{ matrix.runner }}" >&2 - exit 1 - fi - mkdir -p .depot-canary-cache - printf 'Depot cache canary\n' > .depot-canary-cache/probe.txt - printf '| cache hit | %s |\n' "${CACHE_HIT:-false}" \ - >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/native-sdk-artifact.yml b/.github/workflows/native-sdk-artifact.yml index 0f41a4d285..ddb9400622 100644 --- a/.github/workflows/native-sdk-artifact.yml +++ b/.github/workflows/native-sdk-artifact.yml @@ -8,6 +8,16 @@ on: required: false default: '' type: string + original_event_name: + description: Original event used for protected runner and cache policy. + required: false + default: '' + type: string + force_hosted: + description: Force GitHub-hosted placement for runner-contract changes. + required: false + default: false + type: boolean backend: description: Native SDK backend. required: true @@ -88,74 +98,78 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 5 permissions: - contents: none + contents: read outputs: - runner: ${{ steps.policy.outputs.runner }} - allow_depot_remote_cache: ${{ steps.policy.outputs.allow_depot_remote_cache }} + runner: ${{ steps.resolve.outputs.runner }} + allow_depot_remote_cache: ${{ steps.resolve.outputs.allow_depot_remote_cache }} steps: - - name: Derive protected runner policy + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - name: Select provider for the native SDK role id: policy + uses: ./.github/actions/select-ci-runners + with: + event_name: ${{ github.event_name }} + original_event_name: ${{ inputs.original_event_name }} + repository: ${{ github.repository }} + head_repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.ref }} + depot_main_enabled: ${{ vars.DEPOT_RUNNERS_ENABLED == 'true' }} + depot_pr_enabled: ${{ vars.DEPOT_PR_RUNNERS_ENABLED == 'true' }} + force_hosted: ${{ inputs.force_hosted }} + manual_use_depot: ${{ inputs.use_depot }} + - name: Resolve runner size and target + id: resolve shell: bash env: - POLICY_REPOSITORY: ${{ github.repository }} - POLICY_REF: ${{ github.ref }} - POLICY_EVENT_NAME: ${{ github.event.inputs.original_event_name || github.event_name }} - POLICY_DEPOT_ENABLED: ${{ vars.DEPOT_RUNNERS_ENABLED == 'true' }} - POLICY_MANUAL_USE_DEPOT: ${{ inputs.use_depot }} - POLICY_TARGET: ${{ inputs.target }} - POLICY_RUNNER_SIZE: ${{ inputs.runner_size }} + TARGET: ${{ inputs.target }} + RUNNER_SIZE: ${{ inputs.runner_size }} + POLICY_EVENT_NAME: ${{ github.event_name }} + ALLOW_DEPOT_REMOTE_CACHE: ${{ steps.policy.outputs.allow_depot_remote_cache }} + RUNNER_DEFAULT: ${{ steps.policy.outputs.runner }} + RUNNER_4: ${{ steps.policy.outputs.runner_4 }} + RUNNER_8: ${{ steps.policy.outputs.runner_8 }} + RUNNER_16: ${{ steps.policy.outputs.runner_16 }} + RUNNER_ARM: ${{ steps.policy.outputs.runner_arm }} + RUNNER_ARM_4: ${{ steps.policy.outputs.runner_arm_4 }} + RUNNER_ARM_8: ${{ steps.policy.outputs.runner_arm_8 }} + RUNNER_ARM_16: ${{ steps.policy.outputs.runner_arm_16 }} + RUNNER_MACOS: ${{ steps.policy.outputs.runner_macos }} run: | set -euo pipefail - - case "$POLICY_RUNNER_SIZE" in - default|4|8|16) ;; - *) - echo "runner_size must be one of: default, 4, 8, 16" >&2 - exit 1 - ;; + case "$RUNNER_SIZE" in + default) runner="$RUNNER_DEFAULT" ;; + 4) runner="$RUNNER_4" ;; + 8) runner="$RUNNER_8" ;; + 16) runner="$RUNNER_16" ;; + *) echo "runner_size must be one of: default, 4, 8, 16" >&2; exit 1 ;; esac - - case "$POLICY_TARGET" in - x86_64-unknown-linux-gnu) - hosted_runner=ubuntu-24.04 - depot_runner=depot-ubuntu-24.04 - ;; + case "$TARGET" in + x86_64-unknown-linux-gnu) ;; aarch64-unknown-linux-gnu) - hosted_runner=ubuntu-24.04-arm - depot_runner=depot-ubuntu-24.04-arm + case "$RUNNER_SIZE" in + default) runner="$RUNNER_ARM" ;; + 4) runner="$RUNNER_ARM_4" ;; + 8) runner="$RUNNER_ARM_8" ;; + 16) runner="$RUNNER_ARM_16" ;; + esac ;; aarch64-apple-darwin) - hosted_runner=macos-15 - depot_runner= - ;; - *) - echo "unsupported native SDK producer target: $POLICY_TARGET" >&2 - exit 1 + # Preserve the existing trusted-main provider. Only a direct PR + # may consume the PR-specific Depot macOS selection. + if [[ "$POLICY_EVENT_NAME" == "pull_request" ]]; then + runner="$RUNNER_MACOS" + else + runner=macos-15 + fi + ALLOW_DEPOT_REMOTE_CACHE=false ;; + *) echo "unsupported native SDK producer target: $TARGET" >&2; exit 1 ;; esac - - allow_depot_remote_cache=false - runner=$hosted_runner - if [[ "$POLICY_REPOSITORY" == "Mesh-LLM/mesh-llm" && - "$POLICY_REF" == "refs/heads/main" && - ( "$POLICY_EVENT_NAME" == "push" || - "$POLICY_EVENT_NAME" == "workflow_dispatch" ) && - ( "$POLICY_DEPOT_ENABLED" == "true" || - ( "$POLICY_EVENT_NAME" == "workflow_dispatch" && - "$POLICY_MANUAL_USE_DEPOT" == "true" ) ) && - -n "$depot_runner" ]]; then - suffix= - if [[ "$POLICY_RUNNER_SIZE" != "default" ]]; then - suffix="-$POLICY_RUNNER_SIZE" - fi - runner="${depot_runner}${suffix}" - allow_depot_remote_cache=true - fi - - { - echo "runner=$runner" - echo "allow_depot_remote_cache=$allow_depot_remote_cache" - } >> "$GITHUB_OUTPUT" + echo "runner=$runner" >> "$GITHUB_OUTPUT" + echo "allow_depot_remote_cache=$ALLOW_DEPOT_REMOTE_CACHE" >> "$GITHUB_OUTPUT" produce_linux_static_abi: needs: runner_policy @@ -166,6 +180,8 @@ jobs: packages: read uses: ./.github/workflows/static-abi-artifact.yml with: + original_event_name: ${{ inputs.original_event_name }} + force_hosted: ${{ inputs.force_hosted }} backend: ${{ inputs.backend }} target: ${{ inputs.target }} artifact_name: ${{ inputs.static_abi_artifact_name }} @@ -185,15 +201,15 @@ jobs: packages: read container: image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d - credentials: - username: ${{ github.actor }} - password: ${{ github.token }} defaults: run: shell: bash env: LLAMA_STAGE_BUILD_DIR: .deps/llama.cpp/build-stage-abi-static steps: + - uses: Mesh-LLM/mesh-llm/.github/actions/audit-depot-pr-isolation@4f9899c0c381b8cc225c9be73980ffa8db525c57 + with: + original_event_name: ${{ inputs.original_event_name }} - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: ref: ${{ inputs.source_sha || github.sha }} @@ -295,6 +311,9 @@ jobs: permissions: contents: read steps: + - uses: Mesh-LLM/mesh-llm/.github/actions/audit-depot-pr-isolation@4f9899c0c381b8cc225c9be73980ffa8db525c57 + with: + original_event_name: ${{ inputs.original_event_name }} - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: ref: ${{ inputs.source_sha || github.sha }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4397f73ac0..34bb0c2bb2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -140,8 +140,12 @@ jobs: uses: ./.github/actions/select-ci-runners with: event_name: ${{ github.event_name }} + original_event_name: '' + repository: ${{ github.repository }} + head_repository: ${{ github.event.pull_request.head.repo.full_name }} ref: ${{ github.ref }} depot_main_enabled: ${{ vars.DEPOT_RUNNERS_ENABLED == 'true' }} + depot_pr_enabled: 'false' manual_use_depot: ${{ inputs.use_depot == true }} build: @@ -348,6 +352,8 @@ jobs: artifact_suffix: linux-aarch64-cpu uses: ./.github/workflows/native-sdk-artifact.yml with: + original_event_name: ${{ github.event_name }} + force_hosted: false backend: ${{ matrix.backend }} target: ${{ matrix.target }} profile: release @@ -719,6 +725,8 @@ jobs: contents: read uses: ./.github/workflows/swift-sdk-artifact.yml with: + original_event_name: ${{ github.event_name }} + force_hosted: false mode: full artifact_name: release-swift-sdk release_tag: ${{ needs.metadata.outputs.tag }} diff --git a/.github/workflows/static-abi-artifact.yml b/.github/workflows/static-abi-artifact.yml index 696d6d3858..804eb9b98d 100644 --- a/.github/workflows/static-abi-artifact.yml +++ b/.github/workflows/static-abi-artifact.yml @@ -12,6 +12,16 @@ on: required: false default: '' type: string + original_event_name: + description: Original event used for protected runner and cache policy. + required: false + default: '' + type: string + force_hosted: + description: Force GitHub-hosted placement for runner-contract changes. + required: false + default: false + type: boolean backend: description: Static llama ABI backend. required: true @@ -54,69 +64,64 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 5 permissions: - contents: none + contents: read outputs: - runner: ${{ steps.policy.outputs.runner }} + runner: ${{ steps.resolve.outputs.runner }} allow_depot_remote_cache: ${{ steps.policy.outputs.allow_depot_remote_cache }} steps: - - name: Derive protected runner policy + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - name: Select provider for the static ABI role id: policy + uses: ./.github/actions/select-ci-runners + with: + event_name: ${{ github.event_name }} + original_event_name: ${{ inputs.original_event_name }} + repository: ${{ github.repository }} + head_repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.ref }} + depot_main_enabled: ${{ vars.DEPOT_RUNNERS_ENABLED == 'true' }} + depot_pr_enabled: ${{ vars.DEPOT_PR_RUNNERS_ENABLED == 'true' }} + force_hosted: ${{ inputs.force_hosted }} + manual_use_depot: ${{ inputs.use_depot }} + - name: Resolve runner size and target + id: resolve shell: bash env: - POLICY_REPOSITORY: ${{ github.repository }} - POLICY_REF: ${{ github.ref }} - POLICY_EVENT_NAME: ${{ github.event.inputs.original_event_name || github.event_name }} - POLICY_DEPOT_ENABLED: ${{ vars.DEPOT_RUNNERS_ENABLED == 'true' }} - POLICY_MANUAL_USE_DEPOT: ${{ inputs.use_depot }} - POLICY_TARGET: ${{ inputs.target }} - POLICY_RUNNER_SIZE: ${{ inputs.runner_size }} + TARGET: ${{ inputs.target }} + RUNNER_SIZE: ${{ inputs.runner_size }} + RUNNER_DEFAULT: ${{ steps.policy.outputs.runner }} + RUNNER_4: ${{ steps.policy.outputs.runner_4 }} + RUNNER_8: ${{ steps.policy.outputs.runner_8 }} + RUNNER_16: ${{ steps.policy.outputs.runner_16 }} + RUNNER_ARM: ${{ steps.policy.outputs.runner_arm }} + RUNNER_ARM_4: ${{ steps.policy.outputs.runner_arm_4 }} + RUNNER_ARM_8: ${{ steps.policy.outputs.runner_arm_8 }} + RUNNER_ARM_16: ${{ steps.policy.outputs.runner_arm_16 }} run: | set -euo pipefail - - case "$POLICY_RUNNER_SIZE" in - default|4|8|16) ;; - *) - echo "runner_size must be one of: default, 4, 8, 16" >&2 - exit 1 - ;; + case "$RUNNER_SIZE" in + default) runner="$RUNNER_DEFAULT" ;; + 4) runner="$RUNNER_4" ;; + 8) runner="$RUNNER_8" ;; + 16) runner="$RUNNER_16" ;; + *) echo "runner_size must be one of: default, 4, 8, 16" >&2; exit 1 ;; esac - - case "$POLICY_TARGET" in - x86_64-unknown-linux-gnu) - hosted_runner=ubuntu-24.04 - depot_runner=depot-ubuntu-24.04 - ;; + case "$TARGET" in + x86_64-unknown-linux-gnu) ;; aarch64-unknown-linux-gnu) - hosted_runner=ubuntu-24.04-arm - depot_runner=depot-ubuntu-24.04-arm - ;; - *) - echo "unsupported static ABI producer target: $POLICY_TARGET" >&2 - exit 1 + case "$RUNNER_SIZE" in + default) runner="$RUNNER_ARM" ;; + 4) runner="$RUNNER_ARM_4" ;; + 8) runner="$RUNNER_ARM_8" ;; + 16) runner="$RUNNER_ARM_16" ;; + esac ;; + *) echo "unsupported static ABI producer target: $TARGET" >&2; exit 1 ;; esac - - allow_depot_remote_cache=false - runner=$hosted_runner - if [[ "$POLICY_REPOSITORY" == "Mesh-LLM/mesh-llm" && - "$POLICY_REF" == "refs/heads/main" && - ( "$POLICY_EVENT_NAME" == "push" || - "$POLICY_EVENT_NAME" == "workflow_dispatch" ) && - ( "$POLICY_DEPOT_ENABLED" == "true" || - ( "$POLICY_EVENT_NAME" == "workflow_dispatch" && - "$POLICY_MANUAL_USE_DEPOT" == "true" ) ) ]]; then - suffix= - if [[ "$POLICY_RUNNER_SIZE" != "default" ]]; then - suffix="-$POLICY_RUNNER_SIZE" - fi - runner="${depot_runner}${suffix}" - allow_depot_remote_cache=true - fi - - { - echo "runner=$runner" - echo "allow_depot_remote_cache=$allow_depot_remote_cache" - } >> "$GITHUB_OUTPUT" + echo "runner=$runner" >> "$GITHUB_OUTPUT" static_abi_artifact: needs: runner_policy @@ -130,9 +135,6 @@ jobs: toolchain_epoch: ${{ steps.native_toolchain.outputs.epoch }} container: image: ghcr.io/mesh-llm/mesh-llm-cuda-runner@sha256:8d93de6ba30173e825a16fdecf011f9c632edc6e1259df7289e491b0a05f829d - credentials: - username: ${{ github.actor }} - password: ${{ github.token }} defaults: run: shell: bash @@ -142,6 +144,9 @@ jobs: MESH_LLM_REQUIRE_SCCACHE: "1" SCCACHE_GHA_ENABLED: "true" steps: + - uses: Mesh-LLM/mesh-llm/.github/actions/audit-depot-pr-isolation@4f9899c0c381b8cc225c9be73980ffa8db525c57 + with: + original_event_name: ${{ inputs.original_event_name }} - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: ref: ${{ inputs.source_sha || github.sha }} diff --git a/.github/workflows/swift-sdk-artifact.yml b/.github/workflows/swift-sdk-artifact.yml index 316003c339..b77a917858 100644 --- a/.github/workflows/swift-sdk-artifact.yml +++ b/.github/workflows/swift-sdk-artifact.yml @@ -8,6 +8,16 @@ on: required: false default: '' type: string + original_event_name: + description: Original event used for protected runner and cache policy. + required: false + default: '' + type: string + force_hosted: + description: Force GitHub-hosted placement for runner-contract changes. + required: false + default: false + type: boolean mode: description: "XCFramework coverage: host-only for PR iteration or full for main/release." required: true @@ -48,9 +58,35 @@ on: type: number jobs: + runner_policy: + name: Select Swift SDK runner + runs-on: ubuntu-24.04 + permissions: + contents: read + outputs: + runner_macos: ${{ steps.policy.outputs.runner_macos }} + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - name: Select provider for the Swift SDK role + id: policy + uses: ./.github/actions/select-ci-runners + with: + event_name: ${{ github.event_name }} + original_event_name: ${{ inputs.original_event_name }} + repository: ${{ github.repository }} + head_repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.ref }} + depot_main_enabled: 'false' + depot_pr_enabled: ${{ vars.DEPOT_PR_RUNNERS_ENABLED == 'true' }} + force_hosted: ${{ inputs.force_hosted }} + swift_sdk_artifact: name: Swift SDK input (${{ inputs.mode }}) - runs-on: macos-15 + needs: runner_policy + runs-on: ${{ needs.runner_policy.outputs.runner_macos }} timeout-minutes: ${{ inputs.timeout_minutes }} permissions: contents: read @@ -64,6 +100,9 @@ jobs: SCCACHE_GHA_ENABLED: "true" SCCACHE_GHA_RW_MODE: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target' || github.event.inputs.original_event_name == 'pull_request' || github.event.inputs.original_event_name == 'pull_request_target') && 'READ_ONLY' || 'READ_WRITE' }} steps: + - uses: Mesh-LLM/mesh-llm/.github/actions/audit-depot-pr-isolation@4f9899c0c381b8cc225c9be73980ffa8db525c57 + with: + original_event_name: ${{ inputs.original_event_name }} - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 with: ref: ${{ inputs.source_sha || github.sha }} diff --git a/.omo/specs/pr-ci-optimization.md b/.omo/specs/pr-ci-optimization.md index b9b2c1eda7..a28a764631 100644 --- a/.omo/specs/pr-ci-optimization.md +++ b/.omo/specs/pr-ci-optimization.md @@ -199,10 +199,14 @@ scope for same-PR reruns. Website owns the single pnpm publisher and its npm store cache, while platform UI producers are restore-only for the shared pnpm key. Artifacts remain run-scoped correctness inputs, never rerun caches. -scripts/collect-ci-metrics.py is the read-only measurement tool. Timing -experiments must use a new worktree from main. Keep queue, dependency wait, -execution and wall-clock measurements separate; retain raw evidence outside -authoritative topology docs. Capacity is not the optimization. +scripts/collect-ci-metrics.py is the read-only measurement tool. Schema-v3 +reports keep queue, measured dependency wait, execution and wall-clock timing +separate; they also record provider/OS/architecture/role/size, sample counts, +runner-minutes, cancellation, peak workers and capacity contamination. Use +`--compare-input` for provider-separated historical PR cohort comparisons. +Its date-independent queue p95 heuristics are measurement gates, not dated run +conclusions. Timing experiments must use a new worktree from main. Keep raw +evidence outside authoritative topology docs. Capacity is not the optimization. ## Required summary @@ -239,13 +243,17 @@ Before a PR Depot path is enabled, an administrator must prove: 4. the ephemeral runner group is restricted to this repository and exact protected default-branch runner-owning workflow refs; 5. CI-control changes force the GitHub path and rollback is tested; -6. provider parity passes on comparable non-CI-change PRs. +6. provider parity passes on comparable non-CI-change PRs, using + provider-separated queue, execution and critical-path metrics. The investigation and canary are defined in ci/DEPOT_MIGRATION.md. Do not change Depot settings or runner groups in this graph PR. Start a later rollout with remote cache disabled, then canary one non-secret Linux slice, one Rust -test slice and the selected Linux product graph. Keep credential-bearing, -macOS, Windows and hardware work hosted. +test slice and the selected Linux product graph before expanding to equivalent +Depot macOS 15 and Windows 2022 build/test rows. Keep control-plane +planning/required summaries, credential-bearing smokes, `gpu-nvidia` hardware, +and any Intel macOS row without a Depot-equivalent on their approved hosted +providers. ## Extension pattern @@ -280,4 +288,10 @@ the relevant xtask repo-consistency checks. is included in this implementation PR. Required-check migration and Depot PR enablement are separate follow-up changes -after these acceptance cases pass. +after these acceptance cases pass. The final PR-Depot contract preserves the +five native PR checks and every selected row's commands, build profile, +artifacts, tests, summaries, fail-fast behavior and producer/consumer edges. +It is not a direct runner-label replacement: protected workflow refs, +cache authority, source-SHA checkout, fork/no-secret behavior, CI-control +fallback and provider exceptions must all be enforced by the owning runner +policy. diff --git a/ci/DEPOT_MIGRATION.md b/ci/DEPOT_MIGRATION.md index f619033063..8b8fbcc509 100644 --- a/ci/DEPOT_MIGRATION.md +++ b/ci/DEPOT_MIGRATION.md @@ -1,7 +1,11 @@ # Depot runner transition Status: trusted-main support exists; pull-request execution is future work and -is disabled. +is disabled. Automatic Depot Cache and Registry Actions connectivity are +admin-verified off, and the Depot runner group is admin-verified restricted to +this repository and the protected workflow allowlist. The checked-in graph and +collector define the remaining branch/main and same-repository/fork canaries; +those runtime checks still block PR activation. The complete PR/main composition design is in `.omo/specs/pr-ci-optimization.md`. This document contains only the durable @@ -17,19 +21,42 @@ Depot is a placement provider, not a different CI graph. - `.github/actions/select-ci-runners` owns current Linux provider selection. - `DEPOT_RUNNERS_ENABLED == 'true'` permits eligible trusted `main` Linux jobs to select Depot. Every eligible role retains a GitHub-hosted fallback. -- Pull requests, feature refs, tags, credential-bearing smokes, macOS, Windows - and hardware-qualified GPU work retain their approved non-Depot placement - until separately migrated. +- `DEPOT_PR_RUNNERS_ENABLED == 'true'` is the independent same-repository PR + placement gate. It must remain absent or `false` until the branch/main + provider-parity, same-repository, and fork sentinel canaries below pass; a + missing value fails closed. +- Pull requests, feature refs, tags, credential-bearing smokes and + hardware-qualified GPU work retain their approved non-Depot placement until + the protected PR executor is separately activated. The intended executor may + cover eligible build/test jobs in Linux, Depot macOS 15 and Windows 2022 + lanes when an equivalent image/architecture exists; control-plane planning + and required summaries remain hosted, and Intel macOS without an equivalent + remains hosted. - Callers never provide a raw Depot label or a separate remote-cache permission. Runner and cache policy come from one event-derived decision. Disabling Depot must change placement only. It must not change plan membership, commands, artifacts, smoke coverage or required checks. +The PR end state is therefore a protected execution-policy change, not a +`runs-on` label swap. The five native PR entrypoints and their matching +protected reusable lanes remain intact. A selected PR slice keeps its main +commands, profile, artifact identities, tests, `needs` edges, summaries, +fail-fast profile and required-check result. Only the event-derived provider, +cache mode and ephemeral runner allocation may differ. The runner-owning +workflow checks out the immutable PR SHA with `persist-credentials: false`, +receives no PR secrets or registry/cache credentials, and forces the hosted +path for CI-control/workflow/policy changes. The planner-owned +`signals.runner_contract_required` value is passed as `force_hosted` through +every protected lane and runner-owning slice, and the centralized selector +requires it to be false before enabling Depot. Control-plane planning/required +summaries, credential-bearing smoke, `gpu-nvidia` hardware, and any Intel macOS +row without a Depot-equivalent remain their approved provider exceptions. + ## Required migration after the composable graph lands The protected controller and split lane workflows change which workflow files -own and call eligible Linux jobs. The first `main` push after that change lands +own and call eligible jobs. The first `main` push after that change lands can select Depot immediately when `DEPOT_RUNNERS_ENABLED` is already `true`. GitHub does not fall back to a hosted runner when a selected Depot label is blocked by runner-group policy; the job remains queued. Complete this sequence @@ -60,6 +87,16 @@ when landing the composable graph: 6. Roll back by setting `DEPOT_RUNNERS_ENABLED=false`. Re-run the same profile and verify that the identical plan executes on GitHub-hosted Linux workers. +The current administrative posture has been verified outside the repository: +automatic Actions Cache connectivity to Depot is disabled, automatic Registry +Actions authentication is disabled, and the Depot runner group is restricted +to `Mesh-LLM/mesh-llm` with exact protected workflow refs. The repository token +cannot independently inspect organization runner-group settings through the +API (403), so this remains external activation state rather than checked-in +proof. The settings remove automatic credential authority; they do not prove +that a protected main build, same-repository PR, or fork PR receives the +intended provider, cache mode, or rollback behavior. + Do not migrate required checks, enable Depot for PR content, or change cache isolation during this provider rollout. Those are independent changes with their own acceptance gates below. @@ -78,26 +115,45 @@ Mesh-LLM/mesh-llm/.github/workflows/ci-control.yml@refs/heads/main Mesh-LLM/mesh-llm/.github/workflows/ci-quality-lane.yml@refs/heads/main Mesh-LLM/mesh-llm/.github/workflows/ci-linux-lane.yml@refs/heads/main Mesh-LLM/mesh-llm/.github/workflows/ci-quality-slice.yml@refs/heads/main +Mesh-LLM/mesh-llm/.github/workflows/ci-web-slice.yml@refs/heads/main +Mesh-LLM/mesh-llm/.github/workflows/ci-ui-artifact-slice.yml@refs/heads/main Mesh-LLM/mesh-llm/.github/workflows/ci-linux-host-slice.yml@refs/heads/main Mesh-LLM/mesh-llm/.github/workflows/ci-linux-runtime-slice.yml@refs/heads/main +Mesh-LLM/mesh-llm/.github/workflows/ci-linux-product-slice.yml@refs/heads/main +Mesh-LLM/mesh-llm/.github/workflows/ci-rust-tests-slice.yml@refs/heads/main +Mesh-LLM/mesh-llm/.github/workflows/ci-macos-host-slice.yml@refs/heads/main +Mesh-LLM/mesh-llm/.github/workflows/ci-macos-runtime-slice.yml@refs/heads/main +Mesh-LLM/mesh-llm/.github/workflows/ci-macos-product-slice.yml@refs/heads/main +Mesh-LLM/mesh-llm/.github/workflows/ci-windows-host-slice.yml@refs/heads/main +Mesh-LLM/mesh-llm/.github/workflows/ci-windows-runtime-slice.yml@refs/heads/main +Mesh-LLM/mesh-llm/.github/workflows/ci-windows-product-slice.yml@refs/heads/main +Mesh-LLM/mesh-llm/.github/workflows/ci-platform-checks-slice.yml@refs/heads/main Mesh-LLM/mesh-llm/.github/workflows/depot-canary.yml@refs/heads/main Mesh-LLM/mesh-llm/.github/workflows/depot-registry-canary.yml@refs/heads/main Mesh-LLM/mesh-llm/.github/workflows/native-sdk-artifact.yml@refs/heads/main Mesh-LLM/mesh-llm/.github/workflows/release.yml@refs/heads/main Mesh-LLM/mesh-llm/.github/workflows/static-abi-artifact.yml@refs/heads/main +Mesh-LLM/mesh-llm/.github/workflows/swift-sdk-artifact.yml@refs/heads/main ``` Credential-bearing `hf-download-smoke.yml`, `smoke.yml`, `scripted-binary-smoke.yml`, and `sdk-smoke.yml` are not in the allowlist and -remain GitHub-hosted. `swift-sdk-artifact.yml` remains on `macos-15`. +remain GitHub-hosted. `swift-sdk-artifact.yml` must be in the protected workflow +allowlist because it directly owns eligible PR Depot placement; its internal +main gate remains false, so release/main Swift production stays on `macos-15`. -Verify the live runner-group repository and selected-workflow restrictions with -organization-admin authority before any rollout. A repository token returning -`403` is unverified state, not proof of a safe configuration. +The runner-group boundary above is an external administrative prerequisite. +Re-verify it with organization-admin authority if the group, repository, or +workflow allowlist changes. A repository token returning `403` is not an +independent proof of the configuration. -## Why PRs cannot use Depot today +## Why PR canaries remain pending -Cache isolation is the prerequisite for any future PR execution on Depot. +Disabling automatic Depot Cache and Registry Actions connectivity removes the +repository-wide credential path that blocked the first canary. It does not by +itself prove provider parity, branch/main cache behavior, or that a same- +repository or fork PR cannot publish an entry later restored by trusted main. +Those runtime isolation checks remain prerequisites for PR execution on Depot. Depot documents these GitHub Actions cache properties: @@ -123,28 +179,26 @@ Official references: - [GitHub dependency cache isolation](https://docs.github.com/en/actions/reference/workflows-and-actions/dependency-caching) - [GitHub runner-group API](https://docs.github.com/en/rest/actions/self-hosted-runner-groups) -## Required future investigation +## Cache isolation and remaining registry investigation Depot documents an organization setting named **Allow Actions jobs to -automatically connect to Depot Cache**. Before using Depot for PRs, an -organization administrator must disable or isolate automatic cache connectivity -in a controlled canary and prove the resulting behavior. - -Required questions: - -1. Does disabling the setting remove `DEPOT_CACHE_TOKEN`, - `SCCACHE_WEBDAV_*`, tool-specific cache configuration and transparent - GitHub-cache API redirection from the PR job? -2. Does `actions/cache` then use GitHub's native branch-isolated cache, or must - all remote cache actions be disabled on Depot PR runners? -3. Can Depot provide a short-lived cache token scoped to workflow/ref and - read/write policy? Cache authentication does not currently document project - tokens as supported. -4. If automatic connectivity is disabled organization-wide, how will trusted - main cache safely: GitHub native cache, no remote compiler cache, an isolated - Depot tenant/runner group, or a Depot-supported per-workflow policy? -5. Can the runner group be verified as exact-workflow restricted by an - organization administrator? +automatically connect to Depot Cache**. That setting and automatic Registry +Actions authentication are now admin-verified off. The negative +`depot-canary.yml` contract checks that this posture reaches fresh runners +without Depot cache, WebDAV, registry, or transparent GitHub-cache authority. + +The remaining runtime questions are: + +1. Does `actions/cache` on a fresh Depot runner use GitHub's native + branch-isolated cache, or must all cache actions remain disabled for PR jobs? +2. Does a trusted main build retain its intended cache behavior with automatic + Depot connectivity disabled, or should it use GitHub native cache only? +3. Can a same-repository PR and a fork PR both run the protected canary with no + Depot cache/registry authority and no entry that trusted main later restores? +4. Does provider parity hold for the same checked plan, commands, artifacts, + and required results on GitHub and Depot? +5. Does the restricted runner group continue to allow only the exact protected + workflow refs after the PR canary is enabled? Do not introduce a long-lived Depot organization token without a separate security review and explicit authorization. @@ -168,6 +222,12 @@ protected workflows must: - run on ephemeral Depot instances; - be exact selected workflows allowed by the runner group. +After every acceptance canary passes, activate PR placement with the repository +variable `DEPOT_PR_RUNNERS_ENABLED=true`. Roll back immediately by setting it to +`false` (or deleting it) and rerun the identical PR plan; this changes provider +placement only and must leave commands, matrices, artifacts, and required checks +unchanged. `DEPOT_RUNNERS_ENABLED` remains the separate trusted-main gate. + CI workflow, action, planner, ownership, runner and cache-policy changes must remain on the local GitHub-hosted path. A PR may not modify the workflow that grants its own Depot placement. @@ -208,3 +268,23 @@ credential-bearing, macOS, Windows and hardware work on their existing runners. Depot PR activation must be a separate change after the composable CI graph is complete. It must not be combined with routing, required-check, artifact or branch-protection migration. + +## Measurement and rollback evidence + +Use `scripts/collect-ci-metrics.py` to monitor all five focused PR lanes and +their historical GitHub cohorts. Keep raw run/job JSON under `/tmp` or an issue +artifact. Schema-v3 reports separate workflow wall and queue, job runner queue, +measured dependency wait (otherwise `n/a`), execution, runner-minutes, +cancelled minutes and peak workers, grouped by provider, OS, architecture, +semantic role and Depot size. Use `--compare-input` only with matching plan +profile, selected slices, source/change class, image/toolchain epoch and cache +mode; the provider sets must be disjoint and job families common. + +The date-independent rollout signals are deterministic: fewer than three job +queue samples is `insufficient_sample`; queue p95 over 60 seconds is `hold`; +job or terminal-job queue p95 at least 300 seconds is capacity-contaminated and +`rollback`; a candidate cohort is `eligible` only when provider separation and +all other comparability checks pass. A contaminated run may prove correctness +or artifact reuse but cannot prove provider latency. Rollback changes only the +central provider gate to GitHub-hosted and reruns the same plan; it does not +change build shape. Do not place dated conclusions or raw evidence in `ci/`. diff --git a/ci/METRICS.md b/ci/METRICS.md index 60aee15cb6..5b05977e29 100644 --- a/ci/METRICS.md +++ b/ci/METRICS.md @@ -1,9 +1,10 @@ # CI measurement contract Use `scripts/collect-ci-metrics.py` for repeatable, read-only GitHub Actions -measurements. This document defines method, not a historical baseline. Store raw -observations as workflow artifacts or in the owning tracking issue; do not add -dated run conclusions to authoritative CI documentation. +measurements. The collector emits schema version 3 reports and deterministic +rollout signals; this document defines method, not a historical baseline. +Store raw observations as workflow artifacts or in the owning tracking issue; +do not add dated run conclusions to authoritative CI documentation. ## Collection @@ -41,19 +42,61 @@ python3 scripts/collect-ci-metrics.py \ --markdown-out /tmp/ci-canary-recomputed.md ``` +Build a historical PR cohort and compare a candidate cohort without mixing +provider dimensions. The input labelled `provider=github` below is a cohort +label for the report; the runner labels in the report remain the source of +truth for the actual provider. + +```bash +python3 scripts/collect-ci-metrics.py \ + --repo Mesh-LLM/mesh-llm \ + --workflow pr_linux.yml \ + --event pull_request \ + --limit 20 \ + --label cohort=historical-pr \ + --label provider=github \ + --raw-out /tmp/pr-github-history.json \ + --json-out /tmp/pr-github-history.metrics.json + +python3 scripts/collect-ci-metrics.py \ + --input /tmp/pr-depot-candidate.json \ + --label cohort=depot-candidate \ + --label provider=depot \ + --compare-input /tmp/pr-github-history.json \ + --json-out /tmp/pr-depot-comparison.metrics.json \ + --markdown-out /tmp/pr-depot-comparison.metrics.md +``` + +Use the same workflow, event, plan/profile, selected slice IDs and source +conditions for both cohorts. A single `--limit` query is not a provider filter; +inspect `jobs.by_runner` and retain the included/excluded run IDs in the +tracking artifact. Run the command separately for each focused PR entrypoint +when monitoring the complete five-lane graph. + ## Definitions - **Workflow wall time:** run `created_at` to terminal `updated_at` for a first attempt. - **Workflow queue time:** run `created_at` to `started_at` for a first attempt. - **Job queue time:** job `created_at` to `started_at`. -- **Dependency wait:** workflow/job readiness delay caused by `needs`, distinct - from runner queue. -- **Job execution:** job `started_at` to `completed_at`. +- **Dependency wait:** the interval from job `created_at` to an instrumented + dependency-ready time (or explicit `dependency_wait_seconds`), distinct from + runner queue. The standard GitHub jobs API does not expose dependency-ready + timestamps; the collector reports `n/a`, never a fabricated zero, when they + are absent. +- **Runner queue time:** `started_at` minus dependency-ready time when that + timestamp exists. Without instrumentation, the collector falls back to the + raw creation-to-start interval for compatibility and the report's + definitions identify that dependency/runner separation is unavailable. +- **Job execution:** job `started_at` to `completed_at` (`execution_seconds`, + with `duration_seconds` retained as a compatibility alias). +- **Job wall time:** job `created_at` to `completed_at`; this includes runner + queue and any measured dependency wait. - **Step execution:** step start to completion as returned by the jobs API. - **Runner-minutes:** sum of allocated job execution seconds divided by 60. - **Peak workers:** maximum overlap of started, incomplete allocated jobs, - grouped by platform, provider and runner role. + grouped by operating system, provider and semantic runner role when those + dimensions are present. - **Critical path candidate:** the dependency chain ending at the last required successful job, reconstructed from checked workflow `needs` where possible. - **Cancelled work:** execution time consumed by a run later cancelled or @@ -84,7 +127,9 @@ and a larger runner does not explain reduced queue time. Treat a run as capacity-contaminated for provider execution comparisons when job runner-queue p95 or the terminal required job's runner queue is at least five -minutes. It may still prove correctness and artifact reuse. +minutes. The report records this per cohort in `heuristics.capacity_contaminated` +and in the affected run reports. It may still prove correctness and artifact +reuse, but it must not be used as clean provider-latency evidence. ## Required observations @@ -101,6 +146,23 @@ Every PR/main graph comparison records: - cache mode, hit/miss/error counts and cache publication authority; - expected and unexpected skips. +The report's schema-v3 timing fields are deliberately separate: + +| Report path | Meaning | +| --- | --- | +| `workflow.wall_seconds` | first-attempt workflow wall p50/p90/p95 and sample count | +| `workflow.queue_seconds` | first-attempt workflow queue p50/p90/p95 | +| `workflow.dependency_wait_seconds` | measured dependency wait, or zero samples when unavailable | +| `jobs.by_runner` | provider, OS, architecture, role, runner size and timing percentiles | +| `jobs.execution_seconds` | allocated build execution timing | +| `jobs.runner_queue_seconds` | runner assignment queue, separated when dependency-ready instrumentation exists | +| `capacity` | runner-minutes, cancelled runner-minutes and peak workers | +| `heuristics` | queue thresholds, contamination and deterministic state | +| `comparison` | provider separation and candidate-vs-baseline p95 deltas when `--compare-input` is used | + +Every percentile is accompanied by `count`; a missing count or a cohort with +fewer than three job samples is not rollout evidence. + Do not infer compiler-cache behavior from workflow time. Instrument sccache jobs with machine-readable statistics, reset counters immediately before the measured build, and aggregate: @@ -140,7 +202,28 @@ Do not loosen correctness or expand fan-out merely to meet a timing target. ## Depot comparisons Depot comparisons must use the same checked plan and slice inputs as GitHub. -Record provider-specific queue and startup separately from build execution. +Record provider-specific queue and startup separately from build execution and +keep the provider cohorts disjoint. Provider comparison is scoped to build and +test executors; hosted orchestration, summaries, runner selectors, and +credentialed smoke jobs remain visible in the full report but are excluded from +the provider cohort. `compare_reports` marks a comparison `hold` when those +executor provider sets overlap, no job family or OS/architecture family is +common, either baseline or candidate has fewer than three runner-queue samples, +or the candidate queue heuristic is not eligible. This prevents hosted control +jobs or weak historical evidence from masking a provider or build-graph change. + +The built-in, date-independent heuristic constants are: + +| Signal | Rule | +| --- | --- | +| Minimum evidence | fewer than 3 `jobs.runner_queue_seconds` samples → `insufficient_sample` (checked before queue warning and capacity contamination) | +| Queue warning | with minimum evidence met, `jobs.runner_queue_seconds` p95 above 60 seconds, or unavailable → `hold` | +| Capacity contamination | `jobs.runner_queue_seconds` or terminal-job runner queue p95 at least 300 seconds → `rollback` | +| Provider cohort separation | baseline and candidate runner-provider sets must be disjoint | + +These are rollout/rollback heuristics, not historical conclusions. A maintainer +may override a state only with an issue/artifact containing comparable runs, +correctness results and an explicit reason. PR-on-Depot cache isolation is a security gate, not a performance experiment. Do not run untrusted PR content on Depot until `DEPOT_MIGRATION.md` requirements @@ -158,5 +241,11 @@ A migration report states: - correctness, artifact and cache conclusions separately; - rollback decision. +For a PR-Depot rollout, include the five focused lane reports, a comparable +historical PR cohort, and the exact plan/profile and cache mode. Keep queue, +dependency wait, execution and wall-time conclusions separate. Do not infer +cache or compiler throughput from workflow wall time, and do not treat a Depot +run as proof of cache isolation. + Do not commit raw timing JSON, dated tables, canary anecdotes or transient run URLs under `ci/`. Promote only durable methodology or policy into these docs. diff --git a/ci/ci.md b/ci/ci.md index cd9e7b6ab1..7c3bca4afb 100644 --- a/ci/ci.md +++ b/ci/ci.md @@ -212,8 +212,16 @@ platform workflow because artifacts are run-scoped; UI tests still execute only in the Website graph and host producers never rebuild the UI themselves. Timing evidence is collected read-only with `scripts/collect-ci-metrics.py`. -Do not put run-specific durations or historical conclusions in this document; -record an evidence file separately when a timing experiment is authorized. +Schema-v3 reports keep workflow wall/queue, runner queue, dependency wait, +execution, runner-minutes, cancelled runner-minutes and peak workers separate, +and group results by provider, OS, architecture, semantic runner role and +Depot size. When `--compare-input` is supplied, the report always emits a +`comparison` block. Its recommendation is `hold` when job families or +provider sets are not comparable. Deterministic queue p95 and +capacity-contamination heuristics emit `eligible`, `hold`, `rollback` or +`insufficient_sample`; they are rollout signals, not dated conclusions. Keep +raw inputs and dated reports under `/tmp` or a tracking artifact, never in +`ci/`. ### PR failure domains @@ -274,6 +282,20 @@ smokes and other hardware-qualified work retain explicit approved placement. Provider choice never changes plan membership, commands, artifacts, tests or summaries. +The intended PR-Depot end state preserves the same five entry workflows and +matching protected lane calls. After the external cache and runner-group gates +in `ci/DEPOT_MIGRATION.md` are proven, provider policy may select ephemeral +Depot for eligible build/test jobs in Linux, macOS 15 and Windows 2022 lanes +where an equivalent image/architecture exists. This is not a direct label +swap: a PR can edit checked-out workflows/actions, and Depot's automatic +cache/registry authority is repository-scoped unless administrators prove a +per-PR boundary. The owning protected workflow must remain pinned to the +reviewed default branch, derive provider and cache mode from the event/trust +policy, receive no PR secrets or registry/cache tokens, and check out the +immutable PR SHA. Control-plane planning/required summaries, credential- +bearing smokes, `gpu-nvidia` hardware work, and any Intel macOS row without a +Depot-equivalent remain on their approved providers. + ### PR cache audit and rerun behavior GitHub-hosted PR runs may restore caches from their PR merge ref and the base @@ -304,6 +326,13 @@ default-branch runner-owning workflow refs, no-secret/no-token execution and a sentinel canary are prerequisites in `ci/DEPOT_MIGRATION.md`. Do not change Depot settings or runner groups in a workflow refactor. +The external administrative posture now has automatic Depot Cache and Registry +Actions connectivity disabled and the Depot runner group restricted to this +repository and its exact protected workflow refs. The negative main canary, +branch/main provider-parity check, same-repository PR canary, fork PR canary, +and rollback evidence remain pending; those runtime checks are distinct from +the settings verification and must pass before PR placement is enabled. + ## Required extension pattern 1. Read the manage-ci skill, inventory, this file and the optimization spec. diff --git a/scripts/collect-ci-metrics.py b/scripts/collect-ci-metrics.py index 13fb76e715..e62f797ecf 100755 --- a/scripts/collect-ci-metrics.py +++ b/scripts/collect-ci-metrics.py @@ -32,6 +32,9 @@ ) ) SKIPPED = "skipped" +QUEUE_WARN_SECONDS = 60.0 +QUEUE_CONTAMINATION_SECONDS = 300.0 +MIN_HEURISTIC_SAMPLES = 3 def pick(data: dict[str, Any], *names: str, default: Any = None) -> Any: @@ -110,6 +113,16 @@ def normalize_step(raw: dict[str, Any]) -> dict[str, Any]: } +def _number(value: Any) -> float | None: + if isinstance(value, bool): + return None + try: + parsed = float(value) + except (TypeError, ValueError): + return None + return parsed if parsed >= 0 else None + + def normalize_job(raw: dict[str, Any]) -> dict[str, Any]: labels = pick(raw, "labels", "runner_labels", default=[]) raw_steps = pick(raw, "steps", default=[]) @@ -127,6 +140,32 @@ def normalize_job(raw: dict[str, Any]) -> dict[str, Any]: "created": timestamp(pick(raw, "created_at", "createdAt")), "started": timestamp(pick(raw, "started_at", "startedAt")), "completed": timestamp(pick(raw, "completed_at", "completedAt")), + # The jobs API does not expose the time at which `needs` became ready. + # Preserve an optional instrumented value without treating an absent + # value as zero dependency wait. + "dependency_ready": timestamp( + pick( + raw, + "dependency_ready_at", + "needs_completed_at", + "ready_at", + ) + ), + "dependency_wait_seconds": _number( + pick(raw, "dependency_wait_seconds", "needs_wait_seconds") + ), + "needs": ( + [str(item) for item in pick(raw, "needs", default=[])] + if isinstance(pick(raw, "needs", default=[]), list) + else [] + ), + "runner_role": pick(raw, "runner_role", "role"), + "operating_system": pick( + raw, + "operating_system", + "runner_os", + "os", + ), "url": str(pick(raw, "html_url", "url", default="")), "labels": [str(label) for label in labels] if isinstance(labels, list) @@ -161,6 +200,12 @@ def normalize_run(raw: dict[str, Any]) -> dict[str, Any]: "url": str(pick(raw, "html_url", "url", default="")), "sha": str(pick(raw, "head_sha", "headSha", default="")), "branch": str(pick(raw, "head_branch", "headBranch", default="")), + "plan_profile": pick(raw, "plan_profile", "profile"), + "plan_digest": pick(raw, "plan_digest", "planDigest"), + "change_class": pick(raw, "change_class", "changeClass"), + "runner_image": pick(raw, "runner_image", "runnerImage"), + "toolchain_epoch": pick(raw, "toolchain_epoch", "toolchainEpoch"), + "cache_mode": pick(raw, "cache_mode", "cacheMode"), "jobs": [normalize_job(job) for job in raw["jobs"]], } @@ -280,7 +325,32 @@ def fetch_runs(args: argparse.Namespace) -> list[dict[str, Any]]: return runs -def runner_dimensions(labels: list[str]) -> dict[str, str | None]: +def _canonical_os(value: Any) -> str | None: + if not isinstance(value, str) or not value.strip(): + return None + value = value.strip().lower() + if value in {"linux", "ubuntu"} or value.startswith("linux-"): + return "linux" + if value in {"macos", "mac", "darwin"} or value.startswith("macos-"): + return "macos" + if value in {"windows", "win"} or value.startswith("windows-"): + return "windows" + return value + + +def _label_dimension(labels: set[str], prefixes: tuple[str, ...]) -> str | None: + for label in sorted(labels): + for prefix in prefixes: + if label.startswith(prefix) and len(label) > len(prefix): + return label[len(prefix) :] + return None + + +def runner_dimensions( + labels: list[str], + runner_role: Any = None, + operating_system: Any = None, +) -> dict[str, str | None]: """Derive only dimensions encoded by the selected runner label. Depot labels encode architecture and vCPU size. Hosted and legacy labels @@ -289,25 +359,46 @@ def runner_dimensions(labels: list[str]) -> dict[str, str | None]: """ normalized = {label.lower() for label in labels} + role = ( + str(runner_role).strip() + if isinstance(runner_role, str) and runner_role.strip() + else _label_dimension(normalized, ("runner-role:", "role:")) + ) depot_label = next( (label for label in sorted(normalized) if label.startswith("depot-")), None, ) if depot_label: suffix = depot_label.rsplit("-", 1)[-1] + depot_platform = depot_label[len("depot-") :] + if depot_platform.startswith(("macos-", "mac-")): + depot_os = "macos" + elif depot_platform.startswith(("windows-", "win-")): + depot_os = "windows" + else: + depot_os = "linux" return { "provider": "depot", - "architecture": "arm64" if "-arm" in depot_label else "amd64", - "runner_size": suffix if suffix in {"4", "8", "16"} else "default", + "architecture": ( + "arm64" + if "-arm" in depot_label or depot_os == "macos" + else "amd64" + ), + "runner_size": ( + suffix if suffix in {"4", "8", "16", "32", "64"} else "default" + ), + "operating_system": _canonical_os(operating_system) or depot_os, + "runner_role": role, } if "mesh-llm-arm64" in normalized or "ubuntu-24.04-arm" in normalized: architecture = "arm64" + elif "macos-15-intel" in normalized: + architecture = "amd64" elif "macos-15" in normalized: architecture = "arm64" elif ( - "macos-15-intel" in normalized - or "windows-2022" in normalized + "windows-2022" in normalized or "ubuntu-24.04" in normalized or "x64" in normalized or "amd64" in normalized @@ -332,10 +423,32 @@ def runner_dimensions(labels: list[str]) -> dict[str, str | None]: "provider": provider, "architecture": architecture, "runner_size": None, + "operating_system": _canonical_os(operating_system) + or ( + "linux" + if any(label.startswith("ubuntu-") for label in normalized) + else "macos" + if any(label.startswith("macos-") for label in normalized) + else "windows" + if any(label.startswith("windows-") for label in normalized) + else None + ), + "runner_role": role, } def observation(run: dict[str, Any], job: dict[str, Any]) -> dict[str, Any]: + duration = elapsed(job["started"], job["completed"]) + dependency_wait = job["dependency_wait_seconds"] + if dependency_wait is None: + dependency_wait = elapsed(job["created"], job["dependency_ready"]) + runner_queue = elapsed( + job["dependency_ready"] or job["created"], + job["started"], + ) + dimensions = runner_dimensions( + job["labels"], job["runner_role"], job["operating_system"] + ) return { "run_id": run["id"], "run_attempt": run["attempt"], @@ -343,16 +456,38 @@ def observation(run: dict[str, Any], job: dict[str, Any]) -> dict[str, Any]: "job_id": job["id"], "job_url": job["url"], "name": job["name"], + "run_conclusion": run["conclusion"], "conclusion": job["conclusion"], - "duration_seconds": elapsed(job["started"], job["completed"]), + "created_at": job["created"].isoformat() if job["created"] else None, + "started_at": job["started"].isoformat() if job["started"] else None, + "completed_at": ( + job["completed"].isoformat() if job["completed"] else None + ), + "dependency_ready_at": ( + job["dependency_ready"].isoformat() + if job["dependency_ready"] + else None + ), + # `duration_seconds` remains the v2 name. `execution_seconds` is the + # explicit name used by cohort comparisons. + "duration_seconds": duration, + "execution_seconds": duration, + "wall_seconds": elapsed(job["created"], job["completed"]), "queue_seconds": elapsed(job["created"], job["started"]), + "runner_queue_seconds": runner_queue, + "dependency_wait_seconds": dependency_wait, + "dependency_wait_observed": dependency_wait is not None, + "capacity_contaminated": ( + runner_queue is not None + and runner_queue >= QUEUE_CONTAMINATION_SECONDS + ), "start_delay_seconds": ( elapsed(run["created"], job["started"]) if run["attempt"] == 1 else None ), "runner_labels": job["labels"], - "runner_dimensions": runner_dimensions(job["labels"]), + "runner_dimensions": dimensions, } @@ -364,6 +499,288 @@ def included(run: dict[str, Any], requested: str) -> tuple[bool, str]: return False, f"conclusion_{run['conclusion'] or 'missing'}" +def is_comparison_executor(name: str) -> bool: + """Exclude hosted orchestration and credentialed smoke jobs from provider cohorts.""" + + normalized = name.strip().lower() + if normalized in {"changes", "summary"}: + return False + if normalized.startswith(("plan ", "pr /")): + return False + if "/ ci /" in normalized or "lane plan" in normalized: + return False + if "select " in normalized and " runner" in normalized: + return False + if "runner and cache contract" in normalized: + return False + if "smoke" in normalized: + return False + return True + + +def _job_timestamp(sample: dict[str, Any], field: str) -> dt.datetime | None: + return timestamp(sample.get(field)) + + +def capacity_metrics(observations: list[dict[str, Any]]) -> dict[str, Any]: + """Summarize allocated time and bounded runner overlap. + + Job intervals are the only portable evidence available from the Actions + jobs API. Missing timestamps are omitted rather than treated as zero. + """ + + runner_seconds = sum( + sample["execution_seconds"] + for sample in observations + if sample["execution_seconds"] is not None + ) + cancelled_seconds = sum( + sample["execution_seconds"] + for sample in observations + if sample["execution_seconds"] is not None + and ( + sample["run_conclusion"] in {"cancelled", "canceled"} + or sample["conclusion"] in {"cancelled", "canceled"} + ) + ) + events: dict[ + tuple[str | None, str | None, str | None], list[tuple[dt.datetime, int]] + ] = collections.defaultdict(list) + all_events: list[tuple[dt.datetime, int]] = [] + for sample in observations: + started = _job_timestamp(sample, "started_at") + completed = _job_timestamp(sample, "completed_at") + if started is None or completed is None: + continue + dimensions = sample["runner_dimensions"] + key = ( + dimensions.get("provider"), + dimensions.get("operating_system"), + dimensions.get("runner_role"), + ) + points = ((started, 1), (completed, -1)) + events[key].extend(points) + all_events.extend(points) + + peaks: dict[str, int] = {} + for key, points in events.items(): + active = 0 + peak = 0 + # End events sort before starts at the same instant. Two adjacent jobs + # therefore do not count as concurrent workers. + for _, delta in sorted(points, key=lambda point: (point[0], point[1])): + active += delta + peak = max(peak, active) + provider, operating_system, role = key + name = "/".join(value or "unknown" for value in (provider, operating_system, role)) + peaks[name] = peak + + total_active = 0 + total_peak = 0 + # Repeat the same end-before-start sweep across every runner dimension so + # `peak_workers.total` captures simultaneous work in different groups. + for _, delta in sorted(all_events, key=lambda point: (point[0], point[1])): + total_active += delta + total_peak = max(total_peak, total_active) + + return { + "runner_minutes": round(runner_seconds / 60, 3), + "cancelled_runner_minutes": round(cancelled_seconds / 60, 3), + "timestamped_job_count": sum( + 1 + for sample in observations + if _job_timestamp(sample, "started_at") is not None + and _job_timestamp(sample, "completed_at") is not None + ), + "peak_workers": { + "total": total_peak if all_events else None, + "by_provider_os_role": dict(sorted(peaks.items())), + }, + } + + +def queue_heuristics(report: dict[str, Any]) -> dict[str, Any]: + """Return deterministic, date-independent provider rollout signals.""" + + # Without dependency-ready instrumentation runner queue falls back to the + # raw creation-to-start interval; the report still marks dependency wait as + # unavailable so this fallback cannot be mistaken for a clean decomposition. + job_queue = report["jobs"].get( + "runner_queue_seconds", report["jobs"]["queue_seconds"] + ) + terminal_queue = report["workflow"].get( + "terminal_job_runner_queue_seconds", + report["workflow"].get("terminal_job_queue_seconds", {}), + ) + queue_p95 = job_queue.get("p95") + terminal_p95 = terminal_queue.get("p95") + contaminated = any( + value is not None and value >= QUEUE_CONTAMINATION_SECONDS + for value in (queue_p95, terminal_p95) + ) + sample_count = int(job_queue.get("count") or 0) + if sample_count < MIN_HEURISTIC_SAMPLES: + state = "insufficient_sample" + elif contaminated: + state = "rollback" + elif queue_p95 is None or queue_p95 > QUEUE_WARN_SECONDS: + state = "hold" + else: + state = "eligible" + return { + "thresholds_seconds": { + "queue_warn": QUEUE_WARN_SECONDS, + "queue_capacity_contamination": QUEUE_CONTAMINATION_SECONDS, + "minimum_samples": MIN_HEURISTIC_SAMPLES, + }, + "sample_count": sample_count, + "queue_p95_seconds": queue_p95, + "terminal_job_queue_p95_seconds": terminal_p95, + "capacity_contaminated": contaminated, + "state": state, + "interpretation": { + "eligible": "queue p95 <= warn threshold and no capacity contamination", + "hold": "queue p95 exceeds warn threshold or is unavailable", + "rollback": "queue p95 reaches the capacity-contamination threshold", + "insufficient_sample": "collect at least the minimum sample count", + }[state], + } + + +def _comparison_cohort(report: dict[str, Any]) -> dict[str, Any]: + return report["jobs"].get("comparison_cohort", report["jobs"]) + + +def _providers(report: dict[str, Any]) -> set[str]: + cohort = _comparison_cohort(report) + if "providers" in cohort: + return { + str(provider) + for provider in cohort["providers"] + if provider and provider != "unknown" + } + return { + str(item["provider"]) + for item in report["jobs"].get("by_runner", []) + if item.get("provider") and item.get("provider") != "unknown" + } + + +def _runner_dimension_values(report: dict[str, Any], key: str) -> set[str]: + cohort = _comparison_cohort(report) + if "dimensions" in cohort: + return {str(value) for value in cohort["dimensions"].get(key, []) if value} + return { + str(item[key]) + for item in report["jobs"].get("by_runner", []) + if item.get(key) + } + + +def compare_reports( + baseline: dict[str, Any], candidate: dict[str, Any] +) -> dict[str, Any]: + """Compare two reports without conflating provider or capacity effects.""" + + baseline_providers = _providers(baseline) + candidate_providers = _providers(candidate) + provider_separated = bool(baseline_providers and candidate_providers) and ( + baseline_providers.isdisjoint(candidate_providers) + ) + baseline_cohort = _comparison_cohort(baseline) + candidate_cohort = _comparison_cohort(candidate) + baseline_names = set(baseline_cohort.get("job_names", [])) or { + item["name"] for item in baseline["jobs"].get("by_name", []) + } + candidate_names = set(candidate_cohort.get("job_names", [])) or { + item["name"] for item in candidate["jobs"].get("by_name", []) + } + common_names = sorted(baseline_names & candidate_names) + dimension_overlap = { + key: sorted( + _runner_dimension_values(baseline, key) + & _runner_dimension_values(candidate, key) + ) + for key in ("operating_system", "architecture", "runner_role") + } + comparable_dimensions = all( + not _runner_dimension_values(baseline, key) + or not _runner_dimension_values(candidate, key) + or bool(values) + for key, values in dimension_overlap.items() + ) + base_queue_summary = baseline_cohort.get("runner_queue_seconds", {}) + candidate_queue_summary = candidate_cohort.get("runner_queue_seconds", {}) + base_queue = base_queue_summary.get("p95") + candidate_queue = candidate_queue_summary.get("p95") + base_execution = baseline_cohort.get("execution_seconds", {}).get("p95") + candidate_execution = candidate_cohort.get("execution_seconds", {}).get("p95") + baseline_count = int(base_queue_summary.get("count") or 0) + candidate_count = int(candidate_queue_summary.get("count") or 0) + sufficient_evidence = ( + baseline_count >= MIN_HEURISTIC_SAMPLES + and candidate_count >= MIN_HEURISTIC_SAMPLES + ) + if candidate_queue is not None and candidate_queue >= QUEUE_CONTAMINATION_SECONDS: + recommendation = "rollback" + elif ( + not sufficient_evidence + or not provider_separated + or not common_names + or not comparable_dimensions + ): + recommendation = "hold" + elif candidate_queue is None or candidate_queue > QUEUE_WARN_SECONDS: + recommendation = "hold" + else: + recommendation = "eligible" + return { + "baseline_source": baseline.get("source", {}).get("description"), + "candidate_source": candidate.get("source", {}).get("description"), + "provider_cohort_separation": { + "baseline_providers": sorted(baseline_providers), + "candidate_providers": sorted(candidate_providers), + "disjoint": provider_separated, + "status": "pass" if provider_separated else "fail", + }, + "common_job_families": common_names, + "dimension_overlap": dimension_overlap, + "comparable_dimensions": comparable_dimensions, + "sample_counts": { + "baseline_jobs": baseline_count, + "candidate_jobs": candidate_count, + "minimum_each": MIN_HEURISTIC_SAMPLES, + "sufficient": sufficient_evidence, + }, + "p95_seconds": { + "queue": { + "baseline": base_queue, + "candidate": candidate_queue, + "delta": ( + round(candidate_queue - base_queue, 3) + if candidate_queue is not None and base_queue is not None + else None + ), + }, + "execution": { + "baseline": base_execution, + "candidate": candidate_execution, + "delta": ( + round(candidate_execution - base_execution, 3) + if candidate_execution is not None and base_execution is not None + else None + ), + }, + }, + "recommendation": recommendation, + "interpretation": { + "eligible": "provider-separated comparable cohorts with candidate queue heuristics eligible", + "hold": "collect more comparable evidence or resolve provider/capacity mismatch", + "rollback": "candidate queue p95 crossed the deterministic contamination threshold", + }[recommendation], + } + + def analyze( runs: list[dict[str, Any]], requested_status: str, @@ -386,11 +803,25 @@ def analyze( run_reports = [] terminal_counts = collections.Counter() runner_groups: dict[ - tuple[str | None, str | None, str | None], list[dict[str, Any]] + tuple[ + str | None, + str | None, + str | None, + str | None, + str | None, + ], + list[dict[str, Any]], ] = collections.defaultdict(list) step_observations: list[dict[str, Any]] = [] wall_times = [] queue_times = [] + dependency_wait_times = [] + execution_times = [] + job_wall_times = [] + terminal_queue_times = [] + terminal_runner_queue_times = [] + terminal_dependency_wait_times = [] + terminal_execution_times = [] workflow_timing_excluded_reruns = 0 for run in selected: jobs = [job for job in run["jobs"] if job["conclusion"] != SKIPPED] @@ -401,7 +832,9 @@ def analyze( runner_groups[ ( dimensions["provider"], + dimensions["operating_system"], dimensions["architecture"], + dimensions["runner_role"], dimensions["runner_size"], ) ].append(sample) @@ -421,8 +854,24 @@ def analyze( terminal = ( max(completed, key=lambda job: job["completed"]) if completed else None ) + sample_by_job_id = {sample["job_id"]: sample for sample in samples} if terminal: terminal_counts[terminal["name"]] += 1 + terminal_sample = sample_by_job_id.get(terminal["id"]) + if terminal_sample is None: + raise ValueError( + f"terminal job {terminal['id']} is missing from observations" + ) + terminal_queue_times.append(terminal_sample["queue_seconds"]) + terminal_runner_queue_times.append( + terminal_sample["runner_queue_seconds"] + ) + terminal_dependency_wait_times.append( + terminal_sample["dependency_wait_seconds"] + ) + terminal_execution_times.append(terminal_sample["execution_seconds"]) + else: + terminal_sample = None longest = max( (sample for sample in samples if sample["duration_seconds"] is not None), key=lambda sample: sample["duration_seconds"], @@ -455,6 +904,12 @@ def analyze( "conclusion": run["conclusion"], "head_sha": run["sha"], "head_branch": run["branch"], + "plan_profile": run["plan_profile"], + "plan_digest": run["plan_digest"], + "change_class": run["change_class"], + "runner_image": run["runner_image"], + "toolchain_epoch": run["toolchain_epoch"], + "cache_mode": run["cache_mode"], "wall_seconds": wall, "queue_seconds": queue, "workflow_timing_excluded": not workflow_timing_eligible, @@ -463,8 +918,37 @@ def analyze( "terminal_job": terminal["name"] if terminal else None, "longest_job": longest["name"] if longest else None, "longest_job_seconds": longest["duration_seconds"] if longest else None, + "terminal_job_queue_seconds": ( + terminal_sample["queue_seconds"] if terminal_sample else None + ), + "terminal_job_runner_queue_seconds": ( + terminal_sample["runner_queue_seconds"] + if terminal_sample + else None + ), + "terminal_job_dependency_wait_seconds": ( + terminal_sample["dependency_wait_seconds"] + if terminal_sample + else None + ), + "terminal_job_execution_seconds": ( + terminal_sample["execution_seconds"] + if terminal_sample + else None + ), + "capacity_contaminated": any( + sample["runner_queue_seconds"] is not None + and sample["runner_queue_seconds"] + >= QUEUE_CONTAMINATION_SECONDS + for sample in samples + ), } ) + dependency_wait_times.extend( + sample["dependency_wait_seconds"] for sample in samples + ) + execution_times.extend(sample["execution_seconds"] for sample in samples) + job_wall_times.extend(sample["wall_seconds"] for sample in samples) groups: dict[str, list[dict[str, Any]]] = collections.defaultdict(list) for sample in observations: @@ -476,9 +960,21 @@ def analyze( "duration_seconds": summarize( [sample["duration_seconds"] for sample in samples] ), + "execution_seconds": summarize( + [sample["execution_seconds"] for sample in samples] + ), + "wall_seconds": summarize( + [sample["wall_seconds"] for sample in samples] + ), "queue_seconds": summarize( [sample["queue_seconds"] for sample in samples] ), + "runner_queue_seconds": summarize( + [sample["runner_queue_seconds"] for sample in samples] + ), + "dependency_wait_seconds": summarize( + [sample["dependency_wait_seconds"] for sample in samples] + ), "start_delay_seconds": summarize( [sample["start_delay_seconds"] for sample in samples] ), @@ -514,20 +1010,37 @@ def analyze( for name, count in terminal_counts.most_common(top) ] by_runner = [] - for (provider, architecture, runner_size), samples in runner_groups.items(): + for ( + provider, + operating_system, + architecture, + runner_role, + runner_size, + ), samples in runner_groups.items(): by_runner.append( { "provider": provider, + "operating_system": operating_system, "architecture": architecture, + "runner_role": runner_role, "runner_size": runner_size, "sample_count": len(samples), "job_names": sorted({sample["name"] for sample in samples}), "duration_seconds": summarize( [sample["duration_seconds"] for sample in samples] ), + "execution_seconds": summarize( + [sample["execution_seconds"] for sample in samples] + ), "queue_seconds": summarize( [sample["queue_seconds"] for sample in samples] ), + "runner_queue_seconds": summarize( + [sample["runner_queue_seconds"] for sample in samples] + ), + "dependency_wait_seconds": summarize( + [sample["dependency_wait_seconds"] for sample in samples] + ), } ) by_runner.sort( @@ -537,6 +1050,38 @@ def analyze( ), reverse=True, ) + comparison_observations = [ + sample for sample in observations if is_comparison_executor(sample["name"]) + ] + comparison_dimensions = { + key: sorted( + { + str(sample["runner_dimensions"][key]) + for sample in comparison_observations + if sample["runner_dimensions"].get(key) + } + ) + for key in ("operating_system", "architecture", "runner_role") + } + comparison_cohort = { + "scope": "build-test executors; orchestration and credentialed smoke jobs excluded", + "sample_count": len(comparison_observations), + "job_names": sorted({sample["name"] for sample in comparison_observations}), + "providers": sorted( + { + str(sample["runner_dimensions"]["provider"]) + for sample in comparison_observations + if sample["runner_dimensions"].get("provider") not in {None, "unknown"} + } + ), + "dimensions": comparison_dimensions, + "runner_queue_seconds": summarize( + [sample["runner_queue_seconds"] for sample in comparison_observations] + ), + "execution_seconds": summarize( + [sample["execution_seconds"] for sample in comparison_observations] + ), + } step_groups: dict[ tuple[ @@ -545,6 +1090,8 @@ def analyze( str | None, str | None, str | None, + str | None, + str | None, ], list[dict[str, Any]], ] = collections.defaultdict(list) @@ -555,7 +1102,9 @@ def analyze( sample["job_name"], sample["step_name"], dimensions["provider"], + dimensions["operating_system"], dimensions["architecture"], + dimensions["runner_role"], dimensions["runner_size"], ) ].append(sample) @@ -564,7 +1113,9 @@ def analyze( "job_name": job_name, "step_name": step_name, "provider": provider, + "operating_system": operating_system, "architecture": architecture, + "runner_role": runner_role, "runner_size": runner_size, "sample_count": len(samples), "duration_seconds": summarize( @@ -578,7 +1129,9 @@ def analyze( job_name, step_name, provider, + operating_system, architecture, + runner_role, runner_size, ), samples in step_groups.items() ] @@ -589,8 +1142,8 @@ def analyze( ), reverse=True, ) - return { - "schema_version": 2, + report = { + "schema_version": 3, "generated_at": dt.datetime.now(dt.timezone.utc) .isoformat() .replace("+00:00", "Z"), @@ -604,9 +1157,19 @@ def analyze( "first-attempt run created_at to started_at; reruns excluded" ), "job_duration_seconds": "job started_at to completed_at", + "job_execution_seconds": "same interval as job_duration_seconds", + "job_wall_seconds": "job created_at to completed_at", "job_queue_seconds": ( "job created_at to started_at; unavailable without job created_at" ), + "runner_queue_seconds": ( + "job started_at minus dependency-ready timestamp when present; " + "otherwise falls back to creation-to-start for compatibility" + ), + "dependency_wait_seconds": ( + "dependency-ready timestamp to job created_at, or an explicit " + "instrumented duration; unavailable in standard jobs API data" + ), "job_start_delay_seconds": ( "first-attempt workflow created_at to job started_at; includes " "dependency wait; reruns excluded" @@ -615,7 +1178,8 @@ def analyze( "last non-skipped job to finish; a critical-path candidate" ), "runner_dimensions": ( - "provider, architecture, and Depot runner size derived from labels" + "provider, operating system, architecture, semantic runner role, " + "and Depot runner size derived from labels or optional metadata" ), "step_duration_seconds": "step started_at to completed_at", }, @@ -629,20 +1193,44 @@ def analyze( "workflow": { "wall_seconds": summarize(wall_times), "queue_seconds": summarize(queue_times), + "dependency_wait_seconds": summarize(dependency_wait_times), + "execution_seconds": summarize(execution_times), + "job_wall_seconds": summarize(job_wall_times), + "terminal_job_queue_seconds": summarize(terminal_queue_times), + "terminal_job_runner_queue_seconds": summarize( + terminal_runner_queue_times + ), + "terminal_job_dependency_wait_seconds": summarize( + terminal_dependency_wait_times + ), + "terminal_job_execution_seconds": summarize(terminal_execution_times), }, "jobs": { "sample_count": len(observations), "duration_seconds": summarize( [sample["duration_seconds"] for sample in observations] ), + "execution_seconds": summarize( + [sample["execution_seconds"] for sample in observations] + ), + "wall_seconds": summarize( + [sample["wall_seconds"] for sample in observations] + ), "queue_seconds": summarize( [sample["queue_seconds"] for sample in observations] ), + "runner_queue_seconds": summarize( + [sample["runner_queue_seconds"] for sample in observations] + ), + "dependency_wait_seconds": summarize( + [sample["dependency_wait_seconds"] for sample in observations] + ), "start_delay_seconds": summarize( [sample["start_delay_seconds"] for sample in observations] ), "by_name": by_name, "by_runner": by_runner, + "comparison_cohort": comparison_cohort, "critical_finish_candidates": critical, "slowest_observations": slowest, }, @@ -652,6 +1240,9 @@ def analyze( }, "runs": run_reports, } + report["capacity"] = capacity_metrics(observations) + report["heuristics"] = queue_heuristics(report) + return report def human(seconds: float | int | None) -> str: @@ -697,7 +1288,12 @@ def render_markdown(report: dict[str, Any], top: int) -> str: "| Timing | Samples | p50 | p90 | p95 | Max |", "| --- | ---: | ---: | ---: | ---: | ---: |", ] - for label, key in (("Wall time", "wall_seconds"), ("Queue", "queue_seconds")): + for label, key in ( + ("Wall time", "wall_seconds"), + ("Queue", "queue_seconds"), + ("Dependency wait", "dependency_wait_seconds"), + ("Execution", "execution_seconds"), + ): stats = workflow[key] lines.append( f"| {label} | {stats['count']} | {human(stats['p50'])} | " @@ -708,33 +1304,51 @@ def render_markdown(report: dict[str, Any], top: int) -> str: "", "## Runner dimensions", "", - "| Provider | Architecture | Depot size | Jobs | Duration p50 | Duration p95 | Queue p95 |", - "| --- | --- | --- | ---: | ---: | ---: | ---: |", + "| Provider | OS | Architecture | Role | Depot size | Jobs | Execution p50 | Execution p95 | Runner queue p95 |", + "| --- | --- | --- | --- | --- | ---: | ---: | ---: | ---: |", ] for item in jobs["by_runner"][:top]: lines.append( f"| {markdown_escape(item['provider'])} | " + f"{markdown_escape(item['operating_system'] or 'n/a')} | " f"{markdown_escape(item['architecture'] or 'n/a')} | " + f"{markdown_escape(item['runner_role'] or 'n/a')} | " f"{markdown_escape(item['runner_size'] or 'n/a')} | " f"{item['sample_count']} | " - f"{human(item['duration_seconds']['p50'])} | " - f"{human(item['duration_seconds']['p95'])} | " - f"{human(item['queue_seconds']['p95'])} |" + f"{human(item['execution_seconds']['p50'])} | " + f"{human(item['execution_seconds']['p95'])} | " + f"{human(item['runner_queue_seconds']['p95'])} |" ) lines += [ + "", + "## Capacity and rollout heuristics", + "", + ( + f"Runner-minutes: **{report['capacity']['runner_minutes']}**; " + f"cancelled runner-minutes: **{report['capacity']['cancelled_runner_minutes']}**; " + f"peak workers: **{report['capacity']['peak_workers']['total'] or 'n/a'}**." + ), + ( + f"Queue p95: **{human(report['heuristics']['queue_p95_seconds'])}**; " + f"terminal queue p95: **{human(report['heuristics']['terminal_job_queue_p95_seconds'])}**; " + f"capacity contaminated: **{report['heuristics']['capacity_contaminated']}**; " + f"state: **{report['heuristics']['state']}**." + ), "", "## Step timing", "", ] if report["steps"]["by_name"]: lines += [ - "| Provider | Architecture | Runner size | Job | Step | Samples | Duration p50 | Duration p95 |", - "| --- | --- | --- | --- | --- | ---: | ---: | ---: |", + "| Provider | OS | Architecture | Role | Runner size | Job | Step | Samples | Duration p50 | Duration p95 |", + "| --- | --- | --- | --- | --- | --- | --- | ---: | ---: | ---: |", ] for item in report["steps"]["by_name"][:top]: lines.append( f"| {markdown_escape(item['provider'] or 'n/a')} | " + f"{markdown_escape(item['operating_system'] or 'n/a')} | " f"{markdown_escape(item['architecture'] or 'n/a')} | " + f"{markdown_escape(item['runner_role'] or 'n/a')} | " f"{markdown_escape(item['runner_size'] or 'n/a')} | " f"{markdown_escape(item['job_name'])} | " f"{markdown_escape(item['step_name'])} | {item['sample_count']} | " @@ -750,11 +1364,11 @@ def render_markdown(report: dict[str, Any], top: int) -> str: "", "## Slow job families", "", - "| Job | Samples | Duration p50 | Duration p95 | Queue p50 | Queue p95 |", + "| Job | Samples | Duration p50 | Duration p95 | Runner queue p50 | Runner queue p95 |", "| --- | ---: | ---: | ---: | ---: | ---: |", ] for item in jobs["by_name"][:top]: - duration, queue = item["duration_seconds"], item["queue_seconds"] + duration, queue = item["duration_seconds"], item["runner_queue_seconds"] lines.append( f"| {markdown_escape(item['name'])} | {item['sample_count']} | " f"{human(duration['p50'])} | {human(duration['p95'])} | " @@ -788,6 +1402,22 @@ def render_markdown(report: dict[str, Any], top: int) -> str: ) lines += [ "", + ] + if report.get("comparison"): + comparison = report["comparison"] + separation = comparison["provider_cohort_separation"] + lines += [ + "## Provider cohort comparison", + "", + ( + f"Provider separation: **{separation['status']}** " + f"({', '.join(separation['baseline_providers']) or 'n/a'} → " + f"{', '.join(separation['candidate_providers']) or 'n/a'}); " + f"recommendation: **{comparison['recommendation']}**." + ), + "", + ] + lines += [ ( "_Job queue uses GitHub's job creation-to-start interval. Offline " "`gh run view` data may omit creation times; those samples are n/a. " @@ -827,6 +1457,10 @@ def parse_args(argv: list[str]) -> argparse.Namespace: parser.add_argument("--workflow") parser.add_argument("--run-id", type=int, action="append", default=[]) parser.add_argument("--input", help="Detailed run JSON, --raw-out JSON, or -") + parser.add_argument( + "--compare-input", + help="Detailed/raw run JSON for a historical baseline cohort", + ) parser.add_argument("--limit", type=int, default=20) parser.add_argument("--status", default="success") parser.add_argument("--branch") @@ -869,6 +1503,20 @@ def main(argv: list[str]) -> int: } runs = [normalize_run(run) for run in raw_runs] report = analyze(runs, args.status, args.top, source, labels(args.label)) + if args.compare_input: + baseline_raw = load_runs(args.compare_input) + baseline_runs = [normalize_run(run) for run in baseline_raw] + baseline = analyze( + baseline_runs, + args.status, + args.top, + { + "kind": "file", + "description": args.compare_input, + }, + {"cohort": "baseline"}, + ) + report["comparison"] = compare_reports(baseline, report) if args.raw_out: write( args.raw_out, diff --git a/scripts/tests/test_ci_artifact_actions.py b/scripts/tests/test_ci_artifact_actions.py index 9a413fe177..143c32320c 100644 --- a/scripts/tests/test_ci_artifact_actions.py +++ b/scripts/tests/test_ci_artifact_actions.py @@ -36,6 +36,10 @@ def test_external_actions_have_sha_and_release_provenance(self) -> None: f"Mesh-LLM/mesh-llm/.github/workflows/ci-{lane}-lane.yml@main": f"pr_{lane}.yml" for lane in ("quality", "website", "linux", "macos", "windows") } + protected_pre_checkout_action = ( + "Mesh-LLM/mesh-llm/.github/actions/" + "audit-depot-pr-isolation@4f9899c0c381b8cc225c9be73980ffa8db525c57" + ) for path in (*action_files, *workflow_files): for line_number, line in enumerate( @@ -50,6 +54,27 @@ def test_external_actions_have_sha_and_release_provenance(self) -> None: if value in protected_pr_lanes: self.assertEqual(protected_pr_lanes[value], path.name) continue + if value == protected_pre_checkout_action: + self.assertIn(path.name, { + "ci-linux-host-slice.yml", + "ci-linux-product-slice.yml", + "ci-linux-runtime-slice.yml", + "ci-macos-host-slice.yml", + "ci-macos-product-slice.yml", + "ci-macos-runtime-slice.yml", + "ci-platform-checks-slice.yml", + "ci-quality-slice.yml", + "ci-rust-tests-slice.yml", + "ci-ui-artifact-slice.yml", + "ci-web-slice.yml", + "ci-windows-host-slice.yml", + "ci-windows-product-slice.yml", + "ci-windows-runtime-slice.yml", + "native-sdk-artifact.yml", + "static-abi-artifact.yml", + "swift-sdk-artifact.yml", + }) + continue with self.subTest( path=path.relative_to(ROOT), line=line_number, @@ -287,6 +312,10 @@ def run_runner_selector( main_enabled: str, manual_enabled: str, original_event_name: str = "", + repository: str = "Mesh-LLM/mesh-llm", + head_repository: str | None = None, + pr_enabled: str = "false", + force_hosted: str = "false", ) -> dict[str, str]: action = self.read_action("select-ci-runners") run_block = action.split(" run: |\n", maxsplit=1)[1] @@ -303,8 +332,16 @@ def run_runner_selector( **os.environ, "GITHUB_OUTPUT": str(output), "INPUT_EVENT_NAME": event_name, + "INPUT_ORIGINAL_EVENT_NAME": original_event_name, + "GITHUB_EVENT_NAME": event_name, + "INPUT_REPOSITORY": repository, + "INPUT_HEAD_REPOSITORY": head_repository or repository, + "GITHUB_REPOSITORY": repository, "INPUT_REF": ref, + "GITHUB_REF": ref, "INPUT_DEPOT_MAIN_ENABLED": main_enabled, + "INPUT_DEPOT_PR_ENABLED": pr_enabled, + "INPUT_FORCE_HOSTED": force_hosted, "INPUT_MANUAL_USE_DEPOT": manual_enabled, "DISPATCH_ORIGINAL_EVENT_NAME": original_event_name, }, @@ -333,8 +370,15 @@ def run_reusable_runner_policy( workflow = ( ROOT / ".github" / "workflows" / workflow_name ).read_text(encoding="utf-8") + selected = self.run_runner_selector( + event_name=event_name, + ref=ref, + main_enabled=depot_enabled, + manual_enabled=manual_use_depot, + repository=repository, + ) policy = workflow.split( - " - name: Derive protected runner policy\n", + " - name: Resolve runner size and target\n", maxsplit=1, )[1] run_block = policy.split(" run: |\n", maxsplit=1)[1] @@ -356,13 +400,21 @@ def run_reusable_runner_policy( env={ **os.environ, "GITHUB_OUTPUT": str(output), - "POLICY_REPOSITORY": repository, - "POLICY_REF": ref, + "TARGET": target, + "RUNNER_SIZE": runner_size, "POLICY_EVENT_NAME": event_name, - "POLICY_DEPOT_ENABLED": depot_enabled, - "POLICY_MANUAL_USE_DEPOT": manual_use_depot, - "POLICY_TARGET": target, - "POLICY_RUNNER_SIZE": runner_size, + "ALLOW_DEPOT_REMOTE_CACHE": selected[ + "allow_depot_remote_cache" + ], + "RUNNER_DEFAULT": selected["runner"], + "RUNNER_4": selected["runner_4"], + "RUNNER_8": selected["runner_8"], + "RUNNER_16": selected["runner_16"], + "RUNNER_ARM": selected["runner_arm"], + "RUNNER_ARM_4": selected["runner_arm_4"], + "RUNNER_ARM_8": selected["runner_arm_8"], + "RUNNER_ARM_16": selected["runner_arm_16"], + "RUNNER_MACOS": selected["runner_macos"], }, check=False, capture_output=True, @@ -376,6 +428,10 @@ def run_reusable_runner_policy( encoding="utf-8", ).splitlines() ) + outputs.setdefault( + "allow_depot_remote_cache", + selected["allow_depot_remote_cache"], + ) return result, outputs def test_host_action_uses_canonical_dynamic_host_builder(self) -> None: @@ -1014,33 +1070,24 @@ def test_protected_reusable_producers_own_runner_and_cache_policy( workflow, ) self.assertIn( - 'POLICY_REPOSITORY" == "Mesh-LLM/mesh-llm"', - workflow, - ) - self.assertIn( - 'POLICY_REF" == "refs/heads/main"', - workflow, - ) - self.assertIn( - 'POLICY_EVENT_NAME" == "push"', - workflow, - ) - self.assertIn( - 'POLICY_EVENT_NAME" == "workflow_dispatch"', + "depot_main_enabled: " + "${{ vars.DEPOT_RUNNERS_ENABLED == 'true' }}", workflow, ) self.assertIn( - "POLICY_DEPOT_ENABLED: " - "${{ vars.DEPOT_RUNNERS_ENABLED == 'true' }}", + "manual_use_depot: " + "${{ inputs.use_depot }}", workflow, ) + self.assertIn("repository: ${{ github.repository }}", workflow) self.assertIn( - "POLICY_MANUAL_USE_DEPOT: " - "${{ inputs.use_depot }}", + "head_repository: ${{ github.event.pull_request.head.repo.full_name }}", workflow, ) - self.assertIn("default|4|8|16", workflow) - self.assertIn("depot-ubuntu-24.04-arm", workflow) + self.assertIn("ref: ${{ github.ref }}", workflow) + self.assertIn("depot_pr_enabled:", workflow) + self.assertIn("default) runner=", workflow) + self.assertIn("RUNNER_ARM", workflow) def test_protected_reusable_runner_policy_is_fail_closed(self) -> None: hosted_cases = ( @@ -1166,7 +1213,7 @@ def test_protected_reusable_runner_policy_is_fail_closed(self) -> None: "runner_size must be one of", result.stderr, ) - self.assertEqual(outputs, {}) + self.assertNotIn("runner", outputs) result, outputs = self.run_reusable_runner_policy( workflow_name, @@ -1264,7 +1311,11 @@ def test_swift_sdk_build_is_a_shared_immutable_producer(self) -> None: ) self.assertIn("persist-credentials: false", producer) self.assertIn("actions/upload-artifact@", producer) - self.assertIn("runs-on: macos-15", producer) + self.assertIn( + "runs-on: ${{ needs.runner_policy.outputs.runner_macos }}", + producer, + ) + self.assertIn("depot-macos-15", self.read_action("select-ci-runners")) self.assertIn("RUSTC_WRAPPER: sccache", producer) self.assertIn("SCCACHE_GHA_RW_MODE:", producer) self.assertIn( @@ -1709,77 +1760,182 @@ def test_sccache_prefers_depot_webdav_with_disk_fallback(self) -> None: action, ) - def test_runner_selection_never_routes_pull_requests_to_depot(self) -> None: + def test_runner_selection_uses_event_repository_and_ref_policy(self) -> None: action = self.read_action("select-ci-runners") self.assertIn("depot_main_enabled", action) - self.assertNotIn("depot_pr_enabled", action) - self.assertNotIn("head_repository", action) - self.assertNotIn("\n repository:", action) + self.assertIn("depot_pr_enabled", action) + self.assertIn("\n repository:", action) + self.assertIn("INPUT_REPOSITORY", action) + self.assertIn("\n head_repository:", action) + self.assertIn("INPUT_HEAD_REPOSITORY", action) + self.assertIn("INPUT_DEPOT_PR_ENABLED", action) self.assertIn("\n ref:", action) - - runtime = action.split("runs:", maxsplit=1)[1] - effective_event_case = runtime.split( - 'case "$effective_event_name" in', - maxsplit=1, - )[1] - pull_request_case = effective_event_case.split( - "pull_request|pull_request_target)", - maxsplit=1, - )[1].split(";;", maxsplit=1)[0] - self.assertIn("depot_enabled=false", pull_request_case) - self.assertNotIn("depot_enabled=true", pull_request_case) - self.assertNotIn("INPUT_DEPOT_PR_ENABLED", runtime) - self.assertNotIn("INPUT_HEAD_REPOSITORY", runtime) - self.assertNotIn("INPUT_REPOSITORY", runtime) - - dispatch_case = effective_event_case.split( - "workflow_dispatch)", - maxsplit=1, - )[1].split(";;", maxsplit=1)[0] - self.assertIn("INPUT_DEPOT_MAIN_ENABLED", dispatch_case) - self.assertIn("INPUT_MANUAL_USE_DEPOT", dispatch_case) - self.assertIn('INPUT_REF" == "refs/heads/main"', dispatch_case) - self.assertIn("depot_enabled=true", dispatch_case) - - push_case = effective_event_case.split( - "push)", - maxsplit=1, - )[1].split(";;", maxsplit=1)[0] - self.assertIn("INPUT_DEPOT_MAIN_ENABLED", push_case) - self.assertIn('INPUT_REF" == "refs/heads/main"', push_case) - self.assertIn("depot_enabled=true", push_case) - - default_case = effective_event_case.split( - "*)", - maxsplit=1, - )[1].split(";;", maxsplit=1)[0] - self.assertIn("depot_enabled=false", default_case) - self.assertNotIn("depot_enabled=true", default_case) + self.assertIn("refs/pull/[0-9]+/merge", action) + self.assertIn("pull_request_target)", action) + self.assertIn("allow_depot_remote_cache=false", action) self.assertIn("depot-ubuntu-24.04-16", action) self.assertIn("depot-ubuntu-24.04-arm-16", action) + self.assertIn("depot-macos-15", action) + self.assertIn("depot-windows-2022", action) cases = ( - ("pull_request", "refs/pull/12/merge", "true", "true", "false", "ubuntu-24.04"), - ("pull_request_target", "refs/heads/main", "true", "true", "false", "ubuntu-24.04"), - ("workflow_dispatch", "refs/heads/main", "false", "true", "true", "depot-ubuntu-24.04"), - ("workflow_dispatch", "refs/heads/feature", "true", "true", "false", "ubuntu-24.04"), - ("push", "refs/heads/main", "true", "false", "true", "depot-ubuntu-24.04"), - ("push", "refs/heads/feature", "true", "false", "false", "ubuntu-24.04"), - ("push", "refs/tags/v1.2.3", "true", "false", "false", "ubuntu-24.04"), - ("push", "refs/heads/main", "false", "false", "false", "ubuntu-24.04"), - ("schedule", "refs/heads/main", "true", "true", "false", "ubuntu-24.04"), - ) - for event_name, ref, main, manual, enabled, runner in cases: + ( + "pull_request", + "refs/pull/12/merge", + "false", + "false", + "true", + "true", + "depot-ubuntu-24.04", + "false", + ), + ( + "pull_request", + "refs/pull/12/merge", + "false", + "false", + "true", + "false", + "ubuntu-24.04", + "false", + ), + ( + "pull_request_target", + "refs/pull/12/merge", + "true", + "true", + "true", + "false", + "ubuntu-24.04", + "false", + ), + ( + "workflow_dispatch", + "refs/heads/main", + "true", + "false", + "", + "false", + "depot-ubuntu-24.04", + "true", + ), + ( + "workflow_dispatch", + "refs/heads/main", + "false", + "true", + "", + "false", + "depot-ubuntu-24.04", + "true", + ), + ( + "workflow_dispatch", + "refs/heads/main", + "true", + "true", + "", + "false", + "depot-ubuntu-24.04", + "true", + ), + ( + "workflow_dispatch", + "refs/heads/main", + "true", + "true", + "push", + "false", + "depot-ubuntu-24.04", + "true", + ), + ( + "workflow_dispatch", + "refs/heads/feature", + "true", + "true", + "", + "true", + "ubuntu-24.04", + "false", + ), + ( + "push", + "refs/heads/main", + "true", + "false", + "", + "false", + "depot-ubuntu-24.04", + "true", + ), + ( + "push", + "refs/heads/feature", + "true", + "false", + "", + "false", + "ubuntu-24.04", + "false", + ), + ( + "push", + "refs/tags/v1.2.3", + "true", + "false", + "", + "false", + "ubuntu-24.04", + "false", + ), + ( + "push", + "refs/heads/main", + "false", + "false", + "", + "false", + "ubuntu-24.04", + "false", + ), + ( + "schedule", + "refs/heads/main", + "true", + "true", + "", + "false", + "ubuntu-24.04", + "false", + ), + ) + for ( + event_name, + ref, + main, + manual, + original_event_name, + pr_enabled, + runner, + cache_enabled, + ) in cases: with self.subTest(event_name=event_name, ref=ref): outputs = self.run_runner_selector( event_name=event_name, ref=ref, main_enabled=main, manual_enabled=manual, + original_event_name=original_event_name, + pr_enabled=pr_enabled, ) + enabled = "true" if runner.startswith("depot-") else "false" self.assertEqual(outputs["depot_enabled"], enabled) - self.assertEqual(outputs["allow_depot_remote_cache"], enabled) + self.assertEqual( + outputs["allow_depot_remote_cache"], + cache_enabled, + ) self.assertEqual(outputs["runner"], runner) expected_arm = ( "depot-ubuntu-24.04-arm" @@ -1797,6 +1953,65 @@ def test_runner_selection_never_routes_pull_requests_to_depot(self) -> None: outputs[f"runner_arm_{size}"], expected_sized_arm, ) + expected_macos = ( + "depot-macos-15" + if enabled == "true" + else "macos-15" + ) + expected_windows = ( + "depot-windows-2022" + if enabled == "true" + else "windows-2022" + ) + self.assertEqual(outputs["runner_macos"], expected_macos) + self.assertEqual(outputs["runner_windows"], expected_windows) + + untrusted_repository = self.run_runner_selector( + event_name="pull_request", + ref="refs/pull/12/merge", + main_enabled="true", + manual_enabled="true", + pr_enabled="true", + repository="attacker/mesh-llm", + ) + self.assertEqual(untrusted_repository["depot_enabled"], "false") + self.assertEqual(untrusted_repository["runner"], "ubuntu-24.04") + self.assertEqual( + untrusted_repository["allow_depot_remote_cache"], + "false", + ) + + fork_head_repository = self.run_runner_selector( + event_name="pull_request", + ref="refs/pull/12/merge", + main_enabled="true", + manual_enabled="true", + pr_enabled="true", + head_repository="attacker/mesh-llm", + ) + self.assertEqual(fork_head_repository["depot_enabled"], "false") + self.assertEqual(fork_head_repository["runner"], "ubuntu-24.04") + + runner_contract_change = self.run_runner_selector( + event_name="pull_request", + ref="refs/pull/12/merge", + main_enabled="true", + manual_enabled="true", + pr_enabled="true", + force_hosted="true", + ) + self.assertEqual(runner_contract_change["depot_enabled"], "false") + self.assertEqual(runner_contract_change["runner"], "ubuntu-24.04") + + non_merge_ref = self.run_runner_selector( + event_name="pull_request", + ref="refs/pull/12/head", + main_enabled="true", + manual_enabled="true", + pr_enabled="true", + ) + self.assertEqual(non_merge_ref["depot_enabled"], "false") + self.assertEqual(non_merge_ref["runner"], "ubuntu-24.04") untrusted_dispatch = self.run_runner_selector( event_name="workflow_dispatch", @@ -1804,9 +2019,14 @@ def test_runner_selection_never_routes_pull_requests_to_depot(self) -> None: main_enabled="true", manual_enabled="true", original_event_name="pull_request_target", + pr_enabled="true", ) self.assertEqual(untrusted_dispatch["depot_enabled"], "false") self.assertEqual(untrusted_dispatch["runner"], "ubuntu-24.04") + self.assertEqual( + untrusted_dispatch["allow_depot_remote_cache"], + "false", + ) def test_dispatched_pr_cache_writes_remain_blocked_with_depot( self, diff --git a/scripts/tests/test_ci_lane_workflows.py b/scripts/tests/test_ci_lane_workflows.py index 60e828595b..dd256e7f3e 100644 --- a/scripts/tests/test_ci_lane_workflows.py +++ b/scripts/tests/test_ci_lane_workflows.py @@ -342,13 +342,43 @@ def test_dispatched_main_preserves_trusted_runner_policy(self) -> None: encoding="utf-8" ) self.assertIn("DISPATCH_ORIGINAL_EVENT_NAME", selector) - self.assertIn("pull_request|pull_request_target) effective_event_name=pull_request", selector) + self.assertIn("pull_request|pull_request_target)", selector) + self.assertIn("is_dispatched_pull_request=true", selector) for name in ("static-abi-artifact.yml", "native-sdk-artifact.yml"): workflow = self.workflow(name) self.assertIn( - "github.event.inputs.original_event_name || github.event_name", + "original_event_name: ${{ inputs.original_event_name }}", workflow, ) + self.assertIn("repository: ${{ github.repository }}", workflow) + self.assertIn( + "head_repository: ${{ github.event.pull_request.head.repo.full_name }}", + workflow, + ) + self.assertIn( + "depot_pr_enabled: ${{ vars.DEPOT_PR_RUNNERS_ENABLED == 'true' }}", + workflow, + ) + + def test_runner_contract_changes_force_all_pr_build_slices_hosted(self) -> None: + signal = ( + "force_hosted: " + "${{ fromJson(inputs.lane_plan_json).signals.runner_contract_required }}" + ) + minimum_calls = { + "ci-quality-lane.yml": 1, + "ci-website-lane.yml": 1, + "ci-linux-lane.yml": 7, + "ci-macos-lane.yml": 6, + "ci-windows-lane.yml": 5, + } + for name, minimum in minimum_calls.items(): + with self.subTest(workflow=name): + self.assertGreaterEqual(self.workflow(name).count(signal), minimum) + selector = (ROOT / ".github/actions/select-ci-runners/action.yml").read_text( + encoding="utf-8" + ) + self.assertIn('"$INPUT_FORCE_HOSTED" == "false"', selector) def test_reporter_completes_only_correlated_checks(self) -> None: action = (ROOT / ".github/actions/report-ci-lane/action.yml").read_text( diff --git a/scripts/tests/test_collect_ci_metrics.py b/scripts/tests/test_collect_ci_metrics.py index 2789051a0a..0ad2f6383d 100644 --- a/scripts/tests/test_collect_ci_metrics.py +++ b/scripts/tests/test_collect_ci_metrics.py @@ -5,6 +5,7 @@ import sys import tempfile import unittest +from unittest import mock SCRIPT = pathlib.Path(__file__).resolve().parents[1] / "collect-ci-metrics.py" @@ -287,7 +288,7 @@ def test_cli_reads_saved_json_and_writes_both_report_formats(self): report = json.loads(json_path.read_text(encoding="utf-8")) markdown = markdown_path.read_text(encoding="utf-8") - self.assertEqual(report["schema_version"], 2) + self.assertEqual(report["schema_version"], 3) self.assertEqual(report["benchmark_labels"], {"provider": "fixture"}) self.assertIn("# CI timing summary", markdown) self.assertIn("Runner dimensions", markdown) @@ -394,6 +395,237 @@ def test_analysis_preserves_runner_dimensions_and_step_timing(self): slowest = report["jobs"]["slowest_observations"][0] self.assertEqual(slowest["runner_dimensions"]["runner_size"], "8") + def test_timing_decomposition_keeps_dependency_wait_explicit(self): + raw = run( + 105, + "2026-07-04T00:00:00Z", + "2026-07-04T00:00:01Z", + "2026-07-04T00:03:00Z", + [ + job( + "runtime", + "2026-07-04T00:00:10Z", + "2026-07-04T00:00:20Z", + "2026-07-04T00:02:20Z", + labels=["depot-ubuntu-24.04-8"], + ) + ], + ) + raw["jobs"][0]["dependency_ready_at"] = "2026-07-04T00:00:18Z" + raw["jobs"][0]["runner_role"] = "linux-native-16" + raw["jobs"][0]["operating_system"] = "Linux" + normalized = self.collector.normalize_run(raw) + report = self.collector.analyze( + [normalized], + requested_status="success", + top=5, + source={"description": "decomposition"}, + labels={"provider": "depot"}, + ) + runner = report["jobs"]["by_runner"][0] + self.assertEqual(runner["operating_system"], "linux") + self.assertEqual(runner["runner_role"], "linux-native-16") + self.assertEqual(report["jobs"]["queue_seconds"]["p50"], 10.0) + self.assertEqual(report["jobs"]["runner_queue_seconds"]["p50"], 2.0) + self.assertEqual(report["jobs"]["dependency_wait_seconds"]["p50"], 8.0) + self.assertEqual(report["jobs"]["execution_seconds"]["p50"], 120.0) + self.assertEqual(report["jobs"]["wall_seconds"]["p50"], 130.0) + self.assertEqual(report["heuristics"]["state"], "insufficient_sample") + + def test_capacity_contamination_and_queue_heuristic_are_deterministic(self): + raw_runs = self.sample_runs() + for raw_run in raw_runs: + for raw_job in raw_run["jobs"]: + if raw_job["conclusion"] == "skipped": + continue + raw_job["created_at"] = "2026-06-30T00:00:00Z" + raw_job["labels"] = ["depot-ubuntu-24.04-8"] + report = self.collector.analyze( + [self.collector.normalize_run(raw) for raw in raw_runs], + requested_status="success", + top=5, + source={"description": "contaminated"}, + labels={"provider": "depot"}, + ) + self.assertTrue(report["heuristics"]["capacity_contaminated"]) + self.assertEqual(report["heuristics"]["state"], "rollback") + self.assertGreaterEqual(report["capacity"]["runner_minutes"], 28.0) + + def test_capacity_peak_total_merges_runner_dimension_groups(self): + raw = run( + 106, + "2026-07-05T00:00:00Z", + "2026-07-05T00:00:01Z", + "2026-07-05T00:11:00Z", + [ + job( + "linux build", + "2026-07-05T00:00:05Z", + "2026-07-05T00:00:10Z", + "2026-07-05T00:10:00Z", + labels=["depot-ubuntu-24.04-8"], + ), + job( + "windows build", + "2026-07-05T00:00:06Z", + "2026-07-05T00:00:10Z", + "2026-07-05T00:10:00Z", + labels=["depot-windows-2022-8"], + ), + ], + ) + report = self.collector.analyze( + [self.collector.normalize_run(raw)], + requested_status="success", + top=5, + source={"description": "capacity overlap"}, + labels={}, + ) + + self.assertEqual(report["capacity"]["peak_workers"]["total"], 2) + self.assertEqual( + report["capacity"]["peak_workers"]["by_provider_os_role"], + { + "depot/linux/unknown": 1, + "depot/windows/unknown": 1, + }, + ) + + def test_observation_capacity_contamination_uses_runner_queue(self): + raw = run( + 107, + "2026-07-06T00:00:00Z", + "2026-07-06T00:00:01Z", + "2026-07-06T00:10:00Z", + [ + job( + "dependency build", + "2026-07-06T00:00:05Z", + "2026-07-06T00:08:35Z", + "2026-07-06T00:09:35Z", + labels=["depot-ubuntu-24.04-8"], + ) + ], + ) + raw["jobs"][0]["dependency_ready_at"] = "2026-07-06T00:08:25Z" + normalized = self.collector.normalize_run(raw) + sample = self.collector.observation(normalized, normalized["jobs"][0]) + + self.assertEqual(sample["queue_seconds"], 510.0) + self.assertEqual(sample["runner_queue_seconds"], 10.0) + self.assertFalse(sample["capacity_contaminated"]) + + def test_analysis_reuses_terminal_sample(self): + raw = self.sample_runs()[0] + normalized = self.collector.normalize_run(raw) + + with mock.patch.object( + self.collector, + "observation", + wraps=self.collector.observation, + ) as observation_mock: + self.collector.analyze( + [normalized], + requested_status="success", + top=5, + source={"description": "terminal reuse"}, + labels={}, + ) + + self.assertEqual( + observation_mock.call_count, + sum(job["conclusion"] != "skipped" for job in normalized["jobs"]), + ) + + def test_compare_reports_requires_provider_cohort_separation(self): + baseline_raw = self.sample_runs() + candidate_raw = self.sample_runs() + for raw_runs in (baseline_raw, candidate_raw): + raw_runs[0]["jobs"][1]["name"] = "test suite" + raw_runs[1]["jobs"].append( + job( + "Quality / Runner and cache contracts / Runner and cache contract", + "2026-07-02T00:00:20Z", + "2026-07-02T00:00:25Z", + "2026-07-02T00:00:35Z", + ) + ) + baseline = self.collector.analyze( + [self.collector.normalize_run(raw) for raw in baseline_raw], + requested_status="success", + top=5, + source={"description": "github historical"}, + labels={"provider": "github"}, + ) + for raw_run in candidate_raw: + for raw_job in raw_run["jobs"]: + if self.collector.is_comparison_executor(raw_job["name"]): + raw_job["labels"] = ["depot-ubuntu-24.04-8"] + candidate = self.collector.analyze( + [self.collector.normalize_run(raw) for raw in candidate_raw], + requested_status="success", + top=5, + source={"description": "depot candidate"}, + labels={"provider": "depot"}, + ) + comparison = self.collector.compare_reports(baseline, candidate) + self.assertTrue(comparison["provider_cohort_separation"]["disjoint"]) + self.assertEqual(comparison["recommendation"], "eligible") + self.assertEqual( + candidate["jobs"]["comparison_cohort"]["providers"], + ["depot"], + ) + + def test_compare_reports_holds_when_baseline_cohort_is_too_small(self): + baseline_raw = [self.sample_runs()[0]] + candidate_raw = self.sample_runs() + candidate_raw[0]["jobs"][1]["name"] = "test suite" + for raw_run in candidate_raw: + for raw_job in raw_run["jobs"]: + if self.collector.is_comparison_executor(raw_job["name"]): + raw_job["labels"] = ["depot-ubuntu-24.04-8"] + baseline = self.collector.analyze( + [self.collector.normalize_run(raw) for raw in baseline_raw], + requested_status="success", + top=5, + source={"description": "small github baseline"}, + labels={}, + ) + candidate = self.collector.analyze( + [self.collector.normalize_run(raw) for raw in candidate_raw], + requested_status="success", + top=5, + source={"description": "depot candidate"}, + labels={}, + ) + comparison = self.collector.compare_reports(baseline, candidate) + self.assertFalse(comparison["sample_counts"]["sufficient"]) + self.assertEqual(comparison["recommendation"], "hold") + + def test_runner_dimensions_cover_depot_platforms_and_hosted_intel_macos(self): + depot_32 = self.collector.runner_dimensions(["depot-ubuntu-24.04-32"]) + depot_64 = self.collector.runner_dimensions(["depot-ubuntu-24.04-64"]) + depot_2 = self.collector.runner_dimensions(["depot-ubuntu-24.04-2"]) + depot_macos = self.collector.runner_dimensions(["depot-macos-15"]) + depot_windows = self.collector.runner_dimensions(["depot-windows-2022-8"]) + hosted_intel = self.collector.runner_dimensions(["macos-15-intel"]) + self.assertEqual(depot_32["runner_size"], "32") + self.assertEqual(depot_64["runner_size"], "64") + self.assertEqual(depot_2["runner_size"], "default") + self.assertEqual( + depot_macos, + { + "provider": "depot", + "architecture": "arm64", + "runner_size": "default", + "operating_system": "macos", + "runner_role": None, + }, + ) + self.assertEqual(depot_windows["operating_system"], "windows") + self.assertEqual(depot_windows["architecture"], "amd64") + self.assertEqual(hosted_intel["architecture"], "amd64") + def test_cli_rejects_run_list_json_without_detailed_jobs(self): with tempfile.TemporaryDirectory() as directory: input_path = pathlib.Path(directory) / "runs.json" diff --git a/scripts/tests/test_depot_canary_workflow.py b/scripts/tests/test_depot_canary_workflow.py index c75ee954f1..fce1c29950 100644 --- a/scripts/tests/test_depot_canary_workflow.py +++ b/scripts/tests/test_depot_canary_workflow.py @@ -1,5 +1,9 @@ import unittest +import os from pathlib import Path +import subprocess +import tempfile +from textwrap import dedent ROOT = Path(__file__).resolve().parents[2] @@ -31,39 +35,199 @@ def test_canary_covers_measured_depot_sizes(self) -> None: self.assertIn("expected_arch=aarch64", self.workflow) self.assertIn('actual_arch="$(uname -m)"', self.workflow) - def test_canary_uses_a_pinned_cache_action_without_printing_tokens( - self, - ) -> None: + def test_canary_fails_closed_on_cache_and_registry_injection(self) -> None: + self.assertIn("forbidden_names=", self.workflow) + for name in ( + "DEPOT_CACHE_TOKEN", + "DEPOT_CACHE_URL", + "DEPOT_CACHE_API_URL", + "DEPOT_TOKEN", + "DEPOT_REGISTRY_HOST", + "DEPOT_REGISTRY_URL", + "DEPOT_REGISTRY_PULL_TOKEN", + "DEPOT_REGISTRY_TOKEN", + "SCCACHE_WEBDAV_ENDPOINT", + "SCCACHE_WEBDAV_TOKEN", + "SCCACHE_WEBDAV_USERNAME", + "SCCACHE_WEBDAV_PASSWORD", + ): + with self.subTest(name=name): + self.assertIn(name, self.workflow) + self.assertIn('actions_cache_url="${ACTIONS_CACHE_URL:-}"', self.workflow) + self.assertIn('actions_results_url="${ACTIONS_RESULTS_URL:-}"', self.workflow) + self.assertIn("${actions_cache_url,,}", self.workflow) + self.assertIn("${actions_results_url,,}", self.workflow) + self.assertIn("transparently redirected to Depot", self.workflow) + self.assertIn("ACTIONS_RESULTS_URL", self.workflow) + self.assertIn("actions\\.githubusercontent\\.com", self.workflow) + self.assertIn('docker_auth_config="${DOCKER_AUTH_CONFIG:-}"', self.workflow) + self.assertIn("${docker_auth_config,,}", self.workflow) + self.assertIn("Docker config contains Depot registry authentication", self.workflow) self.assertIn( - "actions/cache@caa296126883cff596d87d8935842f9db880ef25 " - "# v5.1.0", + "Depot cache credentials/endpoints injected | no", self.workflow, ) - self.assertIn("${DEPOT_CACHE_TOKEN:-}", self.workflow) - self.assertIn("${SCCACHE_WEBDAV_TOKEN:-}", self.workflow) - self.assertIn("${SCCACHE_WEBDAV_ENDPOINT:-}", self.workflow) - self.assertIn("Depot Cache authentication was not injected", self.workflow) - self.assertIn("Depot runner image identity was not injected", self.workflow) - self.assertIn('"$ImageOS" "$ImageVersion"', self.workflow) - self.assertIn("mozilla-actions/sccache-action@", self.workflow) - self.assertIn("sccache cc", self.workflow) self.assertIn( - "sccache did not select the Depot WebDAV backend", + "Depot registry credentials/config injected | no", self.workflow, ) - self.assertIn("expected a warm Depot sccache hit", self.workflow) - self.assertIn("expect_cache_hit:", self.workflow) + self.assertNotIn("expect_cache_hit", self.workflow) + self.assertNotIn("actions/cache@", self.workflow) + self.assertNotIn("sccache cc", self.workflow) + self.assertNotIn("mozilla-actions/sccache-action@", self.workflow) + self.assertNotIn("Exercise authenticated Depot sccache", self.workflow) + self.assertIn('"$ImageOS" "$ImageVersion"', self.workflow) + self.assertNotIn("printenv", self.workflow) + + def test_pr_audit_uses_portable_endpoint_and_docker_config_checks(self) -> None: + action = ( + ROOT + / ".github" + / "actions" + / "audit-depot-pr-isolation" + / "action.yml" + ).read_text(encoding="utf-8") self.assertIn( - '[[ "$EXPECT_CACHE_HIT" == "true" && "$CACHE_HIT" != "true" ]]', - self.workflow, + 'endpoint_lower="$(printf \'%s\' "$endpoint" | tr ' + "'[:upper:]' '[:lower:]')\"", + action, ) self.assertIn( - "key: depot-runner-canary-v2-exact-${{ matrix.runner }}-probe", - self.workflow, + 'docker_config="${DOCKER_CONFIG:-${HOME:-}/.docker}/config.json"', + action, ) - self.assertNotIn("key: depot-runner-canary-v1-", self.workflow) - self.assertNotIn("echo \"$DEPOT_CACHE_TOKEN\"", self.workflow) - self.assertNotIn("printenv", self.workflow) + self.assertIn("grep -Eiq 'depot\\.dev'", action) + + def test_endpoint_and_docker_auth_probes_fail_closed(self) -> None: + action = ( + ROOT + / ".github" + / "actions" + / "audit-depot-pr-isolation" + / "action.yml" + ).read_text(encoding="utf-8") + action_run = action.split(" run: |\n", maxsplit=1)[1] + action_script = dedent(action_run) + canary_start = self.workflow.index( + " for endpoint_name in ACTIONS_CACHE_URL " + "ACTIONS_RESULTS_URL; do", + ) + canary_end = self.workflow.index( + " {\n", + canary_start, + ) + canary_script = "set -euo pipefail\n" + dedent( + self.workflow[canary_start:canary_end], + ) + + def run_probe( + script: str, + cache_url: str, + results_url: str, + docker_auth_config: str | None = None, + docker_config_content: str | None = None, + ) -> subprocess.CompletedProcess[str]: + with ( + tempfile.TemporaryDirectory() as home, + tempfile.TemporaryDirectory() as docker_config, + ): + if docker_config_content is not None: + (Path(docker_config) / "config.json").write_text( + docker_config_content, + encoding="utf-8", + ) + environment = { + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "HOME": home, + "DOCKER_CONFIG": docker_config, + "GITHUB_EVENT_NAME": "pull_request", + "INPUT_ORIGINAL_EVENT_NAME": "pull_request", + "ACTIONS_CACHE_URL": cache_url, + "ACTIONS_RESULTS_URL": results_url, + } + if docker_auth_config is not None: + environment["DOCKER_AUTH_CONFIG"] = docker_auth_config + return subprocess.run( + ["bash", "-c", script], + check=False, + capture_output=True, + text=True, + env=environment, + ) + + valid_endpoints = ( + ( + "https://actions.githubusercontent.com/cache", + "https://results-receiver.actions.githubusercontent.com/results", + ), + ( + "https://actions.githubusercontent.com:443/cache", + "https://results-receiver.actions.githubusercontent.com:8443/results?x=1#ok", + ), + ( + "HTTPS://ACTIONS.GITHUBUSERCONTENT.COM?cache=1", + "https://results-receiver.actions.githubusercontent.com#results", + ), + ) + invalid_endpoints = ( + ( + "https://actions.githubusercontent.com:443@attacker.example/", + valid_endpoints[0][1], + ), + ( + "http://actions.githubusercontent.com/cache", + valid_endpoints[0][1], + ), + ( + "https://actions.githubusercontent.com.evil/cache", + valid_endpoints[0][1], + ), + ("", valid_endpoints[0][1]), + ) + for script_name, script in ( + ("audit action", action_script), + ("Depot canary", canary_script), + ): + for endpoints in valid_endpoints: + with self.subTest(script=script_name, endpoints=endpoints): + result = run_probe(script, *endpoints) + self.assertEqual(result.returncode, 0, result.stderr) + for endpoints in invalid_endpoints: + with self.subTest(script=script_name, endpoints=endpoints): + result = run_probe(script, *endpoints) + self.assertNotEqual(result.returncode, 0) + + depot_auth = '{"auths":{"REGISTRY.DEPOT.DEV":{"auth":"secret"}}}' + depot_config = '{"auths":{"registry.depot.dev":{"auth":"secret"}}}' + safe_config = '{"auths":{"ghcr.io":{"auth":"secret"}}}' + for script_name, script in ( + ("audit action", action_script), + ("Depot canary", canary_script), + ): + with self.subTest(script=script_name, auth="DOCKER_AUTH_CONFIG"): + result = run_probe( + script, + *valid_endpoints[0], + docker_auth_config=depot_auth, + ) + self.assertNotEqual(result.returncode, 0) + with self.subTest(script=script_name, auth="config.json"): + result = run_probe( + script, + *valid_endpoints[0], + docker_config_content=depot_config, + ) + self.assertNotEqual(result.returncode, 0) + with self.subTest(script=script_name, auth="safe config.json"): + result = run_probe( + script, + *valid_endpoints[0], + docker_config_content=safe_config, + ) + self.assertEqual(result.returncode, 0, result.stderr) + with self.subTest(script=script_name, auth="unset"): + result = run_probe(script, *valid_endpoints[0]) + self.assertEqual(result.returncode, 0, result.stderr) if __name__ == "__main__": diff --git a/scripts/tests/test_reusable_workflow_runner_trust.py b/scripts/tests/test_reusable_workflow_runner_trust.py index b357eefe71..a17d784831 100644 --- a/scripts/tests/test_reusable_workflow_runner_trust.py +++ b/scripts/tests/test_reusable_workflow_runner_trust.py @@ -55,7 +55,7 @@ def test_main_entry_and_orchestrator_do_not_use_pull_request_target(self) -> Non with self.subTest(workflow=name): self.assertNotIn("pull_request_target", self.workflow(name)) - def test_main_macos_and_windows_slices_have_fixed_platforms(self) -> None: + def test_macos_and_windows_slices_use_central_platform_policy(self) -> None: platform_slices = { "macos": ( "macos-15", @@ -78,7 +78,25 @@ def test_main_macos_and_windows_slices_have_fixed_platforms(self) -> None: for name in names: with self.subTest(platform=platform, workflow=name): workflow = self.workflow(name) - self.assertIn(f"runs-on: {runner}", workflow) + output = "runner_macos" if platform == "macos" else "runner_windows" + self.assertIn( + f"{output}: ${{{{ steps.policy.outputs.{output} }}}}", + workflow, + ) + self.assertIn( + f"runs-on: ${{{{ needs.runner_policy.outputs.{output} }}}}", + workflow, + ) + self.assertIn( + "depot_pr_enabled: ${{ vars.DEPOT_PR_RUNNERS_ENABLED == 'true' }}", + workflow, + ) + self.assertIn( + "head_repository: ${{ github.event.pull_request.head.repo.full_name }}", + workflow, + ) + selector = (ROOT / ".github" / "actions" / "select-ci-runners" / "action.yml").read_text(encoding="utf-8") + self.assertIn(f"depot-{runner}", selector) self.assertNotIn("macos-latest", workflow) self.assertNotIn("windows-latest", workflow) @@ -146,15 +164,45 @@ def test_depot_allowlist_excludes_credential_smokes(self) -> None: self.assertIn("ci-linux-runtime-slice.yml@refs/heads/main", allowlist) self.assertIn("depot-canary.yml@refs/heads/main", allowlist) self.assertIn("release.yml@refs/heads/main", allowlist) + for name in ( + "ci-quality-slice.yml", + "ci-web-slice.yml", + "ci-ui-artifact-slice.yml", + "ci-linux-host-slice.yml", + "ci-linux-runtime-slice.yml", + "ci-linux-product-slice.yml", + "ci-rust-tests-slice.yml", + "ci-macos-host-slice.yml", + "ci-macos-runtime-slice.yml", + "ci-macos-product-slice.yml", + "ci-windows-host-slice.yml", + "ci-windows-runtime-slice.yml", + "ci-windows-product-slice.yml", + "ci-platform-checks-slice.yml", + "native-sdk-artifact.yml", + "static-abi-artifact.yml", + "swift-sdk-artifact.yml", + ): + with self.subTest(workflow=name): + self.assertIn(f"{name}@refs/heads/main", allowlist) for name in ( "hf-download-smoke.yml", "smoke.yml", "scripted-binary-smoke.yml", "sdk-smoke.yml", - "swift-sdk-artifact.yml", ): with self.subTest(workflow=name): self.assertNotIn(name, allowlist) + self.assertIn( + "swift-sdk-artifact.yml@refs/heads/main", + allowlist, + ) + swift = self.workflow("swift-sdk-artifact.yml") + self.assertIn("depot_main_enabled: 'false'", swift) + self.assertIn( + "depot_pr_enabled: ${{ vars.DEPOT_PR_RUNNERS_ENABLED == 'true' }}", + swift, + ) def test_pr_entrypoint_maps_no_repository_secret(self) -> None: for lane in ("quality", "website", "linux", "macos", "windows"): diff --git a/tools/xtask/src/workflow_checks.rs b/tools/xtask/src/workflow_checks.rs index fe08ee5ed8..5546e63475 100644 --- a/tools/xtask/src/workflow_checks.rs +++ b/tools/xtask/src/workflow_checks.rs @@ -481,37 +481,38 @@ fn check_protected_reusable_runner_policy(workflow: &str, context: &str) -> DynR ("runner_policy:", "protected runner policy job"), ("runs-on: ubuntu-24.04", "fixed hosted policy runner"), ( - "POLICY_REPOSITORY: ${{ github.repository }}", + "uses: ./.github/actions/select-ci-runners", + "central protected runner selector", + ), + ( + "repository: ${{ github.repository }}", "immutable repository context", ), - ("POLICY_REF: ${{ github.ref }}", "immutable ref context"), ( - "POLICY_EVENT_NAME: ${{ github.event.inputs.original_event_name || github.event_name }}", - "protected original event context", + "head_repository: ${{ github.event.pull_request.head.repo.full_name }}", + "same-repository PR head context", ), + ("ref: ${{ github.ref }}", "immutable ref context"), ( - "POLICY_DEPOT_ENABLED: ${{ vars.DEPOT_RUNNERS_ENABLED == 'true' }}", - "repository Depot gate", + "original_event_name: ${{ inputs.original_event_name }}", + "protected original event context", ), ( - "POLICY_MANUAL_USE_DEPOT: ${{ inputs.use_depot }}", - "typed main-dispatch canary flag", + "depot_main_enabled: ${{ vars.DEPOT_RUNNERS_ENABLED == 'true' }}", + "repository Depot gate", ), ( - r#"POLICY_REPOSITORY" == "Mesh-LLM/mesh-llm""#, - "exact repository guard", + "depot_pr_enabled: ${{ vars.DEPOT_PR_RUNNERS_ENABLED == 'true' }}", + "repository PR Depot gate", ), ( - r#"POLICY_REF" == "refs/heads/main""#, - "exact main-ref guard", + "manual_use_depot: ${{ inputs.use_depot }}", + "typed main-dispatch canary flag", ), ( - r#"POLICY_MANUAL_USE_DEPOT" == "true""#, - "main-dispatch canary decision", + "runner_size must be one of: default, 4, 8, 16", + "bounded runner-size validation", ), - ("default|4|8|16", "bounded runner-size validation"), - ("depot-ubuntu-24.04", "allowlisted Depot AMD64 label"), - ("depot-ubuntu-24.04-arm", "allowlisted Depot ARM64 label"), ( "runs-on: ${{ needs.runner_policy.outputs.runner }}", "derived producer runner",