diff --git a/.github/workflows/hosted-runner-recovery.yaml b/.github/workflows/hosted-runner-recovery.yaml index b7d4500a532..079e3e0babc 100644 --- a/.github/workflows/hosted-runner-recovery.yaml +++ b/.github/workflows/hosted-runner-recovery.yaml @@ -8,9 +8,7 @@ run-name: Hosted runner recovery for source run ${{ github.event.workflow_run.id on: workflow_run: workflows: - - E2E / WSL - - E2E / macOS - - CI / Platform Vitest Main Watch + - CI / Platform Evidence types: - completed @@ -28,11 +26,7 @@ jobs: github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.head_repository.full_name == 'NVIDIA/NemoClaw' && github.event.workflow_run.event == 'push' && - ( - github.event.workflow_run.path == '.github/workflows/wsl-e2e.yaml' || - github.event.workflow_run.path == '.github/workflows/macos-e2e.yaml' || github.event.workflow_run.path == '.github/workflows/platform-vitest-main.yaml' - ) }} concurrency: group: hosted-runner-recovery-${{ github.event.workflow_run.workflow_id }} diff --git a/.github/workflows/macos-e2e.yaml b/.github/workflows/macos-e2e.yaml deleted file mode 100644 index 214d99376c1..00000000000 --- a/.github/workflows/macos-e2e.yaml +++ /dev/null @@ -1,114 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -name: E2E / macOS - -on: - workflow_dispatch: - push: - branches: - - main - paths-ignore: - - "docs/**" - - "**/*.md" - - ".github/workflows/docs-preview-*.yaml" - - "ISSUE_TEMPLATE/**" - - ".github/ISSUE_TEMPLATE/**" - -permissions: - contents: read - -concurrency: - group: macos-e2e-${{ github.ref }} - cancel-in-progress: true - -jobs: - macos-e2e: - runs-on: macos-26 - timeout-minutes: 30 - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Setup Node.js - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: "22" - cache: npm - - - name: Show environment - run: | - set -euo pipefail - echo "Runner: $(uname -a)" - echo "Arch: $(uname -m)" - sw_vers - node --version - npm --version - - - name: Install root dependencies - run: npm ci --ignore-scripts - - - name: Build CLI TypeScript modules - run: npm run build:cli - - - name: Install and build plugin - run: | - set -euo pipefail - cd nemoclaw - npm ci --ignore-scripts - npm run build - - - name: Run gateway lifecycle regressions - run: >- - npx vitest run --project integration - test/tunnel-gateway-port-release-runtime.test.ts - test/onboard-gateway-prelaunch-cutover.test.ts - test/onboard-gateway-legacy-identity-upgrade-runtime.test.ts - - - name: Detect Docker availability - id: docker - run: | - set -euo pipefail - if docker info >/dev/null 2>&1; then - echo "docker_ok=true" >> "$GITHUB_OUTPUT" - echo "Docker is available" - docker version - exit 0 - fi - echo "docker_ok=false" >> "$GITHUB_OUTPUT" - echo "Docker is unavailable on the Apple Silicon runner." - - - name: Run macOS full E2E - if: steps.docker.outputs.docker_ok == 'true' && github.ref == 'refs/heads/main' - env: - NVIDIA_INFERENCE_API_KEY: ${{ github.ref == 'refs/heads/main' && secrets.NVIDIA_INFERENCE_API_KEY || '' }} - GITHUB_TOKEN: ${{ github.token }} - NEMOCLAW_NON_INTERACTIVE: "1" - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" - NEMOCLAW_RECREATE_SANDBOX: "1" - NEMOCLAW_SANDBOX_NAME: "e2e-macos" - run: | - NEMOCLAW_RUN_LIVE_E2E=1 npx vitest run --project e2e-live test/e2e/live/full-e2e.test.ts --silent=false --reporter=default - - - name: Explain skipped macOS live E2E - if: steps.docker.outputs.docker_ok != 'true' || github.ref != 'refs/heads/main' - run: | - if [ "${{ github.ref }}" != "refs/heads/main" ]; then - echo 'Skipping secret-bearing macOS live E2E outside the trusted main branch.' - elif [ "${{ steps.docker.outputs.docker_ok }}" != "true" ]; then - echo 'Skipping macOS live E2E because Docker is unavailable on this runner.' - fi - echo 'The workflow still validated the NemoClaw build on macOS (Apple Silicon).' - - - name: Upload logs on failure - if: failure() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: macos-e2e-logs - path: | - /tmp/nemoclaw-e2e-*.log - ${{ github.workspace }}/e2e-artifacts/live - if-no-files-found: ignore - retention-days: 14 diff --git a/.github/workflows/platform-vitest-main.yaml b/.github/workflows/platform-vitest-main.yaml index f812ebc9dc6..50d1488fb3a 100644 --- a/.github/workflows/platform-vitest-main.yaml +++ b/.github/workflows/platform-vitest-main.yaml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -name: CI / Platform Vitest Main Watch +name: CI / Platform Evidence on: workflow_dispatch: @@ -19,7 +19,7 @@ permissions: contents: read concurrency: - group: platform-vitest-main-${{ github.ref }} + group: platform-evidence-${{ github.ref }} cancel-in-progress: true jobs: @@ -81,12 +81,21 @@ jobs: test/install-preflight.test.ts macos-vitest: + name: macOS compatibility (${{ matrix.shard }}/4) runs-on: macos-26 - timeout-minutes: 30 + timeout-minutes: ${{ matrix.timeout_minutes }} strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4] + include: + - shard: 1 + timeout_minutes: 60 + - shard: 2 + timeout_minutes: 30 + - shard: 3 + timeout_minutes: 30 + - shard: 4 + timeout_minutes: 30 steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -157,17 +166,82 @@ jobs: - name: Run full Vitest suite on macOS run: npx vitest run --testTimeout 60000 --shard="${{ matrix.shard }}/4" + - id: macos_docker + name: Detect Docker availability for macOS E2E + if: ${{ matrix.shard == 1 }} + shell: bash + run: | + set -euo pipefail + if docker info >/dev/null 2>&1; then + echo "docker_ok=true" >>"$GITHUB_OUTPUT" + docker version + else + echo "docker_ok=false" >>"$GITHUB_OUTPUT" + echo "Docker is unavailable on the Apple silicon runner." + fi + + - name: Run macOS live E2E + if: ${{ matrix.shard == 1 && steps.macos_docker.outputs.docker_ok == 'true' && github.ref == 'refs/heads/main' }} + env: + GITHUB_TOKEN: ${{ github.token }} + NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_RECREATE_SANDBOX: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-macos" + NEMOCLAW_RUN_LIVE_E2E: "1" + run: >- + npx vitest run --project e2e-live + test/e2e/live/full-e2e.test.ts + --silent=false --reporter=default + + - name: Explain skipped macOS live E2E + if: ${{ matrix.shard == 1 && (steps.macos_docker.outputs.docker_ok != 'true' || github.ref != 'refs/heads/main') }} + shell: bash + run: | + set -euo pipefail + if [[ "$GITHUB_REF" != "refs/heads/main" ]]; then + echo "The macOS live E2E did not run because this workflow does not test main." + else + echo "The macOS live E2E did not run because Docker is unavailable." + fi + echo "The macOS build and Vitest shard still provide platform evidence." + + - name: Upload macOS E2E logs on failure + if: ${{ matrix.shard == 1 && failure() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: macos-platform-evidence-logs + path: | + /tmp/nemoclaw-e2e-*.log + ${{ github.workspace }}/e2e-artifacts/live + if-no-files-found: ignore + retention-days: 14 + wsl-vitest: + name: WSL compatibility (${{ matrix.shard }}/4) runs-on: windows-latest - timeout-minutes: 90 + timeout-minutes: ${{ matrix.timeout_minutes }} strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4] + include: + - shard: 1 + timeout_minutes: 180 + - shard: 2 + timeout_minutes: 90 + - shard: 3 + timeout_minutes: 90 + - shard: 4 + timeout_minutes: 90 env: WSL_DISTRO: Ubuntu WSL_TEST_USER: nemoclaw-ci TRUSTED_WSL_HELPER: ${{ github.workspace }}\trusted-wsl-ci\tools\wsl\ci-helper.ps1 + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_RECREATE_SANDBOX: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-wsl" steps: - name: Force LF line endings for checkout shell: powershell @@ -297,3 +371,71 @@ jobs: -t 'requires both fixed files to match|reclaims a root-owned collapsed config|leaves a root-owned recovery baseline untouched' "@ Invoke-WslScript -Distro $env:WSL_DISTRO -User root -Script $script + + - id: wsl_docker + name: Detect Docker availability in WSL + if: ${{ matrix.shard == 1 }} + shell: powershell + run: | + . "$env:TRUSTED_WSL_HELPER" + $script = @' + if docker info >/dev/null 2>&1; then + echo DOCKER_OK=1 + else + echo DOCKER_OK=0 + fi + '@ + $result = Invoke-WslScript -Distro $env:WSL_DISTRO -Script $script -CaptureOutput + if ($result -match 'DOCKER_OK=1') { + 'docker_ok=true' | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append + Write-Host 'Docker is available in WSL' + } else { + 'docker_ok=false' | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append + Write-Host 'Docker is unavailable in WSL' + } + + - name: Run WSL live E2E + if: ${{ matrix.shard == 1 && steps.wsl_docker.outputs.docker_ok == 'true' && github.ref == 'refs/heads/main' }} + shell: powershell + env: + GITHUB_TOKEN: ${{ github.token }} + NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + run: | + . "$env:TRUSTED_WSL_HELPER" + $workdir = ConvertTo-BashLiteral -Value $env:WSL_WORKDIR + $exports = @( + 'export NVIDIA_INFERENCE_API_KEY=' + (ConvertTo-BashLiteral -Value ([string]$env:NVIDIA_INFERENCE_API_KEY)) + 'export GITHUB_TOKEN=' + (ConvertTo-BashLiteral -Value ([string]$env:GITHUB_TOKEN)) + 'export NEMOCLAW_NON_INTERACTIVE=' + (ConvertTo-BashLiteral -Value $env:NEMOCLAW_NON_INTERACTIVE) + 'export NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=' + (ConvertTo-BashLiteral -Value $env:NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE) + 'export NEMOCLAW_RECREATE_SANDBOX=' + (ConvertTo-BashLiteral -Value $env:NEMOCLAW_RECREATE_SANDBOX) + 'export NEMOCLAW_SANDBOX_NAME=' + (ConvertTo-BashLiteral -Value $env:NEMOCLAW_SANDBOX_NAME) + ) -join "`n" + $script = @" + set -euo pipefail + cd $workdir + $exports + export NEMOCLAW_RUN_LIVE_E2E=1 + npx vitest run --project e2e-live test/e2e/live/full-e2e.test.ts --silent=false --reporter=default + "@ + Invoke-WslScript -Distro $env:WSL_DISTRO -Script $script + + - name: Explain skipped WSL live E2E + if: ${{ matrix.shard == 1 && (steps.wsl_docker.outputs.docker_ok != 'true' || github.ref != 'refs/heads/main') }} + shell: powershell + run: | + if ($env:GITHUB_REF -ne 'refs/heads/main') { + Write-Host 'The WSL live E2E did not run because this workflow does not test main.' + } else { + Write-Host 'The WSL live E2E did not run because Docker is unavailable.' + } + Write-Host 'The WSL build and Vitest shard still provide platform evidence.' + + - name: Upload WSL E2E log on failure + if: ${{ matrix.shard == 1 && failure() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: wsl-platform-evidence-install-log + path: C:\Users\runneradmin\AppData\Local\Temp\nemoclaw-e2e-install.log + if-no-files-found: ignore + retention-days: 14 diff --git a/.github/workflows/wsl-e2e.yaml b/.github/workflows/wsl-e2e.yaml deleted file mode 100644 index 0a55683284d..00000000000 --- a/.github/workflows/wsl-e2e.yaml +++ /dev/null @@ -1,177 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -name: E2E / WSL - -on: - workflow_dispatch: - push: - branches: - - main - -permissions: - contents: read - -concurrency: - group: wsl-e2e-${{ github.ref }} - cancel-in-progress: true - -jobs: - wsl-e2e: - runs-on: windows-latest - timeout-minutes: 90 - env: - WSL_DISTRO: Ubuntu - TRUSTED_WSL_HELPER: ${{ github.workspace }}\trusted-wsl-ci\tools\wsl\ci-helper.ps1 - NEMOCLAW_NON_INTERACTIVE: "1" - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" - NEMOCLAW_RECREATE_SANDBOX: "1" - NEMOCLAW_SANDBOX_NAME: "e2e-wsl" - steps: - - name: Force LF line endings for checkout - shell: powershell - run: git config --global core.autocrlf false - - - name: Check out the trusted WSL helper - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.workflow_sha }} - path: trusted-wsl-ci - persist-credentials: false - sparse-checkout: | - tools/wsl/ci-helper.ps1 - sparse-checkout-cone-mode: false - - - name: Check out candidate source - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - path: source - persist-credentials: false - - - name: Resolve workspace paths for WSL - shell: powershell - run: | - . "$env:TRUSTED_WSL_HELPER" - $null = Set-WslWorkflowPaths ` - -Workspace "$env:GITHUB_WORKSPACE\source" ` - -WorkdirPrefix '/tmp/nemoclaw-wsl-workdir' ` - -RunId $env:GITHUB_RUN_ID ` - -RunAttempt $env:GITHUB_RUN_ATTEMPT ` - -EnvironmentFile $env:GITHUB_ENV - - - name: Ensure Ubuntu WSL exists - shell: powershell - run: | - . "$env:TRUSTED_WSL_HELPER" - Ensure-WslDistro -Distro $env:WSL_DISTRO - - - name: Verify WSL - shell: powershell - run: | - . "$env:TRUSTED_WSL_HELPER" - Invoke-WslScript -Distro $env:WSL_DISTRO -Script "uname -a`ncat /etc/os-release" - - - name: Install Ubuntu dependencies - shell: powershell - run: | - . "$env:TRUSTED_WSL_HELPER" - $packages = @( - 'bash', 'ca-certificates', 'curl', 'git', 'jq', 'lsb-release', 'make', - 'python3', 'python3-pip', 'rsync', 'tar', 'unzip', 'xz-utils' - ) - Install-WslUbuntuDependencies -Distro $env:WSL_DISTRO -Packages $packages - - - name: Install Node.js 22 in WSL - shell: powershell - run: | - . "$env:TRUSTED_WSL_HELPER" - Install-WslNode -Distro $env:WSL_DISTRO - - - name: Copy checkout into WSL ext4 workspace - shell: powershell - run: | - . "$env:TRUSTED_WSL_HELPER" - Sync-WslCheckout ` - -Distro $env:WSL_DISTRO ` - -Checkout $env:WSL_CHECKOUT_DIR ` - -Workdir $env:WSL_WORKDIR - - - name: Install project dependencies and build plugin - shell: powershell - run: | - . "$env:TRUSTED_WSL_HELPER" - $workdir = ConvertTo-BashLiteral -Value $env:WSL_WORKDIR - $script = @" - set -euo pipefail - cd $workdir - npm install --ignore-scripts - npm run build:cli - cd nemoclaw - npm install --ignore-scripts - npm run build - "@ - Invoke-WslScript -Distro $env:WSL_DISTRO -Script $script - - - name: Detect Docker availability in WSL - id: docker - shell: powershell - run: | - . "$env:TRUSTED_WSL_HELPER" - $script = @' - if docker info >/dev/null 2>&1; then - echo DOCKER_OK=1 - else - echo DOCKER_OK=0 - fi - '@ - $result = Invoke-WslScript -Distro $env:WSL_DISTRO -Script $script -CaptureOutput - if ($result -match 'DOCKER_OK=1') { - 'docker_ok=true' | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append - Write-Host 'Docker is available in WSL' - } else { - 'docker_ok=false' | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append - Write-Host 'Docker is not available in WSL; full E2E will be skipped' - } - - - name: Run WSL full E2E - if: steps.docker.outputs.docker_ok == 'true' - shell: powershell - env: - NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} - GITHUB_TOKEN: ${{ github.token }} - run: | - . "$env:TRUSTED_WSL_HELPER" - $workdir = ConvertTo-BashLiteral -Value $env:WSL_WORKDIR - $exports = @( - 'export NVIDIA_INFERENCE_API_KEY=' + (ConvertTo-BashLiteral -Value ([string]$env:NVIDIA_INFERENCE_API_KEY)) - 'export GITHUB_TOKEN=' + (ConvertTo-BashLiteral -Value ([string]$env:GITHUB_TOKEN)) - 'export NEMOCLAW_NON_INTERACTIVE=' + (ConvertTo-BashLiteral -Value $env:NEMOCLAW_NON_INTERACTIVE) - 'export NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=' + (ConvertTo-BashLiteral -Value $env:NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE) - 'export NEMOCLAW_RECREATE_SANDBOX=' + (ConvertTo-BashLiteral -Value $env:NEMOCLAW_RECREATE_SANDBOX) - 'export NEMOCLAW_SANDBOX_NAME=' + (ConvertTo-BashLiteral -Value $env:NEMOCLAW_SANDBOX_NAME) - ) -join "`n" - $script = @" - set -euo pipefail - cd $workdir - $exports - export NEMOCLAW_RUN_LIVE_E2E=1 - npx vitest run --project e2e-live test/e2e/live/full-e2e.test.ts --silent=false --reporter=default - "@ - Invoke-WslScript -Distro $env:WSL_DISTRO -Script $script - - - name: Explain skipped full E2E - if: steps.docker.outputs.docker_ok != 'true' - shell: powershell - run: | - Write-Host 'Skipping WSL full E2E because Docker is unavailable on this runner.' - Write-Host 'The workflow still validated the NemoClaw build flow inside Ubuntu WSL.' - - - name: Upload install log on failure - if: failure() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: wsl-e2e-install-log - path: | - C:\Users\runneradmin\AppData\Local\Temp\nemoclaw-e2e-install.log - if-no-files-found: ignore - retention-days: 14 diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 6c97be295c9..3ea56bb7448 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -258,7 +258,7 @@ }, { "file": "test/hosted-runner-recovery-workflow.test.ts", - "test": "locks recovery identities to the source workflows' runtime names (#7140)", + "test": "locks recovery to the platform workflow name (#7140)", "category": "security" }, { diff --git a/src/lib/onboard/preflight.test.ts b/src/lib/onboard/preflight.test.ts index c59df0e9b93..faf6f8d7144 100644 --- a/src/lib/onboard/preflight.test.ts +++ b/src/lib/onboard/preflight.test.ts @@ -509,7 +509,7 @@ describe("assessHost", () => { // // The fixtures here explicitly pin `release` and override `readFileImpl` // for /proc/version so the underlying `detectWsl` heuristic does not - // pick up the test runner's actual environment (e.g. the wsl-e2e job + // pick up the test runner's actual environment (e.g. the platform WSL job // running on real WSL would otherwise see kernel 5.15.x-microsoft-WSL // and flip isWsl true, gating off the conflict). it("flags Docker 26+ containerd-snapshotter overlayfs as a nested overlay conflict", () => { diff --git a/test/e2e/README.md b/test/e2e/README.md index 32c3f1f3e66..849d054424e 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -22,10 +22,13 @@ before those targets run; local runners must provide it themselves. attempts, requests at most two failed-job reruns, and uploads attempt evidence. - The `staging-brev-launchable` job in `.github/workflows/e2e.yaml` validates the baked candidate without installing or copying NemoClaw source. -- `.github/workflows/macos-e2e.yaml`, `.github/workflows/wsl-e2e.yaml`, and - `.github/workflows/sandbox-images-and-e2e.yaml` call focused E2E targets directly. - `.github/workflows/e2e.yaml` selects free-standing jobs, including - `whatsapp-qr-compact` and `ollama-auth-proxy`. +- `.github/workflows/platform-vitest-main.yaml` publishes `CI / Platform Evidence` for Ubuntu 26.04, macOS, and WSL. + On shard 1, its macOS and WSL live E2E run only when the workflow tests `main` and Docker is available. + This workflow does not publish or satisfy `Release qualification`. +- `.github/workflows/portable-profile-e2e.yaml` publishes experimental portable-profile evidence. +- `.github/workflows/podman-cpu-proof.yaml` publishes PR-only experimental runtime evidence. +- `.github/workflows/sandbox-images-and-e2e.yaml` provides reusable sandbox-image build and test evidence. + `.github/workflows/e2e.yaml` selects free-standing jobs, including `whatsapp-qr-compact` and `ollama-auth-proxy`. ## CI execution shape @@ -143,12 +146,25 @@ The former top-level `test/e2e/test-*.sh` suite has been removed. Keep real shell, installer, process, Docker, OpenShell, `/proc`, and sandbox boundaries in E2E tests when those boundaries are the behavior under test. -## Platform Vitest main watch +## Platform Evidence -`.github/workflows/platform-vitest-main.yaml` runs the full Vitest suite in -four independent shards on each of macOS and WSL, with `fail-fast` disabled. -Each macOS shard has a 30-minute budget and each WSL shard has a 90-minute -budget. The additional root-required WSL contracts run only on shard 1. +`.github/workflows/platform-vitest-main.yaml` publishes the `CI / Platform Evidence` workflow. +It runs the Ubuntu 26.04 compatibility contracts and the full Vitest suite in four shards on macOS and WSL. +The matrix disables `fail-fast`. +The first macOS shard has a 60-minute budget for live E2E; the other shards have 30 minutes. +The first WSL shard has a 180-minute budget for root-required contracts and live E2E; the other shards have 90 minutes. + +On shard 1, the workflow runs focused macOS and WSL live E2E only when the run tests `main` and Docker is available. +Otherwise, the workflow records the skip and retains the platform contract evidence. +Therefore, the workflow is platform evidence, not `Release qualification`. +Only a full manual `.github/workflows/e2e.yaml` run can publish the release check. + +The live steps give candidate test code the job-scoped `GITHUB_TOKEN` and repository `NVIDIA_INFERENCE_API_KEY`. +The macOS step sets both in its process environment. +The WSL step uses the trusted PowerShell helper to forward both into the WSL test process. +The workflow sets these credentials only for the live steps, but candidate code can copy either value while a step runs. +GitHub invalidates `GITHUB_TOKEN` after the job. +`NVIDIA_INFERENCE_API_KEY` remains valid until it expires or is revoked; the workflow does not revoke it. ## Retired Brev source-install coverage @@ -625,12 +641,10 @@ pre-tag E2E denominator. ### Hosted-Runner Recovery -Hosted Runner Recovery can request one full rerun for eligible WSL, macOS, and -platform-watch push runs. It does not handle `E2E main`. The complete non-passing -job listing must contain only authenticated hosted-runner-loss evidence for the -workflow's approved runner labels. An ordinary assertion failure, mixed failure -set, incomplete listing, custom or self-hosted label, changed evidence, or -ambiguous pagination prevents recovery. +Hosted Runner Recovery can request one full rerun for an eligible `CI / Platform Evidence` push. +It does not handle `E2E main`. +The complete non-passing job listing must contain only authenticated hosted-runner-loss evidence for the workflow's approved runner labels. +An ordinary assertion failure, mixed failure set, incomplete listing, custom or self-hosted label, changed evidence, or ambiguous pagination prevents recovery. For eligible `E2E main` push runs, `E2E / Main Retry` asks GitHub Actions to rerun failed jobs and their dependent jobs. A successful CLI artifact producer is not rerun. @@ -1022,9 +1036,10 @@ The Actions run is advisory for the pull request and is not a required merge con Treat it as passing evidence only when the `E2E` workflow concludes with `success` for the recorded PR number, PR source repository, candidate commit SHA, base commit SHA, and trusted workflow SHA. A changed PR source repository, candidate commit SHA, or base commit SHA invalidates the evidence and requires a new manual run. -The macOS and WSL workflows also run on configured pushes to `main`. -The portable-profile workflow runs on `main` when one of its configured paths changes. -Their manual dispatch paths remain available for branch diagnosis. +The platform-evidence workflow runs on configured pushes to `main` and supports manual dispatch for branch diagnosis. +The experimental portable-profile workflow runs on `main` when one of its configured paths changes. +The Podman CPU proof runs only for matching pull request changes. +The sandbox-image workflow accepts manual and reusable workflow calls for image build and test evidence. ## Onboard performance budget diff --git a/test/e2e/docs/README.md b/test/e2e/docs/README.md index 2b9ce04a8d4..ccfd7c073c7 100644 --- a/test/e2e/docs/README.md +++ b/test/e2e/docs/README.md @@ -309,15 +309,24 @@ test/e2e/ its result counts to the expected and tested candidate SHA, correlation ID, job ID, and shard ID. The workflow boundary requires every selected job shard to upload its evidence artifact. -- `.github/workflows/platform-vitest-main.yaml` runs the full Vitest suite in - four independent shards on each of macOS and WSL, with `fail-fast` disabled. - Each macOS shard installs the pinned OpenShell formula and has a 30-minute - budget. Each WSL shard has a 90-minute budget, and WSL runs its additional - root-required contracts on shard 1 only. - `.github/workflows/macos-e2e.yaml`, `.github/workflows/wsl-e2e.yaml`, and - `.github/workflows/sandbox-images-and-e2e.yaml` call focused E2E targets - directly. `.github/workflows/e2e.yaml` selects free-standing jobs, including - `whatsapp-qr-compact` and `ollama-auth-proxy`. +- `.github/workflows/platform-vitest-main.yaml` publishes `CI / Platform Evidence`. + It runs the Ubuntu 26.04 compatibility contracts and four full-suite Vitest shards on each of macOS and WSL. + Each macOS shard installs the pinned OpenShell formula. + Shard 1 has a 60-minute budget for live E2E; the other shards have 30 minutes. + WSL shard 1 has a 180-minute budget for root-required contracts and live E2E; the other shards have 90 minutes. + On shard 1, the workflow runs focused macOS and WSL live E2E only when the run tests `main` and Docker is available. + Otherwise, those live tests skip and the platform contracts remain as evidence. + This conditional result is platform evidence, not `Release qualification`. + The live steps give candidate test code the job-scoped `GITHUB_TOKEN` and repository `NVIDIA_INFERENCE_API_KEY`. + The macOS step sets both in its process environment. + The WSL step uses the trusted PowerShell helper to forward both into the WSL test process. + The workflow sets these credentials only for the live steps, but candidate code can copy either value while a step runs. + GitHub invalidates `GITHUB_TOKEN` after the job. + `NVIDIA_INFERENCE_API_KEY` remains valid until it expires or is revoked; the workflow does not revoke it. +- `.github/workflows/portable-profile-e2e.yaml` provides experimental portable-profile evidence on matching `main` changes or manual dispatches. +- `.github/workflows/podman-cpu-proof.yaml` provides PR-only experimental runtime evidence with Docker disabled. +- `.github/workflows/sandbox-images-and-e2e.yaml` provides reusable image build and test evidence through manual dispatch and `workflow_call`. + `.github/workflows/e2e.yaml` selects free-standing jobs, including `whatsapp-qr-compact` and `ollama-auth-proxy`. - The `staging-brev-launchable` job validates the exact baked candidate in preinstalled mode. Generic Brev VMs with source overlays are not a qualification boundary. diff --git a/test/helpers/vitest-watch-triggers.ts b/test/helpers/vitest-watch-triggers.ts index 435541a6773..accc3bbe00c 100644 --- a/test/helpers/vitest-watch-triggers.ts +++ b/test/helpers/vitest-watch-triggers.ts @@ -180,12 +180,12 @@ export const vitestWatchTriggerPatterns: VitestWatchTriggerPattern[] = [ }, { pattern: - /(?:^|\/)\.github\/workflows\/(?:hosted-runner-recovery|wsl-e2e|macos-e2e|platform-vitest-main)\.yaml$/, + /(?:^|\/)\.github\/workflows\/(?:hosted-runner-recovery|platform-vitest-main)\.yaml$/, testsToRun: runTests("test/hosted-runner-recovery-workflow.test.ts"), }, { pattern: - /(?:^|\/)(?:\.github\/workflows\/(?:platform-vitest-main|wsl-e2e)\.yaml|tools\/wsl\/ci-helper\.ps1)$/, + /(?:^|\/)(?:\.github\/workflows\/platform-vitest-main\.yaml|tools\/wsl\/ci-helper\.ps1)$/, testsToRun: runTests( "test/platform-vitest-main-workflow.test.ts", "test/wsl-ci-helper.test.ts", diff --git a/test/hosted-runner-recovery-workflow.test.ts b/test/hosted-runner-recovery-workflow.test.ts index 327236e8bb7..df0ba8613f4 100644 --- a/test/hosted-runner-recovery-workflow.test.ts +++ b/test/hosted-runner-recovery-workflow.test.ts @@ -5,14 +5,9 @@ import { describe, expect, it } from "vitest"; import { readYaml, type WorkflowJob, type WorkflowStep } from "./helpers/e2e-workflow-contract.ts"; const WORKFLOW_PATH = ".github/workflows/hosted-runner-recovery.yaml"; -const E2E_WORKFLOW_PATH = ".github/workflows/e2e.yaml"; -const WSL_WORKFLOW_PATH = ".github/workflows/wsl-e2e.yaml"; -const MACOS_WORKFLOW_PATH = ".github/workflows/macos-e2e.yaml"; const PLATFORM_WORKFLOW_PATH = ".github/workflows/platform-vitest-main.yaml"; const TRUSTED_CHECKOUT = "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1"; const TRUSTED_SETUP_NODE = "actions/setup-node@820762786026740c76f36085b0efc47a31fe5020"; -const E2E_RUN_NAME = - "${{ inputs.checkout_sha != '' && format('E2E PR #{0} ({1})', inputs.pr_number, inputs.correlation_id) || inputs.correlation_id != '' && format('E2E {0} ({1})', github.ref_name, inputs.correlation_id) || format('E2E {0}', github.ref_name) }}"; type RecoveryWorkflow = { name: string; @@ -59,12 +54,12 @@ function collectStrings(value: unknown): string[] { } describe("hosted-runner recovery workflow boundary", () => { - it("subscribes only to completed runs from the three platform workflows (#7140)", () => { + it("subscribes only to completed platform-evidence runs (#7140)", () => { const value = workflow(); expect(value.name).toBe("Hosted Runner Recovery"); expect(value.on).toEqual({ workflow_run: { - workflows: ["E2E / WSL", "E2E / macOS", "CI / Platform Vitest Main Watch"], + workflows: ["CI / Platform Evidence"], types: ["completed"], }, }); @@ -78,28 +73,16 @@ describe("hosted-runner recovery workflow boundary", () => { expect(Object.keys(value.jobs)).toEqual(["recover"]); }); - // source-shape-contract: security -- Exact source workflow names and run-name expressions keep the write-capable recovery subscription bound to reviewed trusted-main identities - it("locks recovery identities to the source workflows' runtime names (#7140)", () => { - const e2e = sourceWorkflow(E2E_WORKFLOW_PATH); - const wsl = sourceWorkflow(WSL_WORKFLOW_PATH); - const macos = sourceWorkflow(MACOS_WORKFLOW_PATH); + // source-shape-contract: security -- The exact source workflow name keeps the write-capable recovery subscription bound to the reviewed trusted-main identity + it("locks recovery to the platform workflow name (#7140)", () => { const platform = sourceWorkflow(PLATFORM_WORKFLOW_PATH); - expect(e2e).toMatchObject({ name: "E2E", "run-name": E2E_RUN_NAME }); - expect(E2E_RUN_NAME).toContain("inputs.correlation_id != ''"); - expect(E2E_RUN_NAME).toContain("format('E2E {0}', github.ref_name)"); - expect([wsl.name, macos.name, platform.name]).toEqual([ - "E2E / WSL", - "E2E / macOS", - "CI / Platform Vitest Main Watch", - ]); - expect(wsl).not.toHaveProperty("run-name"); - expect(macos).not.toHaveProperty("run-name"); + expect(platform.name).toBe("CI / Platform Evidence"); expect(platform).not.toHaveProperty("run-name"); - expect(workflow().on.workflow_run.workflows).toEqual([wsl.name, macos.name, platform.name]); + expect(workflow().on.workflow_run.workflows).toEqual([platform.name]); }); - it("fails closed on controller, source, repository, branch, event, path, and title (#7140)", () => { + it("fails closed on controller, source, repository, branch, event, and path (#7140)", () => { const guard = workflow().jobs.recover.if ?? ""; for (const fragment of [ "github.run_attempt == 1", @@ -109,8 +92,6 @@ describe("hosted-runner recovery workflow boundary", () => { "github.event.workflow_run.conclusion == 'failure'", "github.event.workflow_run.head_branch == 'main'", "github.event.workflow_run.head_repository.full_name == 'NVIDIA/NemoClaw'", - "github.event.workflow_run.path == '.github/workflows/wsl-e2e.yaml'", - "github.event.workflow_run.path == '.github/workflows/macos-e2e.yaml'", "github.event.workflow_run.path == '.github/workflows/platform-vitest-main.yaml'", ]) { expect(guard).toContain(fragment); diff --git a/test/hosted-runner-recovery.test.ts b/test/hosted-runner-recovery.test.ts index 32f3689bdb1..a976fb71cfc 100644 --- a/test/hosted-runner-recovery.test.ts +++ b/test/hosted-runner-recovery.test.ts @@ -58,17 +58,17 @@ function sourceRun(overrides: SourceOverrides = {}) { const id = overrides.id ?? SOURCE_RUN_ID; return { id, - name: "E2E main", - path: ".github/workflows/e2e.yaml", + name: "CI / Platform Evidence", + path: ".github/workflows/platform-vitest-main.yaml", workflow_id: 304_268_429, created_at: "2026-07-25T10:00:00Z", - event: "schedule", + event: "push", head_branch: "main", head_sha: MAIN_SHA, run_attempt: 1, status: "completed", conclusion: "failure", - display_title: "E2E main", + display_title: "refactor(e2e): consolidate platform evidence", html_url: `https://github.com/${REPOSITORY}/actions/runs/${id}`, repository: { full_name: REPOSITORY }, head_repository: { full_name: REPOSITORY }, @@ -77,15 +77,12 @@ function sourceRun(overrides: SourceOverrides = {}) { } function workflowForSource(source: ReturnType) { - const names = new Map([ - [".github/workflows/e2e.yaml", "E2E"], - [".github/workflows/wsl-e2e.yaml", "E2E / WSL"], - [".github/workflows/macos-e2e.yaml", "E2E / macOS"], - [".github/workflows/platform-vitest-main.yaml", "CI / Platform Vitest Main Watch"], - ]); return { id: source.workflow_id, - name: names.get(source.path) ?? "unknown", + name: + source.path === ".github/workflows/platform-vitest-main.yaml" + ? "CI / Platform Evidence" + : "unknown", path: source.path, state: "active", }; @@ -314,64 +311,19 @@ function mutationRequests(requests: RecordedGitHubRequest[]) { describe("hosted-runner recovery controller", () => { it.each([ - { - label: "scheduled E2E", - source: sourceRun(), - job: hostedRunnerLossJob(), - }, - { - label: "manually dispatched main E2E", - source: sourceRun({ event: "workflow_dispatch" }), - job: hostedRunnerLossJob(), - }, - { - label: "WSL main push", - source: sourceRun({ - name: "E2E / WSL", - path: ".github/workflows/wsl-e2e.yaml", - event: "push", - display_title: "main", - }), - job: internalErrorJob("windows-latest"), - }, - { - label: "macOS main push", - source: sourceRun({ - name: "E2E / macOS", - path: ".github/workflows/macos-e2e.yaml", - event: "push", - display_title: "main", - }), - job: internalErrorJob("macos-26"), - }, { label: "platform Ubuntu main push", - source: sourceRun({ - name: "CI / Platform Vitest Main Watch", - path: ".github/workflows/platform-vitest-main.yaml", - event: "push", - display_title: "main", - }), + source: sourceRun(), job: hostedRunnerLossJob(), }, { label: "platform Windows main push", - source: sourceRun({ - name: "CI / Platform Vitest Main Watch", - path: ".github/workflows/platform-vitest-main.yaml", - event: "push", - display_title: "main", - }), + source: sourceRun(), job: internalErrorJob("windows-latest"), }, { label: "platform macOS main push", - source: sourceRun({ - name: "CI / Platform Vitest Main Watch", - path: ".github/workflows/platform-vitest-main.yaml", - event: "push", - display_title: "main", - }), + source: sourceRun(), job: internalErrorJob("macos-26"), }, ])("requests one full rerun for exact $label runner loss (#7140)", async ({ source, job }) => { @@ -422,56 +374,20 @@ describe("hosted-runner recovery controller", () => { it.each([ { - label: "E2E on Windows", + label: "platform on a self-hosted runner", source: sourceRun(), - job: hostedRunnerLossJob({ labels: ["windows-latest"] }), + job: hostedRunnerLossJob({ labels: ["self-hosted"] }), }, { - label: "E2E on macOS", + label: "platform on an unapproved runner", source: sourceRun(), - job: hostedRunnerLossJob({ labels: ["macos-26"] }), + job: hostedRunnerLossJob({ labels: ["ubuntu-24.04"] }), }, { - label: "E2E with multiple runner labels", + label: "platform with multiple runner labels", source: sourceRun(), job: hostedRunnerLossJob({ labels: ["ubuntu-latest", "self-hosted"] }), }, - { - label: "WSL on Ubuntu", - source: sourceRun({ - name: "E2E / WSL", - path: ".github/workflows/wsl-e2e.yaml", - event: "push", - }), - job: hostedRunnerLossJob(), - }, - { - label: "WSL with a macOS internal error", - source: sourceRun({ - name: "E2E / WSL", - path: ".github/workflows/wsl-e2e.yaml", - event: "push", - }), - job: internalErrorJob("macos-26"), - }, - { - label: "macOS on Ubuntu", - source: sourceRun({ - name: "E2E / macOS", - path: ".github/workflows/macos-e2e.yaml", - event: "push", - }), - job: hostedRunnerLossJob(), - }, - { - label: "macOS with a Windows internal error", - source: sourceRun({ - name: "E2E / macOS", - path: ".github/workflows/macos-e2e.yaml", - event: "push", - }), - job: internalErrorJob("windows-latest"), - }, ])("does not broaden allowed runner labels for $label (#7140)", async ({ source, job }) => { const listing = { total_count: 1, jobs: [job] }; const requests = setupRoutes({ @@ -499,48 +415,8 @@ describe("hosted-runner recovery controller", () => { expect(requests.some((request) => request.url.includes("/git/ref/heads/main"))).toBe(false); }); - it.each([ - { - label: "WSL pull request", - source: sourceRun({ - name: "E2E / WSL", - path: ".github/workflows/wsl-e2e.yaml", - event: "pull_request", - }), - }, - { - label: "macOS manual dispatch", - source: sourceRun({ - name: "E2E / macOS", - path: ".github/workflows/macos-e2e.yaml", - event: "workflow_dispatch", - }), - }, - { - label: "platform manual dispatch", - source: sourceRun({ - name: "CI / Platform Vitest Main Watch", - path: ".github/workflows/platform-vitest-main.yaml", - event: "workflow_dispatch", - }), - }, - ])("ignores non-push platform source: $label (#7140)", async ({ source }) => { - const requests = setupRoutes({ sources: [source] }); - await expect(recoverHostedRunnerLoss(recoveryRequest())).resolves.toMatchObject({ - action: "ignored", - }); - expect(mutationRequests(requests)).toEqual([]); - }); - - it("ignores an E2E PR child run title (#7140)", async () => { - const requests = setupRoutes({ - sources: [ - sourceRun({ - event: "workflow_dispatch", - display_title: "E2E PR #42 (12345678-1234-4123-8123-123456789abc)", - }), - ], - }); + it("ignores a manually dispatched platform run (#7140)", async () => { + const requests = setupRoutes({ sources: [sourceRun({ event: "workflow_dispatch" })] }); await expect(recoverHostedRunnerLoss(recoveryRequest())).resolves.toMatchObject({ action: "ignored", }); @@ -565,20 +441,19 @@ describe("hosted-runner recovery controller", () => { expect(mutationRequests(requests)).toEqual([]); }); - it("ignores a newer E2E PR child when selecting the latest eligible main run (#7140)", async () => { + it("ignores a newer manual run when selecting the latest eligible main push (#7140)", async () => { const source = sourceRun(); - const newerPrChild = sourceRun({ + const newerManualRun = sourceRun({ id: SOURCE_RUN_ID + 100, - name: "E2E PR #42", created_at: "2026-07-25T11:00:00Z", event: "workflow_dispatch", head_sha: "e".repeat(40), conclusion: "success", - display_title: "E2E PR #42 (12345678-1234-4123-8123-123456789abc)", + display_title: "Manual platform evidence", }); const requests = setupRoutes({ sources: [source], - runListings: [{ total_count: 2, workflow_runs: [newerPrChild, source] }], + runListings: [{ total_count: 2, workflow_runs: [newerManualRun, source] }], }); await expect(recoverHostedRunnerLoss(recoveryRequest())).resolves.toMatchObject({ action: "rerun-requested", @@ -675,26 +550,25 @@ describe("hosted-runner recovery controller", () => { it("rejects source evidence that changes on the confirmation read (#7140)", async () => { const requests = setupRoutes({ - sources: [sourceRun(), sourceRun({ display_title: "E2E changed" })], + sources: [sourceRun(), sourceRun({ display_title: "changed commit title" })], }); await expect(recoverHostedRunnerLoss(recoveryRequest())).rejects.toThrow( - /identity changed during runner-loss evidence collection/u, + /evidence changed during runner-loss evidence collection/u, ); expect(mutationRequests(requests)).toEqual([]); }); it("rejects latest-run evidence that changes after job collection (#7140)", async () => { const source = sourceRun(); - const olderPrChild = sourceRun({ + const olderManualRun = sourceRun({ id: SOURCE_RUN_ID - 100, - name: "E2E PR #41", created_at: "2026-07-25T09:00:00Z", event: "workflow_dispatch", conclusion: "success", - display_title: "E2E PR #41 (12345678-1234-4123-8123-123456789abc)", + display_title: "Manual platform evidence", }); const baseline = { total_count: 1, workflow_runs: [source] }; - const changed = { total_count: 2, workflow_runs: [source, olderPrChild] }; + const changed = { total_count: 2, workflow_runs: [source, olderManualRun] }; const requests = setupRoutes({ sources: [source], runListings: [baseline, changed], @@ -707,16 +581,15 @@ describe("hosted-runner recovery controller", () => { it("rejects latest-run evidence that changes between snapshots (#7140)", async () => { const source = sourceRun(); - const olderPrChild = sourceRun({ + const olderManualRun = sourceRun({ id: SOURCE_RUN_ID - 100, - name: "E2E PR #41", created_at: "2026-07-25T09:00:00Z", event: "workflow_dispatch", conclusion: "success", - display_title: "E2E PR #41 (12345678-1234-4123-8123-123456789abc)", + display_title: "Manual platform evidence", }); const baseline = { total_count: 1, workflow_runs: [source] }; - const changed = { total_count: 2, workflow_runs: [source, olderPrChild] }; + const changed = { total_count: 2, workflow_runs: [source, olderManualRun] }; const requests = setupRoutes({ sources: [source], runListings: [baseline, baseline, changed, changed], @@ -743,7 +616,7 @@ describe("hosted-runner recovery controller", () => { }); it.each([ - ["name", { ...workflowForSource(sourceRun()), name: "E2E renamed" }], + ["name", { ...workflowForSource(sourceRun()), name: "Platform workflow renamed" }], ["path", { ...workflowForSource(sourceRun()), path: ".github/workflows/other.yaml" }], ["state", { ...workflowForSource(sourceRun()), state: "disabled_manually" }], ])("ignores source workflow metadata with the wrong %s (#7140)", async (_field, workflow) => { diff --git a/test/platform-vitest-main-workflow.test.ts b/test/platform-vitest-main-workflow.test.ts index d785e9c77cd..7f1b989ab9a 100644 --- a/test/platform-vitest-main-workflow.test.ts +++ b/test/platform-vitest-main-workflow.test.ts @@ -12,11 +12,9 @@ import { } from "./helpers/e2e-workflow-contract"; const WORKFLOW_PATH = ".github/workflows/platform-vitest-main.yaml"; -const WSL_E2E_WORKFLOW_PATH = ".github/workflows/wsl-e2e.yaml"; const WSL_HELPER_PATH = "tools/wsl/ci-helper.ps1"; const MACOS_REQUIREMENTS_PATH = "ci/platform-vitest-macos-requirements.lock"; const workflow = readYaml(WORKFLOW_PATH); -const wslE2eWorkflow = readYaml(WSL_E2E_WORKFLOW_PATH); const wslHelperSource = readRepoText(WSL_HELPER_PATH); function job(name: string): WorkflowJob { @@ -31,7 +29,7 @@ function step(jobName: string, name: string): WorkflowStep { return candidate!; } -describe("platform Vitest main workflow", () => { +describe("platform evidence workflow", () => { it("marks the container checkout safe before generating build identity", () => { const run = step("ubuntu-2604-contract", "Build CLI").run ?? ""; expect(run).toContain('git config --global --add safe.directory "$GITHUB_WORKSPACE"'); @@ -70,31 +68,16 @@ describe("platform Vitest main workflow", () => { expect(run).not.toMatch(/\bcurl\b[^\n]*\|\s*bash\b/u); } expect(step("wsl-vitest", "Install Node.js 22 in WSL").run).toContain("Install-WslNode"); - expect( - wslE2eWorkflow.jobs["wsl-e2e"]?.steps?.find( - (entry) => entry.name === "Install Node.js 22 in WSL", - )?.run, - ).toContain("Install-WslNode"); }); // source-shape-contract: security -- Sparse immutable helper checkouts must precede candidate code before root-capable WSL execution it("loads the WSL helper from trusted revisions before candidate execution (#6958)", () => { expect( (workflow as Workflow & { on?: Record }).on, - "platform main-watch workflow must not execute candidate code on pull requests", + "platform evidence workflow must not execute candidate code on pull requests", ).not.toHaveProperty("pull_request"); - const cases = [ - { - helperRef: "${{ github.workflow_sha }}", - job: job("wsl-vitest"), - }, - { - helperRef: - "${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.workflow_sha }}", - job: wslE2eWorkflow.jobs["wsl-e2e"], - }, - ]; + const cases = [{ helperRef: "${{ github.workflow_sha }}", job: job("wsl-vitest") }]; for (const workflowCase of cases) { expect(workflowCase.job, "missing WSL job").toBeDefined(); @@ -134,9 +117,6 @@ describe("platform Vitest main workflow", () => { } expect(readRepoText(WORKFLOW_PATH)).not.toMatch(/WriteAllText|wslpath|wsl\s+--install/u); - expect(readRepoText(WSL_E2E_WORKFLOW_PATH)).not.toMatch( - /WriteAllText|wslpath|wsl\s+--install/u, - ); }); // source-shape-contract: compatibility -- macOS must use the same modern shell/tool semantics as the Linux sandbox fixtures @@ -148,10 +128,17 @@ describe("platform Vitest main workflow", () => { const installOpenShell = step("macos-vitest", "Install pinned OpenShell"); const run = install.run ?? ""; - expect(job("macos-vitest")["timeout-minutes"]).toBe(30); + expect(job("macos-vitest")["timeout-minutes"]).toBe("${{ matrix.timeout_minutes }}"); expect(job("macos-vitest").strategy).toMatchObject({ "fail-fast": false, - matrix: { shard: [1, 2, 3, 4] }, + matrix: { + include: [ + { shard: 1, timeout_minutes: 60 }, + { shard: 2, timeout_minutes: 30 }, + { shard: 3, timeout_minutes: 30 }, + { shard: 4, timeout_minutes: 30 }, + ], + }, }); expect(checkout.with).toMatchObject({ "fetch-depth": 0, @@ -190,6 +177,12 @@ describe("platform Vitest main workflow", () => { expect(step("macos-vitest", "Run full Vitest suite on macOS").run).toContain( '--shard="${{ matrix.shard }}/4"', ); + expect(step("macos-vitest", "Detect Docker availability for macOS E2E").if).toBe( + "${{ matrix.shard == 1 }}", + ); + expect(step("macos-vitest", "Run macOS live E2E").run).toContain( + "test/e2e/live/full-e2e.test.ts", + ); const requirements = readRepoText(MACOS_REQUIREMENTS_PATH); expect(requirements).toContain("pyyaml==6.0.3"); @@ -210,10 +203,17 @@ describe("platform Vitest main workflow", () => { const fullSuite = step("wsl-vitest", "Run full Vitest suite in WSL").run ?? ""; const rootSuite = step("wsl-vitest", "Run root-required Vitest contracts in WSL").run ?? ""; - expect(job("wsl-vitest")["timeout-minutes"]).toBe(90); + expect(job("wsl-vitest")["timeout-minutes"]).toBe("${{ matrix.timeout_minutes }}"); expect(job("wsl-vitest").strategy).toMatchObject({ "fail-fast": false, - matrix: { shard: [1, 2, 3, 4] }, + matrix: { + include: [ + { shard: 1, timeout_minutes: 180 }, + { shard: 2, timeout_minutes: 90 }, + { shard: 3, timeout_minutes: 90 }, + { shard: 4, timeout_minutes: 90 }, + ], + }, }); expect(checkout.with).toMatchObject({ "fetch-depth": 0, @@ -240,5 +240,37 @@ describe("platform Vitest main workflow", () => { "keeps the locked Hermes entry sticky-protected|lets a sandbox-group peer create state", "requires both fixed files to match|reclaims a root-owned collapsed config|leaves a root-owned recovery baseline untouched", ]); + expect(step("wsl-vitest", "Detect Docker availability in WSL").if).toBe( + "${{ matrix.shard == 1 }}", + ); + expect(step("wsl-vitest", "Run WSL live E2E").run).toContain( + "test/e2e/live/full-e2e.test.ts", + ); + }); + + it("limits credentialed platform E2E to the first main-branch shard", () => { + const cases = [ + { + job: "macos-vitest", + step: "Run macOS live E2E", + dockerOutput: "steps.macos_docker.outputs.docker_ok == 'true'", + }, + { + job: "wsl-vitest", + step: "Run WSL live E2E", + dockerOutput: "steps.wsl_docker.outputs.docker_ok == 'true'", + }, + ]; + + for (const workflowCase of cases) { + const live = step(workflowCase.job, workflowCase.step); + expect(live.if).toContain("matrix.shard == 1"); + expect(live.if).toContain(workflowCase.dockerOutput); + expect(live.if).toContain("github.ref == 'refs/heads/main'"); + expect(live.env).toMatchObject({ + GITHUB_TOKEN: "${{ github.token }}", + NVIDIA_INFERENCE_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}", + }); + } }); }); diff --git a/test/vitest-watch-triggers.test.ts b/test/vitest-watch-triggers.test.ts index a528ea41b4f..2ea610943c1 100644 --- a/test/vitest-watch-triggers.test.ts +++ b/test/vitest-watch-triggers.test.ts @@ -73,8 +73,6 @@ const OPAQUE_INPUTS = [ ".github/workflows/pr-review-advisor.yaml", "tools/pr-review-advisor/openshell-policy.yaml", ".github/workflows/hosted-runner-recovery.yaml", - ".github/workflows/wsl-e2e.yaml", - ".github/workflows/macos-e2e.yaml", ".github/workflows/platform-vitest-main.yaml", "tools/wsl/ci-helper.ps1", "ci/platform-vitest-macos-requirements.lock", @@ -193,14 +191,6 @@ describe("Vitest opaque-input watch triggers", () => { expect(triggeredBy(".github/workflows/hosted-runner-recovery.yaml")).toEqual([ "test/hosted-runner-recovery-workflow.test.ts", ]); - expect(triggeredBy(".github/workflows/wsl-e2e.yaml")).toEqual([ - "test/hosted-runner-recovery-workflow.test.ts", - "test/platform-vitest-main-workflow.test.ts", - "test/wsl-ci-helper.test.ts", - ]); - expect(triggeredBy(".github/workflows/macos-e2e.yaml")).toEqual([ - "test/hosted-runner-recovery-workflow.test.ts", - ]); expect(triggeredBy(".github/workflows/platform-vitest-main.yaml")).toEqual([ "test/hosted-runner-recovery-workflow.test.ts", "test/platform-vitest-main-workflow.test.ts", diff --git a/tools/e2e/hosted-runner-recovery.mts b/tools/e2e/hosted-runner-recovery.mts index de899273aa2..15d865cd54b 100755 --- a/tools/e2e/hosted-runner-recovery.mts +++ b/tools/e2e/hosted-runner-recovery.mts @@ -19,18 +19,6 @@ const USER_AGENT = "nemoclaw-hosted-runner-recovery"; const SHA_PATTERN = /^[a-f0-9]{40}$/u; const GITHUB_TIMESTAMP_PATTERN = /^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$/u; const MAX_WORKFLOW_RUN_PAGES = 10; -const WINDOWS_INTERNAL_ERROR_POLICY: HostedRunnerLossPolicy = { - githubInternalError: { - approvedRunnerLabels: ["windows-latest"], - approvedJobConclusions: ["cancelled"], - }, -}; -const MACOS_INTERNAL_ERROR_POLICY: HostedRunnerLossPolicy = { - githubInternalError: { - approvedRunnerLabels: ["macos-26"], - approvedJobConclusions: ["cancelled"], - }, -}; const PLATFORM_INTERNAL_ERROR_POLICY: HostedRunnerLossPolicy = { githubInternalError: { approvedRunnerLabels: ["windows-latest", "macos-26"], @@ -38,41 +26,14 @@ const PLATFORM_INTERNAL_ERROR_POLICY: HostedRunnerLossPolicy = { }, }; -const SOURCE_WORKFLOWS = [ - { - workflowName: "E2E", - path: ".github/workflows/e2e.yaml", - runName: "E2E main", - events: ["schedule", "workflow_dispatch"], - displayTitle: "E2E main", - policy: {}, - allowedRunnerLabels: ["ubuntu-latest"], - }, - { - workflowName: "E2E / WSL", - path: ".github/workflows/wsl-e2e.yaml", - runName: "E2E / WSL", - events: ["push"], - policy: WINDOWS_INTERNAL_ERROR_POLICY, - allowedRunnerLabels: ["windows-latest"], - }, - { - workflowName: "E2E / macOS", - path: ".github/workflows/macos-e2e.yaml", - runName: "E2E / macOS", - events: ["push"], - policy: MACOS_INTERNAL_ERROR_POLICY, - allowedRunnerLabels: ["macos-26"], - }, - { - workflowName: "CI / Platform Vitest Main Watch", - path: ".github/workflows/platform-vitest-main.yaml", - runName: "CI / Platform Vitest Main Watch", - events: ["push"], - policy: PLATFORM_INTERNAL_ERROR_POLICY, - allowedRunnerLabels: ["ubuntu-latest", "windows-latest", "macos-26"], - }, -] as const; +const SOURCE_WORKFLOW = { + workflowName: "CI / Platform Evidence", + path: ".github/workflows/platform-vitest-main.yaml", + runName: "CI / Platform Evidence", + events: ["push"], + policy: PLATFORM_INTERNAL_ERROR_POLICY, + allowedRunnerLabels: ["ubuntu-latest", "windows-latest", "macos-26"], +} as const; type SourceWorkflowRun = { id: number; @@ -219,17 +180,14 @@ function eligibleRunDefinition( source: SourceWorkflowRun, workflow: SourceWorkflow, repository: string, -): (typeof SOURCE_WORKFLOWS)[number] | null { - const definition = SOURCE_WORKFLOWS.find( - (candidate) => - candidate.workflowName === workflow.name && - candidate.path === workflow.path && - candidate.path === source.path && - candidate.runName === source.runName, - ); - if (!definition) return null; +): typeof SOURCE_WORKFLOW | null { + const definition = SOURCE_WORKFLOW; const exactRunUrl = `https://github.com/${repository}/actions/runs/${source.id}`; if ( + definition.workflowName !== workflow.name || + definition.path !== workflow.path || + definition.path !== source.path || + definition.runName !== source.runName || workflow.id !== source.workflowId || workflow.state !== "active" || source.repository !== repository || @@ -240,9 +198,6 @@ function eligibleRunDefinition( ) { return null; } - if ("displayTitle" in definition && source.displayTitle !== definition.displayTitle) { - return null; - } return definition; } @@ -251,7 +206,7 @@ function eligibleSourceDefinition( workflow: SourceWorkflow, repository: string, sourceRunId: number, -): (typeof SOURCE_WORKFLOWS)[number] | null { +): typeof SOURCE_WORKFLOW | null { const definition = eligibleRunDefinition(source, workflow, repository); return definition && source.id === sourceRunId &&