diff --git a/.github/workflows/e2e-scenarios-all.yaml b/.github/workflows/e2e-scenarios-all.yaml deleted file mode 100644 index ce74552eaf4..00000000000 --- a/.github/workflows/e2e-scenarios-all.yaml +++ /dev/null @@ -1,92 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Scenario-based E2E fan-out. Generates the job matrix from the typed -# scenario registry (`test/e2e-scenario/scenarios/registry.ts`) so that -# adding a scenario in `baseline.ts` automatically produces a tile here on -# the next run -- no workflow edits required. -# -# The single-scenario runner workflow `e2e-scenarios.yaml` remains the -# authoritative execution path; this file just fans out one call per -# scenario id with the runner label resolved by `--emit-matrix`. - -name: E2E / Scenario Runner / All - -on: - workflow_dispatch: - -permissions: - contents: read - -concurrency: - group: e2e-scenarios-all-${{ github.ref }} - cancel-in-progress: false - -jobs: - generate-matrix: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.emit.outputs.matrix }} - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - persist-credentials: false - - - name: Set up Node - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 - with: - node-version: 22 - cache: npm - - - name: Install root dependencies - run: npm ci --ignore-scripts - - - id: emit - name: Emit scenario matrix from typed registry - run: | - set -euo pipefail - matrix="$(npx tsx test/e2e-scenario/scenarios/run.ts --emit-matrix)" - # Sanity-check that the output is non-empty JSON before handing it - # to GHA, so a registry error fails this job loudly instead of - # producing zero matrix tiles. - if [ -z "${matrix}" ] || [ "${matrix}" = "[]" ]; then - echo "::error::scenario matrix is empty; check typed registry" >&2 - exit 1 - fi - echo "matrix=${matrix}" >> "$GITHUB_OUTPUT" - - - name: Render matrix summary - env: - MATRIX_JSON: ${{ steps.emit.outputs.matrix }} - run: | - { - echo '## E2E scenario matrix' - echo '' - echo '| Scenario | Runner | Label |' - echo '| --- | --- | --- |' - python3 - <<'PY' - import json, os - for e in json.loads(os.environ["MATRIX_JSON"]): - print(f"| `{e['id']}` | {e['runner']} | {e['label']} |") - PY - } >> "$GITHUB_STEP_SUMMARY" - - run-scenario: - needs: generate-matrix - name: ${{ matrix.label }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJson(needs.generate-matrix.outputs.matrix) }} - uses: ./.github/workflows/e2e-scenarios.yaml - with: - scenarios: ${{ matrix.id }} - secrets: - NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} - - ubuntu-rebuild-openclaw: - uses: ./.github/workflows/e2e-scenarios.yaml - with: - scenarios: ubuntu-rebuild-openclaw - secrets: - NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} diff --git a/.github/workflows/e2e-scenarios.yaml b/.github/workflows/e2e-scenarios.yaml deleted file mode 100644 index 804426f1331..00000000000 --- a/.github/workflows/e2e-scenarios.yaml +++ /dev/null @@ -1,393 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -name: E2E / Scenario Runner - -on: - workflow_dispatch: - inputs: - scenarios: - description: "Comma-separated canonical typed scenario ids (for example: ubuntu-repo-cloud-openclaw,ubuntu-repo-cloud-hermes)" - required: true - type: string - workflow_call: - inputs: - scenarios: - description: "Comma-separated canonical typed scenario ids" - required: true - type: string - secrets: - NVIDIA_API_KEY: - required: false - -permissions: - contents: read - -concurrency: - group: e2e-scenarios-${{ inputs.scenarios || github.event.inputs.scenarios }} - cancel-in-progress: false - -jobs: - resolve-runner: - runs-on: ubuntu-latest - outputs: - runner: ${{ steps.pick.outputs.runner }} - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - persist-credentials: false - - - name: Set up Node - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 - with: - node-version: 22 - cache: npm - - - name: Install root dependencies - run: npm ci --ignore-scripts - - - id: pick - name: Resolve typed scenario runners - env: - SCENARIOS: ${{ inputs.scenarios || github.event.inputs.scenarios }} - run: | - set -euo pipefail - # Keep routing visible here while typed registry metadata is the source - # of the canonical scenario ids. Multi-runner mixed batches are rejected - # so each workflow job still runs on one correct runner. - declare -A ROUTES=( - [macos-repo-cloud-openclaw]=macos-26 - [wsl-repo-cloud-openclaw]=windows-latest - [gpu-repo-local-ollama-openclaw]=linux-amd64-gpu-rtxpro6000-latest-1 - [brev-launchable-cloud-openclaw]=ubuntu-latest - [ubuntu-gateway-port-conflict-negative]=ubuntu-latest - [ubuntu-invalid-nvidia-key-negative]=ubuntu-latest - [ubuntu-no-docker-preflight-negative]=ubuntu-latest - [ubuntu-rebuild-openclaw]=ubuntu-latest - [ubuntu-repo-cloud-hermes]=ubuntu-latest - [ubuntu-repo-cloud-hermes-discord]=ubuntu-latest - [ubuntu-repo-cloud-hermes-slack]=ubuntu-latest - [ubuntu-repo-cloud-openclaw]=ubuntu-latest - [ubuntu-repo-cloud-openclaw-brave]=ubuntu-latest - [ubuntu-repo-cloud-openclaw-custom-policies]=ubuntu-latest - [ubuntu-repo-cloud-openclaw-discord]=ubuntu-latest - [ubuntu-repo-cloud-openclaw-double-provider-switch]=ubuntu-latest - [ubuntu-repo-cloud-openclaw-double-same-provider]=ubuntu-latest - [ubuntu-repo-cloud-openclaw-repair]=ubuntu-latest - [ubuntu-repo-cloud-openclaw-resume]=ubuntu-latest - [ubuntu-repo-cloud-openclaw-slack]=ubuntu-latest - [ubuntu-repo-cloud-openclaw-telegram]=ubuntu-latest - [ubuntu-repo-cloud-openclaw-token-rotation]=ubuntu-latest - [ubuntu-repo-docker-post-reboot-recovery]=ubuntu-latest - [ubuntu-repo-openai-compatible-openclaw]=ubuntu-latest - ) - selected="" - IFS=',' read -ra IDS <<< "${SCENARIOS}" - for raw in "${IDS[@]}"; do - id="${raw//[[:space:]]/}" - [ -n "${id}" ] || continue - runner="${ROUTES[$id]:-}" - if [ -z "${runner}" ]; then - echo "::error::No runner route for scenario: ${id}" >&2 - exit 1 - fi - if [ -n "${selected}" ] && [ "${selected}" != "${runner}" ]; then - echo "::error::Scenario batch spans multiple runner types (${selected}, ${runner}); split dispatch." >&2 - exit 1 - fi - selected="${runner}" - done - echo "runner=${selected:-ubuntu-latest}" >> "$GITHUB_OUTPUT" - - run-scenario: - needs: resolve-runner - runs-on: ${{ needs.resolve-runner.outputs.runner }} - timeout-minutes: 90 - env: - WSL_DISTRO: Ubuntu - NEMOCLAW_RECREATE_SANDBOX: "1" - E2E_CONTEXT_DIR: ${{ github.workspace }} - steps: - - name: Force LF line endings for WSL checkout - if: contains(inputs.scenarios || github.event.inputs.scenarios, 'wsl-repo-cloud-openclaw') - shell: powershell - run: git config --global core.autocrlf false - - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - persist-credentials: false - - - name: Set up Node - if: ${{ !contains(inputs.scenarios || github.event.inputs.scenarios, 'wsl-repo-cloud-openclaw') }} - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 - with: - node-version: 22 - cache: npm - - - name: Install root dependencies - if: ${{ !contains(inputs.scenarios || github.event.inputs.scenarios, 'wsl-repo-cloud-openclaw') }} - run: npm ci --ignore-scripts - - - name: Run typed scenarios - if: ${{ !contains(inputs.scenarios || github.event.inputs.scenarios, 'wsl-repo-cloud-openclaw') }} - env: - NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} - SCENARIOS: ${{ inputs.scenarios || github.event.inputs.scenarios }} - run: | - set -euo pipefail - if [[ ! "${SCENARIOS}" =~ ^[A-Za-z0-9._-]+(,[A-Za-z0-9._-]+)*$ ]]; then - echo "::error::Invalid scenario input: ${SCENARIOS}" >&2 - exit 1 - fi - npx tsx test/e2e-scenario/scenarios/run.ts --scenarios "${SCENARIOS}" - - - name: Resolve workspace paths for WSL - if: contains(inputs.scenarios || github.event.inputs.scenarios, 'wsl-repo-cloud-openclaw') - shell: powershell - run: | - $winPath = "${{ github.workspace }}" - $drive = $winPath.Substring(0,1).ToLower() - $rest = $winPath.Substring(2).Replace('\','/') - $wslCheckoutPath = "/mnt/$drive$rest" - $wslWorkdir = "/tmp/nemoclaw-scenario-wsl/${env:GITHUB_RUN_ID}-${env:GITHUB_RUN_ATTEMPT}" - "WSL_CHECKOUT_DIR=$wslCheckoutPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - "WSL_WORKDIR=$wslWorkdir" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - - - name: Ensure Ubuntu WSL exists - if: contains(inputs.scenarios || github.event.inputs.scenarios, 'wsl-repo-cloud-openclaw') - shell: powershell - run: | - wsl --list --verbose 2>&1 | Out-Default - $null = wsl -d $env:WSL_DISTRO -- echo ok 2>&1 - if ($LASTEXITCODE -ne 0) { - $maxAttempts = 3 - $installed = $false - for ($attempt = 1; $attempt -le $maxAttempts; $attempt++) { - Write-Host "Ubuntu not found - installing via wsl --install (attempt $attempt/$maxAttempts)" - wsl --install -d $env:WSL_DISTRO --no-launch --web-download - $installExitCode = $LASTEXITCODE - if ($installExitCode -eq 0) { - wsl -d $env:WSL_DISTRO -- bash -c 'echo distro initialised' - $launchExitCode = $LASTEXITCODE - if ($launchExitCode -eq 0) { - $installed = $true - break - } - Write-Warning "distro first-launch failed with exit code $launchExitCode" - } else { - Write-Warning "wsl --install failed with exit code $installExitCode" - } - - $null = wsl -d $env:WSL_DISTRO -- echo ok 2>&1 - if ($LASTEXITCODE -eq 0) { - Write-Host 'Ubuntu became available after the install command returned non-zero' - $installed = $true - break - } - - if ($attempt -lt $maxAttempts) { - Write-Host 'Cleaning up any partial WSL registration before retrying' - $null = wsl --unregister $env:WSL_DISTRO 2>&1 - $delaySeconds = [Math]::Min(60, 20 * $attempt) - Write-Host "Retrying WSL install in $delaySeconds seconds..." - Start-Sleep -Seconds $delaySeconds - } - } - - if (-not $installed) { - throw ("failed to install and initialize $env:WSL_DISTRO after $maxAttempts attempts") - } - } else { - Write-Host 'Ubuntu already available' - } - wsl --set-default $env:WSL_DISTRO - if ($LASTEXITCODE -ne 0) { - throw ('wsl --set-default failed with exit code ' + $LASTEXITCODE) - } - - - name: Verify WSL - if: contains(inputs.scenarios || github.event.inputs.scenarios, 'wsl-repo-cloud-openclaw') - shell: powershell - run: | - wsl -d $env:WSL_DISTRO -- bash -lc "uname -a" - wsl -d $env:WSL_DISTRO -- bash -lc "cat /etc/os-release" - - - name: Install Ubuntu dependencies - if: contains(inputs.scenarios || github.event.inputs.scenarios, 'wsl-repo-cloud-openclaw') - shell: powershell - run: | - $script = @' - set -euo pipefail - export DEBIAN_FRONTEND=noninteractive - printf '%s\n' \ - 'Acquire::ForceIPv4 "true";' \ - 'Acquire::Retries "5";' \ - >/etc/apt/apt.conf.d/99github-actions-network - apt-get update - apt-get install -y bash ca-certificates curl git jq lsb-release make python3 python3-pip rsync tar unzip xz-utils - '@ - $tmp = "$env:RUNNER_TEMP\wsl-step.sh" - [IO.File]::WriteAllText($tmp, ($script -replace "`r",""), (New-Object System.Text.UTF8Encoding $false)) - $wslTmp = wsl -d $env:WSL_DISTRO -- wslpath -u ($tmp -replace '\\','/') - wsl -d $env:WSL_DISTRO -- bash -l $wslTmp - - - name: Install Node.js 22 in WSL - if: contains(inputs.scenarios || github.event.inputs.scenarios, 'wsl-repo-cloud-openclaw') - shell: powershell - run: | - $script = @' - set -euo pipefail - curl -fsSL https://deb.nodesource.com/setup_22.x | bash - - apt-get install -y nodejs - node --version - npm --version - '@ - $tmp = "$env:RUNNER_TEMP\wsl-step.sh" - [IO.File]::WriteAllText($tmp, ($script -replace "`r",""), (New-Object System.Text.UTF8Encoding $false)) - $wslTmp = wsl -d $env:WSL_DISTRO -- wslpath -u ($tmp -replace '\\','/') - wsl -d $env:WSL_DISTRO -- bash -l $wslTmp - - - name: Copy checkout into WSL ext4 workspace - if: contains(inputs.scenarios || github.event.inputs.scenarios, 'wsl-repo-cloud-openclaw') - shell: powershell - run: | - $checkout = $env:WSL_CHECKOUT_DIR - $workdir = $env:WSL_WORKDIR - $workdirParent = $workdir.Substring(0, $workdir.LastIndexOf('/')) - $script = @" - set -euo pipefail - echo 'Syncing checkout from $checkout to $workdir' - if [ ! -d '$checkout/.git' ]; then - echo 'Expected a Git checkout at $checkout' >&2 - exit 1 - fi - rm -rf '$workdir' - mkdir -p '$workdirParent' - rsync -a --no-owner --no-group --delete \ - --exclude '/node_modules/' \ - --exclude '/nemoclaw/node_modules/' \ - --exclude '/nemoclaw-blueprint/.venv/' \ - '$checkout'/ '$workdir'/ - git config --global --add safe.directory '$workdir' - git -C '$workdir' reset --hard HEAD - git -C '$workdir' clean -ffdx - git -C '$workdir' status --short - echo 'WSL ext4 workspace ready at $workdir' - "@ - $tmp = "$env:RUNNER_TEMP\wsl-step.sh" - [IO.File]::WriteAllText($tmp, ($script -replace "`r",""), (New-Object System.Text.UTF8Encoding $false)) - $wslTmp = wsl -d $env:WSL_DISTRO -- wslpath -u ($tmp -replace '\\','/') - wsl -d $env:WSL_DISTRO -- bash -l $wslTmp - - - name: Run typed scenarios in WSL - if: contains(inputs.scenarios || github.event.inputs.scenarios, 'wsl-repo-cloud-openclaw') - shell: powershell - env: - NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} - SCENARIOS: ${{ inputs.scenarios || github.event.inputs.scenarios }} - run: | - if ($env:SCENARIOS -notmatch '^[A-Za-z0-9._-]+(,[A-Za-z0-9._-]+)*$') { - Write-Error "Invalid scenario input: $env:SCENARIOS" - exit 1 - } - $workdir = $env:WSL_WORKDIR - $checkout = $env:WSL_CHECKOUT_DIR - $scenarios = $env:SCENARIOS - $script = @" - set -euo pipefail - workdir='$workdir' - checkout_dir='$checkout' - scenarios='$scenarios' - mkdir -p "`$workdir" - cd "`$workdir" - export E2E_CONTEXT_DIR="`$workdir" - npm ci --ignore-scripts - set +e - npx tsx test/e2e-scenario/scenarios/run.ts --scenarios "`$scenarios" - status=`$? - if [ -d "`$workdir/.e2e" ]; then - rm -rf "`$checkout_dir/.e2e" - cp -a "`$workdir/.e2e" "`$checkout_dir/.e2e" - fi - if [ -d "`$workdir/test/e2e/logs" ]; then - mkdir -p "`$checkout_dir/test/e2e" - rm -rf "`$checkout_dir/test/e2e/logs" - cp -a "`$workdir/test/e2e/logs" "`$checkout_dir/test/e2e/logs" - fi - exit "`$status" - "@ - $tmp = "$env:RUNNER_TEMP\wsl-step.sh" - [IO.File]::WriteAllText($tmp, ($script -replace "`r",""), (New-Object System.Text.UTF8Encoding $false)) - $wslTmp = wsl -d $env:WSL_DISTRO -- wslpath -u ($tmp -replace '\\','/') - $apiKeyArg = "NVIDIA_API_KEY=$env:NVIDIA_API_KEY" - wsl -d $env:WSL_DISTRO -- env $apiKeyArg bash -l $wslTmp - if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE - } - - - name: Append typed scenario summary - if: always() - shell: bash - run: | - { - echo '## E2E typed scenario run' - echo '' - echo 'Mode: `test/e2e-scenario/scenarios/run.ts --scenarios ` (live).' - echo '' - if [ -f .e2e/run-plan.json ]; then - python3 - <<'PY' - import json - from pathlib import Path - - plans = json.loads(Path('.e2e/run-plan.json').read_text(encoding='utf-8')) - print('| Scenario | Manifest | Expected state | Suites | Phases |') - print('| --- | --- | --- | --- | --- |') - for plan in plans: - suites = ', '.join(f"`{suite}`" for suite in plan.get('suiteIds', [])) or '_none_' - phases = ', '.join(f"`{phase.get('name')}`" for phase in plan.get('phases', [])) or '_none_' - print( - '| `{scenario}` | `{manifest}` | `{state}` | {suites} | {phases} |'.format( - scenario=plan.get('scenarioId', ''), - manifest=plan.get('manifestPath') or '_none_', - state=plan.get('expectedStateId') or '_none_', - suites=suites, - phases=phases, - ) - ) - PY - elif [ -f .e2e/plan.txt ]; then - echo '```text' - cat .e2e/plan.txt - echo '```' - else - echo '_No typed scenario plan artifact was produced._' - fi - } >> "$GITHUB_STEP_SUMMARY" - - - name: Upload scenario artifacts - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: e2e-scenario-${{ inputs.scenarios || github.event.inputs.scenarios }} - # Explicit subpath list, NOT a blanket .e2e/ + hidden files. - # The framework redacts every byte that flows from spawned - # children into actions/*.log, logs/*.log, and onboard.log via - # orchestrators/redaction.ts::pipeRedacted. Anything outside - # the listed paths (notably the raw context.env file) is - # excluded so secret-bearing key=value lines cannot leak via - # the artifact even if a future helper writes there. - # Diagnostic dumps of context use e2e_context_dump, which - # redacts on emit (runtime/lib/context.sh). - path: | - .e2e/run-plan.json - .e2e/plan.txt - .e2e/environment.result.json - .e2e/onboarding.result.json - .e2e/runtime.result.json - .e2e/actions/ - .e2e/logs/ - .e2e/onboard.log - test/e2e/logs/ - if-no-files-found: warn - retention-days: 14 - include-hidden-files: false diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index f072a29c7bd..a5f5ae864a8 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -54,6 +54,18 @@ jobs: fi matrix="$(npx tsx test/e2e-scenario/scenarios/run.ts "${args[@]}")" echo "matrix=${matrix}" >> "$GITHUB_OUTPUT" + MATRIX_JSON="${matrix}" python - <<'PY' >> "$GITHUB_STEP_SUMMARY" + import json + import os + + rows = json.loads(os.environ["MATRIX_JSON"]) + print("## Vitest E2E Scenario Matrix") + print() + print("| Scenario | Runner | Label |") + print("| --- | --- | --- |") + for row in rows: + print(f"| `{row['id']}` | `{row['runner']}` | {row['label']} |") + PY live-scenarios: needs: generate-matrix @@ -64,10 +76,9 @@ jobs: matrix: include: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }} env: - E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/${{ matrix.id }} + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_E2E_SCENARIOS: "1" - NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: @@ -87,6 +98,7 @@ jobs: - name: Run Vitest live E2E scenarios env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} SCENARIO_ID: ${{ matrix.id }} run: | set -euo pipefail @@ -98,27 +110,56 @@ jobs: SCENARIO_ID: ${{ matrix.id }} SCENARIO_LABEL: ${{ matrix.label }} run: | - { - echo "## Vitest E2E Scenarios" - echo - echo "- Project: \`e2e-scenarios-live\`" - printf '%s%s%s\n' '- Scenario: `' "${SCENARIO_ID}" '`' - printf '%s%s%s\n' '- Label: `' "${SCENARIO_LABEL}" '`' - echo "- Artifact root: \`${E2E_ARTIFACT_DIR}\`" - echo - if [ -d "${E2E_ARTIFACT_DIR}" ]; then - echo "### Captured Files" - find "${E2E_ARTIFACT_DIR}" -type f | sort | sed "s#${E2E_ARTIFACT_DIR}/#- #" - else - echo "No fixture artifacts were captured." - fi - } >> "$GITHUB_STEP_SUMMARY" + python - <<'PY' >> "$GITHUB_STEP_SUMMARY" + import json + import os + from pathlib import Path + + root = Path(os.environ["E2E_ARTIFACT_DIR"]) / os.environ["SCENARIO_ID"] + plan_path = root / "run-plan.json" + print("## Vitest E2E Scenarios") + print() + print("- Project: `e2e-scenarios-live`") + print(f"- Scenario: `{os.environ['SCENARIO_ID']}`") + print(f"- Label: `{os.environ['SCENARIO_LABEL']}`") + print(f"- Artifact root: `{root}`") + print() + print("| Scenario | Manifest | Expected state | Suites | Phases |") + print("| --- | --- | --- | --- | --- |") + if plan_path.exists(): + plan = json.loads(plan_path.read_text(encoding="utf-8")) + suites = ", ".join(plan.get("suiteIds") or []) or "(none)" + phases = ", ".join(plan.get("phases") or []) or "(none)" + print( + "| " + f"`{plan.get('scenarioId') or os.environ['SCENARIO_ID']}` | " + f"`{plan.get('manifestPath') or 'not-yet-defined'}` | " + f"`{plan.get('expectedStateId') or 'not-yet-defined'}` | " + f"{suites} | {phases} |" + ) + else: + print( + "| " + f"`{os.environ['SCENARIO_ID']}` | `(missing run-plan.json)` | " + "`(missing run-plan.json)` | `(missing)` | `(missing)` |" + ) + PY - name: Upload Vitest E2E artifacts if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: e2e-vitest-scenarios-${{ matrix.id }} - path: e2e-artifacts/vitest/${{ matrix.id }}/ + path: | + e2e-artifacts/vitest/${{ matrix.id }}/run-plan.json + e2e-artifacts/vitest/${{ matrix.id }}/scenario.json + e2e-artifacts/vitest/${{ matrix.id }}/scenario-result.json + e2e-artifacts/vitest/${{ matrix.id }}/environment.result.json + e2e-artifacts/vitest/${{ matrix.id }}/onboarding.result.json + e2e-artifacts/vitest/${{ matrix.id }}/state-validation.result.json + e2e-artifacts/vitest/${{ matrix.id }}/actions/ + e2e-artifacts/vitest/${{ matrix.id }}/logs/ + e2e-artifacts/vitest/${{ matrix.id }}/shell/ include-hidden-files: false if-no-files-found: ignore + retention-days: 14 diff --git a/scripts/e2e/lint-conventions.ts b/scripts/e2e/lint-conventions.ts deleted file mode 100755 index 4a602aee09b..00000000000 --- a/scripts/e2e/lint-conventions.ts +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/env tsx -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -/** - * E2E convention lint. - * - * Enforces conventions for `test/e2e-scenario/validation_suites/**` step scripts and - * keeps the new typed scenario suite isolated under `test/e2e-scenario/**`. - * Existing top-level `test/e2e/test-*.sh` entrypoints remain valid until a - * separate migration explicitly retires them. - */ - -import fs from "node:fs"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; - -interface Rule { - id: string; - describe: string; - test: (body: string) => string | null; -} - -const STEP_RULES: Rule[] = [ - { - id: "no-noninteractive-reexport", - describe: "suite step re-exports non-interactive env vars", - test: (body) => { - const patterns = [ - /export\s+DEBIAN_FRONTEND\s*=\s*noninteractive/, - /export\s+NEMOCLAW_NON_INTERACTIVE\s*=\s*1/, - ]; - for (const p of patterns) { - if (p.test(body)) - return `matched ${p.source}; non-interactive setup belongs to shared runtime helpers`; - } - return null; - }, - }, - { - id: "no-own-trap", - describe: "suite step registers its own trap", - test: (body) => { - for (const raw of body.split("\n")) { - const line = raw.trimStart(); - if (line.startsWith("#")) continue; - if (/^trap\s+[^#]/.test(line)) - return "registered own trap; cleanup belongs to orchestrators/shared helpers"; - } - return null; - }, - }, - { - id: "no-section-helper", - describe: "suite step calls section helper directly", - test: (body) => - /^\s*section\s+["']/m.test(body) || /^\s*section\s*\(/m.test(body) - ? "step calls section; plan/phase output owns sections" - : null, - }, - { - id: "no-tmp-log", - describe: "suite step writes logs under /tmp", - test: (body) => - /\/tmp\/[^\s'\"]+\.log/.test(body) ? "write logs under E2E_CONTEXT_DIR, not /tmp" : null, - }, - { - id: "no-git-rev-parse-root", - describe: "suite step uses non-standard repo-root discovery", - test: (body) => - /git\s+rev-parse\s+--show-toplevel/.test(body) - ? "avoid git rev-parse repo-root discovery in suite steps" - : null, - }, -]; - -interface LintFinding { - file: string; - rule: string; - message: string; -} - -function walk(dir: string): string[] { - if (!fs.existsSync(dir)) return []; - const out: string[] = []; - for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { - const full = path.join(dir, entry.name); - if (entry.isDirectory()) out.push(...walk(full)); - else out.push(full); - } - return out; -} - -function lintSuiteSteps(root: string): LintFinding[] { - const suitesDir = path.join(root, "test/e2e-scenario/validation_suites"); - const findings: LintFinding[] = []; - for (const file of walk(suitesDir).filter((entry) => entry.endsWith(".sh"))) { - const rel = path.relative(root, file); - const body = fs.readFileSync(file, "utf8"); - for (const rule of STEP_RULES) { - const message = rule.test(body); - if (message) findings.push({ file: rel, rule: rule.id, message }); - } - } - return findings; -} - -function lint(root: string): LintFinding[] { - return lintSuiteSteps(root); -} - -function parseArgs(argv: string[]): { root: string } { - let root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); - const args = argv.slice(2); - while (args.length > 0) { - const arg = args.shift(); - if (arg === "--root") { - const value = args.shift(); - if (!value) throw new Error("--root requires a value"); - root = path.resolve(value); - } else if (arg === "--help" || arg === "-h") { - process.stdout.write("tsx scripts/e2e/lint-conventions.ts [--root ]\n"); - process.exit(0); - } else if (arg) { - throw new Error(`unexpected arg: ${arg}`); - } - } - return { root }; -} - -try { - const { root } = parseArgs(process.argv); - const findings = lint(root); - if (findings.length > 0) { - for (const finding of findings) { - process.stderr.write(`${finding.file}: ${finding.rule}: ${finding.message}\n`); - } - process.exit(1); - } - process.stdout.write("e2e convention lint passed\n"); -} catch (err) { - process.stderr.write(`lint-conventions: ${(err as Error).message}\n`); - process.exit(2); -} diff --git a/test/e2e-scenario/docs/MIGRATION.md b/test/e2e-scenario/docs/MIGRATION.md index 170b3871d0d..167f74ea10d 100644 --- a/test/e2e-scenario/docs/MIGRATION.md +++ b/test/e2e-scenario/docs/MIGRATION.md @@ -1,143 +1,99 @@ -# E2E scenario migration notes - -This file records the current migration model for contributors. It is not the -source of truth for per-domain status. Mutable migration state is tracked -outside the repository in GitHub issues and pull requests so reviewers can -discuss, update, and close work in one place. - -## Current migration state - -The scenario E2E migration is in a hybrid phase: - -- typed scenario builders drive scenario workflow fan-out and dry-run plans; -- product-facing `NemoClawInstance` manifests describe desired setup and - onboarding state; -- YAML metadata still drives the shell scenario runner and live suite - resolution; -- legacy `test/e2e/test-*.sh` scripts still provide most live nightly and - platform coverage. - -This hybrid shape is not the target end state. #3588 and #4941 should converge -on one live execution path: **Vitest as the scenario runner, extended by -NemoClaw fixtures and typed domain helpers**. Until that path owns live -execution, resolver behavior, assertions, evidence, cleanup, and redaction, -treat YAML and bash runner updates as bridge work rather than durable -architecture. - -Do not assume legacy scripts are deletion-ready just because a scenario or suite -name exists. The final reconciliation phase must show either evidence-complete -coverage or an explicit audit amendment before legacy executable tests are -removed. - -## Target architecture - -The final scenario framework should have one execution path: - -- Vitest owns live execution, filtering, reporters, timeouts, fixture lifecycle, - skip handling, and CI integration; -- NemoClaw fixtures own setup, onboarding, runtime actions, expected-state - probes, assertion helpers, expected-failure matching, evidence artifacts, - cleanup, and secret redaction; -- typed scenario definitions and matrix helpers describe stable scenario IDs and - supported combinations without becoming a second execution framework; -- reusable assertions prefer TypeScript probes and typed clients; -- shell scripts remain only for host, sandbox, process, or platform boundaries - where shell is the thing being tested or the lowest-risk adapter; -- shell execution is wrapped by fixtures so environment scoping, timeout, - redaction, artifact capture, and argument validation are consistent. - -When a bridge PR adds behavior to the current YAML/bash runner, preserve the -requirement it proves, but port that requirement into the Vitest fixture path -before removing legacy runner pieces. Do not deepen the bash runner as a second -long-term source of truth. - -## Active issue tracking - -Use these GitHub issues for status and follow-up work: - -| Issue | Purpose | -| --- | --- | -| #3588 | Parent architecture epic for layered single-runner scenario E2E | -| #4941 | Decision issue for using Vitest fixtures as the scenario execution model | -| #4347–#4356 | Domain-specific audit-coverage phases | -| #4357 | Final audit reconciliation, placeholder cleanup, and deletion-readiness review | -| #4378 | Friendly `setup_scenarios` aliases for layered test plans | - -If a migration discovery needs durable tracking, add it to the relevant issue or -open a focused child issue. Avoid adding long-lived checklists here. - -## What belongs in the repo - -Keep durable framework guidance here: - -- how to run the scenario runner, -- where scenario metadata, typed builders, manifests, and suites live, -- how to add or review a scenario, expected state, assertion, or suite, -- stable conventions that should not change with every migration batch. - -Do not add migration status tables, per-legacy-script checklists, temporary -coverage counts, or owner queues to this file. Put those in the issue or PR -that owns the work instead. - -The one repo-local exception is the machine-readable deletion gate inventory at -`test/e2e-scenario/migration/legacy-inventory.json`. Keep that file focused on -deletion-readiness evidence that prevents accidental legacy E2E deletion. It -must cover every direct legacy shell entrypoint under `test/e2e/test-*.sh`, -plus any explicitly retained bridge entrypoints such as Brev. It also tracks -coarse internal legacy runner surfaces such as the YAML/bash scenario workers, -validation suites, TypeScript shell-runner orchestrators, and runtime helper -libraries so those surfaces cannot be removed without #4357 evidence. It is not -a progress dashboard or owner queue: - -- `not-migrated`: legacy coverage still has no equivalent Vitest scenario. +# E2E Scenario Migration Notes + +This file describes how to move coverage into the Vitest scenario framework +without confusing that work with the retired typed-shell scenario runner. +Changing status, ownership, and per-test decisions belong in GitHub issues and +PRs. + +Migration state is tracked outside the repository in GitHub issues and pull +requests. +Use GitHub issues and pull requests for status changes. + +## Current State + +The scenario runner cutover is complete: + +- `e2e-vitest-scenarios.yaml` is the scenario workflow. +- `test/e2e-scenario/live/registry-scenarios.test.ts` is the registry-driven + live scenario entrypoint. +- `test/e2e-scenario/framework/` owns phase fixtures, clients, artifact + capture, redaction, cleanup, and shell-probe bridges. +- `test/e2e-scenario/scenarios/run.ts` only lists scenarios and emits the live + Vitest matrix. +- The typed-shell scenario runner, shell validation-suite tree, and retiring + scenario workflows are removed. See `RETIREMENT.md`. + +Direct legacy E2E scripts under `test/e2e/test-*.sh` remain in place. Many are +expected to stay because they test shell/install/user-flow behavior or preserve +umbrella integration smoke value. #5098 tracks family-by-family migration, +augmentation, and eventual deletion decisions for those scripts. + +## Target Architecture + +The durable scenario framework has one execution path: + +- Vitest owns execution, filtering, reporters, timeouts, fixture lifecycle, + skip handling, and CI integration. +- NemoClaw fixtures own setup, onboarding, lifecycle mutations, + expected-state probes, assertion helpers, expected-failure evidence, + cleanup, artifacts, and secret redaction. +- Typed scenario definitions and matrix helpers describe stable scenario IDs + and supported combinations without becoming a second runner. +- Product-facing manifests describe desired setup/onboarding state, not test + execution logic. +- Shell scripts remain only for direct legacy E2Es or narrow system-boundary + probes where shell is the contract or lowest-risk adapter. + +## Deletion Inventory + +`test/e2e-scenario/migration/legacy-inventory.json` is a machine-readable +deletion gate. + +It must cover: + +- every direct legacy shell entrypoint under `test/e2e/test-*.sh`; +- explicitly retained bridge entrypoints such as `test/e2e/brev-e2e.test.ts`; +- retired internal scenario-runner surfaces removed by the cutover. + +Status values: + +- `not-migrated`: legacy coverage has no equivalent typed scenario yet. - `bridge-probe`: coverage is temporarily represented by a bridge path. - `covered`: equivalent Vitest live scenario coverage exists. -- `retired`: maintainers agreed the legacy coverage is no longer required. - -Do not set `deletionReady: true` on a script entry or internal surface unless -the record is `covered` or `retired` and the deletion approval is recorded -through #4357. - -After #4357 completes final legacy E2E reconciliation, remove the inventory if -there are no remaining legacy entrypoints to guard. If maintainers keep it, keep -it as an audit artifact rather than as a living migration checklist. - -## What to migrate next - -When moving behavior from a legacy E2E script into the scenario framework: - -1. Identify the relevant audit issue (#4347–#4356). -2. Add or update the product-facing manifest only when the desired setup or - onboarding state changes. -3. Add typed scenario registry coverage when the workflow matrix needs a new - canonical scenario ID. -4. Add or update current YAML metadata only when the existing bridge runner must - keep resolving the scenario during the migration. -5. Add the reusable assertion or probe in the Vitest fixture direction whenever - possible instead of adding new bash-runner-only behavior. -6. Add reusable suite or assertion helpers instead of copying entire legacy - scripts. -7. Add framework tests that prevent the typed registry, YAML aliases, workflow - routes, manifests, suites, and runner behavior from drifting. -8. Leave legacy executable scripts in place until deletion readiness is - recorded in the owning issue or PR. The bash scenario entrypoints - (`runtime/run-scenario.sh`, `runtime/run-suites.sh`) and the YAML resolver - tree are already gone — the TypeScript runner is the sole canonical - executor. - -## Useful commands +- `retired`: maintainers agreed the legacy surface is no longer required. + +Do not set `deletionReady: true` on a direct legacy script unless the record is +`covered` or `retired` and the approval issue records the deletion rationale. +The retired internal scenario-runner surfaces are already marked through #5098; +that does not imply direct legacy bash scripts are deletion-ready. + +## Migration Pattern + +When moving behavior from a legacy E2E script: + +1. Identify the test family and policy from #5098: KEEP_BASH, HYBRID, or + MIGRATE_TYPED. +2. Add or update manifests only when product setup/onboarding state changes. +3. Add typed scenario registry coverage when the live matrix needs a stable + scenario ID. +4. Add fixture helpers before copying shell logic. +5. For HYBRID tests, keep the bash test and add a focused typed peer for the + contract being strengthened. +6. For MIGRATE_TYPED tests, prove parity first, then mark the inventory row + covered before any deletion PR. +7. Leave umbrella KEEP_BASH tests in place unless the tracking issue explicitly + revises their classification. + +## Useful Commands ```bash -# Typed registry inventory and execution +# Scenario registry and matrix npx tsx test/e2e-scenario/scenarios/run.ts --list -npx tsx test/e2e-scenario/scenarios/run.ts --emit-matrix -npx tsx test/e2e-scenario/scenarios/run.ts --scenarios - -# Local debug only: print the compiled plan without executing -npx tsx test/e2e-scenario/scenarios/run.ts --scenarios --plan-only +npx tsx test/e2e-scenario/scenarios/run.ts --emit-live-matrix +npx tsx test/e2e-scenario/scenarios/run.ts --emit-live-matrix --scenarios ubuntu-repo-cloud-openclaw # Framework tests npx vitest run --project e2e-scenario-framework --silent=false --reporter=default @@ -147,15 +103,5 @@ npm run build:cli NEMOCLAW_RUN_E2E_SCENARIOS=1 npx vitest run --project e2e-scenarios-live --silent=false --reporter=default ``` -## Cleanup rules - -- Prefer new scenario-matrix coverage over new legacy-style `test-*.sh` scripts. -- Prefer Vitest fixture behavior over new YAML/bash-runner behavior; if a bridge - change is unavoidable, document the porting requirement in the owning issue or - PR. -- Do not reintroduce the removed workflow-level parity report unless maintainers - explicitly reopen that direction. -- Do not delete legacy executable E2Es as part of ordinary domain migration PRs; - queue deletion candidates for #4357. -- Keep docs focused on how the framework works now. Put changing progress status - in issues and PRs. +The old `--emit-matrix`, direct `--scenarios` execution, and `--plan-only` +interfaces are retired. diff --git a/test/e2e-scenario/docs/README.md b/test/e2e-scenario/docs/README.md index 8f0f12f704b..68a750dc1e2 100644 --- a/test/e2e-scenario/docs/README.md +++ b/test/e2e-scenario/docs/README.md @@ -1,226 +1,124 @@ -# NemoClaw E2E scenario framework +# NemoClaw E2E Scenario Framework -NemoClaw's scenario E2E framework is currently a **hybrid** migration model. -It combines typed scenario builders, product-facing setup manifests, YAML -runtime metadata, and reusable shell suites while the older live E2E scripts -continue to run in parallel. +NemoClaw scenario E2E now uses **Vitest as the scenario execution runner**. +Vitest owns discovery, filtering, timeouts, reporters, fixture lifecycle, +skips, and CI integration. NemoClaw owns the domain layer: scenario metadata, +phase fixtures, product clients, evidence artifacts, redaction, cleanup, +expected-state probes, and typed assertion helpers. -This hybrid model is transitional. The target architecture for #3588 and #4941 -is **Vitest as the single scenario execution runner**, extended by NemoClaw -fixtures and typed domain helpers. Vitest owns test discovery, filtering, -timeouts, reporters, fixture lifecycle, skips, and CI integration. NemoClaw owns -scenario vocabulary, setup/onboarding helpers, product clients, evidence -collection, redaction, cleanup, and assertion helpers. +The retired typed-shell scenario runner is documented in +[`RETIREMENT.md`](./RETIREMENT.md). Do not add new durable behavior to the old +YAML/bash scenario-runner shape. -Shell scripts should be kept to the smallest practical set of system-boundary -probes or command fixtures, not a second planning or assertion-control runtime. +Direct legacy E2E scripts under `test/e2e/test-*.sh` still provide most live +nightly and platform coverage. Those scripts are not deleted by the scenario +runner cutover; migrate or augment them family by family using the inventory +rules in `MIGRATION.md`. -## Current sources of truth +## Sources Of Truth -Use the source that matches the task while the migration is in progress: - -| Task | Current source | +| Task | Source | | --- | --- | -| Scenario workflow fan-out and live execution | `test/e2e-scenario/scenarios/registry.ts`, `test/e2e-scenario/scenarios/scenarios/baseline.ts`, and `test/e2e-scenario/scenarios/run.ts` | -| Typed expected-state registry (single source of truth) | `test/e2e-scenario/scenarios/expected-states.ts` | -| Product-facing desired setup/onboarding state | `test/e2e-scenario/manifests/*.yaml` | -| Shell runner scenario resolution and live scenario execution | `test/e2e-scenario/nemoclaw_scenarios/scenarios.yaml` and `validation_suites/suites.yaml` (legacy YAML resolver path retired) | -| Reusable live suite assertions | `test/e2e-scenario/validation_suites/` | -| Existing nightly and platform E2E coverage | legacy `test/e2e/test-*.sh` scripts and their workflows | - -The near-term migration goal is to keep these surfaces aligned while coverage is -being moved into scenario contracts and suites. The long-term goal is to remove -the split between typed planning and shell execution. Do not add new -legacy-style `test/e2e/test-*.sh` entrypoints unless there is a specific -maintainer-approved reason. - -## Target runner model - -Future scenario coverage should move toward one Vitest-based runner with these -properties: - -- Vitest is the execution surface for live scenarios and owns lifecycle, - filtering, reporting, timeouts, and fixture scopes; -- NemoClaw fixtures expose scenario-level helpers for setup, onboarding, host - CLI access, gateway checks, sandbox checks, provider fixtures, evidence - artifacts, redaction, and cleanup; -- typed scenario data and matrix helpers describe stable scenario IDs and - supported combinations without becoming a second runner; -- product-facing manifests remain declarative setup inputs, not executable test - programs; -- assertion modules prefer TypeScript probes and typed client helpers; -- shell is used only when the system under test is a shell command, host - process, container command, or platform-specific probe; -- every shell call goes through a controlled spawn boundary with scoped - environment, timeout, redaction, artifact capture, and command/argument - validation; -- bridge work that expands the YAML/bash runner must also identify how that - behavior will move into Vitest fixtures before legacy runner paths are - removed. - -The #4347-#4357 audit-phase issues should be read as acceptance coverage -requirements, not as a permanent requirement to keep YAML resolver or bash -runner deliverables. If a phase issue names YAML or shell-runner artifacts, map -that requirement to equivalent single-runner behavior unless maintainers -explicitly decide to keep a bridge path for the current migration step. - -## Layered scenario model - -The conceptual model is layered: +| Live scenario IDs and metadata | `test/e2e-scenario/scenarios/registry.ts`, `test/e2e-scenario/scenarios/scenarios/baseline.ts` | +| GitHub Actions matrix emission | `test/e2e-scenario/scenarios/run.ts --emit-live-matrix` | +| Live scenario execution | `test/e2e-scenario/live/registry-scenarios.test.ts` | +| Phase fixtures and clients | `test/e2e-scenario/framework/` | +| Expected-state probes | `test/e2e-scenario/scenarios/expected-states.ts` | +| Product-facing setup/onboarding state | `test/e2e-scenario/manifests/*.yaml` | +| Legacy direct E2E coverage | `test/e2e/test-*.sh` and their workflows | +| Deletion guard inventory | `test/e2e-scenario/migration/legacy-inventory.json` | + +## Scenario Model + +The typed registry still describes scenarios as layered metadata: ```text base environment - → onboarding profile / manifest - → onboarding assertions - → expected state - → post-onboard suites + -> onboarding profile / manifest + -> expected state + -> optional lifecycle profile + -> suite metadata for migration tracking ``` -The current YAML shell runner expresses this through: - -- `base_scenarios`: platform + install + runtime -- `onboarding_profiles`: user onboarding choices -- `test_plans`: base + onboarding + expected state + suites -- `setup_scenarios`: friendly aliases and compatibility metadata -- `onboarding_assertions`: setup/onboarding checks that run before suites - -The typed scenario registry expresses the same intent as deterministic code and -is used by the scenario workflow matrix and dry-run plan artifacts. The target -Vitest fixture model should collapse these parallel expressions into one live -execution path. - -## Fixture-first scenario shape - -Final-state live scenarios should read like regular Vitest tests that depend on -NemoClaw fixtures: - -```ts -import { test } from "../framework/e2e-test.ts"; - -test("ubuntu repo cloud OpenClaw", async ({ - repo, - openclaw, - gateway, - sandbox, - inference, -}) => { - await repo.installCurrent(); - - const instance = await openclaw.onboard({ - agent: "openclaw", - provider: "nvidia", - }); - - await gateway.expectHealthy(instance); - await sandbox.expectRunning(instance); - await inference.expectLocalChat(instance, { prompt: "Say ok.", expect: /ok/i }); -}); -``` +Live execution happens through Vitest fixtures: -The test body should express product behavior. Fixture implementations should -hide redacted process spawning, artifact paths, cleanup registration, secret -gating, and retry/flake classification. +- `environment` checks CLI/install/runtime readiness. +- `onboard` performs supported onboarding profiles. +- `lifecycle` performs supported post-onboard mutations. +- `stateValidation` probes host-observable expected state. +- `artifacts`, `secrets`, `cleanup`, and `shellProbe` provide shared fixture + services. -## How to run +`suiteIds` remain metadata for reporting and migration planning. They do not +dispatch shell validation suites. -The TypeScript runner is the canonical entrypoint. There is one execution -mode — live — and `--plan-only` is for local debug only (it must not appear -in any CI workflow). +## How To Run ```bash # List canonical scenario ids npx tsx test/e2e-scenario/scenarios/run.ts --list # Emit the GitHub Actions fan-out matrix payload -npx tsx test/e2e-scenario/scenarios/run.ts --emit-matrix +npx tsx test/e2e-scenario/scenarios/run.ts --emit-live-matrix -# Execute one or more scenarios live -npx tsx test/e2e-scenario/scenarios/run.ts --scenarios +# Emit the matrix for selected scenario ids +npx tsx test/e2e-scenario/scenarios/run.ts --emit-live-matrix --scenarios ubuntu-repo-cloud-openclaw -# Local debug only: print the compiled plan without executing -npx tsx test/e2e-scenario/scenarios/run.ts --scenarios --plan-only +# Framework tests +npx vitest run --project e2e-scenario-framework --silent=false --reporter=default -# Opt-in Vitest live scenario path +# Opt-in live Vitest scenarios npm run build:cli NEMOCLAW_RUN_E2E_SCENARIOS=1 npx vitest run --project e2e-scenarios-live --silent=false --reporter=default ``` -Override the runtime context directory with `E2E_CONTEXT_DIR=` (default -`.e2e/`, gitignored). Suites communicate through `$E2E_CONTEXT_DIR/context.env`; -suites should not rediscover setup state. +The retired `--emit-matrix`, direct `--scenarios` execution, and `--plan-only` +paths must not be reintroduced. -## Repository layout +## Repository Layout ```text test/e2e-scenario/ - docs/ # This guide and migration notes - manifests/ # Product-facing NemoClawInstance desired state - scenarios/ # Typed builders, registry, compiler, assertions, dry-run orchestration - nemoclaw_scenarios/ # YAML runtime metadata and setup helpers - scenarios.yaml - install/ - onboard/ - fixtures/ - helpers/ - validation_suites/ # Suite definitions and shell assertion steps - suites.yaml - smoke/ - inference/ - messaging/ - platform/ - security/ - sandbox/ - runtime/ # Shared shell helper libs sourced by validation_suites - lib/ + docs/ # Framework guide, migration notes, retirement record + framework/ # Vitest fixtures, clients, redaction, artifacts, cleanup + framework-tests/ # Fast framework and metadata tests + live/ # Opt-in live Vitest scenario tests + manifests/ # Product-facing NemoClawInstance desired state + migration/ # Machine-readable deletion guard inventory + scenarios/ # Typed registry, matrix helpers, expected states ``` -## CI entry points +## CI Entry Points -- `.github/workflows/e2e-scenarios.yaml` runs typed scenario dry-runs for - manually selected scenario IDs. -- `.github/workflows/e2e-scenarios-all.yaml` fans out typed scenario dry-runs - from the typed registry matrix. -- `.github/workflows/e2e-vitest-scenarios.yaml` runs the opt-in Vitest live - scenario project and uploads non-hidden `e2e-artifacts/vitest/` fixture artifacts. +- `.github/workflows/e2e-vitest-scenarios.yaml` runs selected or all supported + live Vitest scenarios and uploads an explicit artifact allowlist with + JSON summaries plus action, log, and shell command-evidence directories under + 14-day retention. - Existing workflows such as `nightly-e2e.yaml`, `e2e-branch-validation.yaml`, `macos-e2e.yaml`, `wsl-e2e.yaml`, `ollama-proxy-e2e.yaml`, and - `regression-e2e.yaml` still run legacy live E2E scripts during the migration. -- `vitest.config.ts` contains the `e2e-scenario-framework` project for framework - and metadata tests. The live scenario target should be a separate opt-in - Vitest project so ordinary `npm test` remains fast and local-friendly. - -## Migration tracking - -Migration status is tracked outside the repository in GitHub issues and PRs, -not in repo-local checklists. The parent architecture issue is #3588. Active -audit-coverage work is tracked by the #4347–#4357 issue set, with focused -follow-ups such as #4378 for specific drift fixes. The execution-model decision -is tracked in #4941. - -The narrow repo-local exception is -`test/e2e-scenario/migration/legacy-inventory.json`, a machine-readable deletion -gate for direct legacy `test/e2e/test-*.sh` entrypoints and explicit bridge -entrypoints. It also tracks coarse internal legacy runner surfaces such as -scenario shell workers, validation suites, shell-runner orchestrators, and -runtime helper libraries so they cannot be removed without #4357 evidence. It -should prevent accidental deletions, not become a parallel status table. Remove -it after #4357 completes final legacy E2E reconciliation, or keep it only as an -audit artifact if maintainers still need that record. - -The old workflow-level parity report has been removed. Use scenario framework -tests, the coverage report, PR review, and the audit issues to decide what to -migrate next. - -When adding a suite assertion, emit or preserve a stable `PASS: ` / -`FAIL: ` log line, and record migration evidence or follow-up state in the -owning issue or PR. Sandbox lifecycle assertions should use -`validation_suites/lib/sandbox_lifecycle.sh`, consume -`$E2E_CONTEXT_DIR/context.env`, and keep destructive snapshot restore checks -isolated in the opt-in `snapshot-lifecycle` suite. Platform-specific scenarios -such as GPU, macOS, WSL, Brev, or DGX Spark must also list -`runner_requirements` in `scenarios.yaml`. - -Prefer new scenario-matrix coverage over new legacy-style `test-*.sh` scripts. + `regression-e2e.yaml` still run direct legacy E2E scripts during migration. +- `vitest.config.ts` contains `e2e-scenario-framework` for fast framework tests + and `e2e-scenarios-live` for opt-in live scenario execution. + +## Migration Tracking + +Migration status is tracked outside the repository. + +GitHub issues and PRs own changing migration status. The key issues are: + +- #3588: parent layered E2E architecture epic +- #4941: Vitest fixtures as the scenario execution model +- #4990: phase fixtures and registry-driven live discovery +- #5098: direct legacy bash-suite migration epic + +The repo-local inventory at +`test/e2e-scenario/migration/legacy-inventory.json` is a deletion gate, not a +progress dashboard. It prevents accidental deletion of direct legacy E2E +scripts and records the retired internal typed-shell runner surfaces. + +Prefer new scenario coverage in Vitest fixtures unless shell itself is the +contract or an existing legacy umbrella test is intentionally kept for +end-to-end install/user-flow fidelity. diff --git a/test/e2e-scenario/docs/RETIREMENT.md b/test/e2e-scenario/docs/RETIREMENT.md new file mode 100644 index 00000000000..6db7bc94ab1 --- /dev/null +++ b/test/e2e-scenario/docs/RETIREMENT.md @@ -0,0 +1,74 @@ + + + +# Typed-Shell Scenario Runner Retirement + +PR #5106 retired the typed-shell scenario runner as part of #5098 Phase 0. + +## What Was Removed + +- `.github/workflows/e2e-scenarios.yaml` +- `.github/workflows/e2e-scenarios-all.yaml` +- `test/e2e-scenario/scenarios/compiler.ts` +- `test/e2e-scenario/scenarios/orchestrators/` +- `test/e2e-scenario/scenarios/assertions/` +- `test/e2e-scenario/scenarios/probes/` +- `test/e2e-scenario/nemoclaw_scenarios/` +- `test/e2e-scenario/onboarding_assertions/` +- `test/e2e-scenario/validation_suites/` +- `test/e2e-scenario/runtime/lib/` +- `test/e2e-scenario/runtime/reports/` +- `scripts/e2e/lint-conventions.ts` + +## Why + +The project chose Vitest fixtures as the scenario execution model in #4941. +Keeping the typed-shell runner meant maintaining a second execution path with +its own compiler, phase orchestration, shell workers, suite dispatcher, and +workflows. + +Before deleting that path, the surviving Vitest workflow gained the reporting +and artifact shape operators needed from the retired workflows: + +- dispatch-time matrix summary with Scenario, Runner, and Label columns; +- per-scenario `run-plan.json`; +- per-phase `environment.result.json`, `onboarding.result.json`, and + `state-validation.result.json`; +- per-scenario step summary rendered from `run-plan.json`; +- explicit artifact upload allowlist with action, log, shell command-evidence, + and JSON summary paths plus 14-day retention. + +## What Replaced It + +- `test/e2e-scenario/scenarios/run.ts --emit-live-matrix` emits the live + GitHub Actions matrix. +- `.github/workflows/e2e-vitest-scenarios.yaml` runs the live matrix. +- `test/e2e-scenario/live/registry-scenarios.test.ts` executes supported + registry scenarios through Vitest. +- `test/e2e-scenario/framework/` owns fixtures, clients, shell-probe bridges, + artifact writing, cleanup, and redaction. + +## What Was Not Removed + +Direct legacy E2E scripts under `test/e2e/test-*.sh` remain in place. Those +scripts are governed by #5098 and +`test/e2e-scenario/migration/legacy-inventory.json`. They should be migrated, +augmented, or kept by family according to their KEEP_BASH, HYBRID, or +MIGRATE_TYPED classification. + +That includes the security and messaging contracts that the deleted typed-shell +validation suites used to mirror. Until #5098 migrates those families into +Vitest scenario fixtures, the active source of truth remains: + +- `test/e2e/test-credential-sanitization.sh` and + `test/e2e/test-credential-migration.sh` for credential leak prevention and + host credential-store hardening. +- `test/e2e/test-network-policy.sh`, `test/e2e/test-brave-search-e2e.sh`, and + `test/e2e/test-openshell-gateway-upgrade.sh` for network policy and gateway + credential-rewrite behavior. +- `test/e2e/test-shields-config.sh` for shields, config permissions, and + redacted config output. +- `test/e2e/test-telegram-injection.sh`, `test/e2e/test-messaging-providers.sh`, + `test/e2e/test-channels-add-remove.sh`, and + `test/e2e/test-channels-stop-start.sh` for messaging injection, channel + policy preservation, bridge credential isolation, and provider rewrite paths. diff --git a/test/e2e-scenario/framework-tests/e2e-assertion-modules.test.ts b/test/e2e-scenario/framework-tests/e2e-assertion-modules.test.ts deleted file mode 100644 index 65fe0699ca0..00000000000 --- a/test/e2e-scenario/framework-tests/e2e-assertion-modules.test.ts +++ /dev/null @@ -1,149 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { describe, expect, it } from "vitest"; -import fs from "node:fs"; -import path from "node:path"; -import yaml from "js-yaml"; - -import { - assertionGroupForSuite, - assertionGroupsForScenario, - assertionRegistry, - validateAssertionGroups, -} from "../scenarios/assertions/registry.ts"; -import { listScenarios } from "../scenarios/registry.ts"; -import type { AssertionGroup } from "../scenarios/types.ts"; - -const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); -const E2E_DIR = path.join(REPO_ROOT, "test/e2e-scenario"); -const SUITES_PATH = path.join(E2E_DIR, "validation_suites", "suites.yaml"); - -type AnyRecord = Record; - -function loadYaml(filePath: string): AnyRecord { - const doc = yaml.load(fs.readFileSync(filePath, "utf8")); - if (!doc || typeof doc !== "object") { - throw new Error(`${filePath} did not parse to an object`); - } - return doc as AnyRecord; -} - -function allPlannedAssertionGroupIds(): Set { - return new Set( - listScenarios().flatMap((scenario) => - assertionGroupsForScenario(scenario).map((group) => group.id), - ), - ); -} - -describe("assertion modules", () => { - it("should define onboarding assertions in modules", () => { - const onboardingGroups = assertionRegistry.groups.filter( - (group) => group.phase === "onboarding", - ); - const stepIds = new Set( - onboardingGroups.flatMap((group) => group.steps.map((step) => step.id)), - ); - - for (const id of [ - "onboarding.base.cli-installed", - "onboarding.preflight.passed", - "onboarding.preflight.expected-failed", - ]) { - expect(stepIds.has(id), `missing onboarding step ${id}`).toBe(true); - } - for (const step of onboardingGroups.flatMap((group) => group.steps)) { - expect(step.phase).toBe("onboarding"); - expect(step.implementation?.ref).toMatch(/^test\/e2e-scenario\/onboarding_assertions\//); - } - }); - - it("should map every old validation suite to canonical assertion group", () => { - const suites = loadYaml(SUITES_PATH).suites as AnyRecord; - - for (const suiteId of Object.keys(suites)) { - const group = assertionGroupForSuite(suiteId); - expect(group?.id, `missing assertion group for suite ${suiteId}`).toBe(`suite.${suiteId}`); - expect(group?.steps.length, `suite ${suiteId} must not be alias-only`).toBeGreaterThan(0); - expect(group?.steps.every((step) => step.implementation?.kind !== "pending")).toBe(true); - } - }); - - it("should keep snapshot suite distinct from snapshot lifecycle", () => { - const snapshot = assertionGroupForSuite("snapshot"); - const snapshotLifecycle = assertionGroupForSuite("snapshot-lifecycle"); - - expect(snapshot?.steps.map((step) => step.id)).toEqual(["runtime.snapshot.sandbox-listed"]); - expect(snapshot?.steps.map((step) => step.implementation?.ref)).toEqual([ - "test/e2e-scenario/validation_suites/smoke/02-sandbox-listed.sh", - ]); - expect(snapshotLifecycle?.steps.map((step) => step.implementation?.ref)).toEqual([ - "test/e2e-scenario/validation_suites/sandbox/snapshot/00-create-list-restore.sh", - ]); - }); - - it("should require each assertion group to have steps", () => { - const emptyGroup: AssertionGroup = { id: "empty", phase: "runtime", steps: [] }; - - expect(() => - validateAssertionGroups([...assertionRegistry.groups, emptyGroup], E2E_DIR), - ).toThrow(/empty/); - }); - - it("should require each assertion group to be used by a scenario plan", () => { - const planned = allPlannedAssertionGroupIds(); - const unused = assertionRegistry.groups - .map((group) => group.id) - .filter((id) => !planned.has(id)); - - expect(unused, `unused assertion groups: ${unused.join(", ")}`).toEqual([]); - }); - - it("should fail when assertion step references missing script", () => { - const badGroup: AssertionGroup = { - id: "bad.missing-script", - phase: "runtime", - steps: [ - { - id: "bad.missing-script.step", - phase: "runtime", - implementation: { - kind: "shell", - ref: "test/e2e-scenario/validation_suites/does-not-exist.sh", - }, - evidencePath: ".e2e/bad.log", - }, - ], - }; - - expect(() => validateAssertionGroups([badGroup], E2E_DIR)).toThrow(/does-not-exist/); - }); - - it("should fail when retry attempts lack classifier", () => { - const badGroup: AssertionGroup = { - id: "bad.retry", - phase: "runtime", - steps: [ - { - id: "bad.retry.step", - phase: "runtime", - implementation: { kind: "probe", ref: "fakeProbe" }, - evidencePath: ".e2e/bad.log", - reliability: { retry: { attempts: 2, on: [] } }, - }, - ], - }; - - expect(() => validateAssertionGroups([badGroup], E2E_DIR)).toThrow(/classifier|retry/i); - }); - - it("should block complete status for manual classification steps", () => { - expect(() => validateAssertionGroups(assertionRegistry.groups, E2E_DIR)).not.toThrow( - /needs-manual-classification/, - ); - expect(assertionRegistry.groups.every((group) => group.migrationStatus === "complete")).toBe( - true, - ); - }); -}); diff --git a/test/e2e-scenario/framework-tests/e2e-context-helper.test.ts b/test/e2e-scenario/framework-tests/e2e-context-helper.test.ts deleted file mode 100644 index 303c1b2a512..00000000000 --- a/test/e2e-scenario/framework-tests/e2e-context-helper.test.ts +++ /dev/null @@ -1,88 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { describe, it, expect } from "vitest"; -import { spawnSync, type SpawnSyncReturns } from "node:child_process"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); -const CONTEXT_LIB = path.join(REPO_ROOT, "test/e2e-scenario/runtime/lib/context.sh"); - -function runBash(script: string, env: Record = {}): SpawnSyncReturns { - return spawnSync("bash", ["--noprofile", "--norc"], { - env: { ...process.env, ...env }, - encoding: "utf8", - input: script, - timeout: Number(process.env.E2E_SPAWN_TIMEOUT_MS ?? 60_000), - cwd: REPO_ROOT, - }); -} - -describe("E2E context helper (runtime/lib/context.sh)", () => { - it("context helper writes and sources values", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-ctx-")); - try { - const script = ` - set -euo pipefail - . "${CONTEXT_LIB}" - export E2E_CONTEXT_DIR="${tmp}" - e2e_context_init - e2e_context_set E2E_SCENARIO ubuntu-repo-cloud-openclaw - e2e_context_set E2E_AGENT openclaw - # In a fresh shell, source the context and print the values. - bash -c 'set -euo pipefail; . "${tmp}/context.env"; echo "SCENARIO=$E2E_SCENARIO"; echo "AGENT=$E2E_AGENT"' - `; - const r = runBash(script); - expect(r.status, r.stderr).toBe(0); - expect(r.stdout).toContain("SCENARIO=ubuntu-repo-cloud-openclaw"); - expect(r.stdout).toContain("AGENT=openclaw"); - expect(fs.existsSync(path.join(tmp, "context.env"))).toBe(true); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("context require fails for missing values", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-ctx-")); - try { - const script = ` - set -euo pipefail - . "${CONTEXT_LIB}" - export E2E_CONTEXT_DIR="${tmp}" - e2e_context_init - e2e_context_require E2E_SANDBOX_NAME - `; - const r = runBash(script); - expect(r.status).not.toBe(0); - expect(r.stderr).toMatch(/E2E_SANDBOX_NAME/); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("context dump redacts sensitive values", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-ctx-")); - try { - const script = ` - set -euo pipefail - . "${CONTEXT_LIB}" - export E2E_CONTEXT_DIR="${tmp}" - e2e_context_init - e2e_context_set E2E_SCENARIO ubuntu-repo-cloud-openclaw - e2e_context_set NVIDIA_API_KEY super-secret-api-key-value - e2e_context_set OPENAI_API_TOKEN nothing-to-see-here-token - e2e_context_dump - `; - const r = runBash(script); - expect(r.status, r.stderr).toBe(0); - expect(r.stdout).not.toContain("super-secret-api-key-value"); - expect(r.stdout).not.toContain("nothing-to-see-here-token"); - expect(r.stdout).toMatch(/NVIDIA_API_KEY=.*REDACTED/); - expect(r.stdout).toContain("ubuntu-repo-cloud-openclaw"); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); -}); diff --git a/test/e2e-scenario/framework-tests/e2e-convention-lint.test.ts b/test/e2e-scenario/framework-tests/e2e-convention-lint.test.ts deleted file mode 100644 index 1c58fed66f8..00000000000 --- a/test/e2e-scenario/framework-tests/e2e-convention-lint.test.ts +++ /dev/null @@ -1,107 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { describe, it, expect, beforeEach, afterEach } from "vitest"; -import { spawnSync, type SpawnSyncReturns } from "node:child_process"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); -const LINT_BIN = path.join(REPO_ROOT, "scripts/e2e/lint-conventions.ts"); - -function runTsx( - scriptPath: string, - args: string[] = [], - env: Record = {}, -): SpawnSyncReturns { - const tsx = path.join(REPO_ROOT, "node_modules/.bin/tsx"); - return spawnSync(tsx, [scriptPath, ...args], { - env: { ...process.env, ...env }, - encoding: "utf8", - timeout: Number(process.env.E2E_SPAWN_TIMEOUT_MS ?? 60_000), - cwd: REPO_ROOT, - }); -} - -/** - * Create a synthetic repo layout mirroring the paths the lint walks: - * /test/e2e-scenario/validation_suites//.sh (suite step scripts) - * /test/e2e/test-*.sh (legacy scripts) - */ -function makeSyntheticRepo(): string { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-lint-")); - fs.mkdirSync(path.join(tmp, "test/e2e-scenario/validation_suites/example"), { recursive: true }); - fs.mkdirSync(path.join(tmp, "test/e2e"), { recursive: true }); - return tmp; -} - -function writeStep(tmp: string, name: string, body: string) { - const p = path.join(tmp, "test/e2e-scenario/validation_suites/example", name); - fs.writeFileSync(p, `#!/usr/bin/env bash\n${body}\n`); -} - -function writeLegacy(tmp: string, name: string, body: string) { - const p = path.join(tmp, "test/e2e", name); - fs.writeFileSync(p, `#!/usr/bin/env bash\n${body}\n`); -} - -describe("Phase 1.G convention lint", () => { - let tmp: string; - beforeEach(() => { - tmp = makeSyntheticRepo(); - }); - afterEach(() => { - fs.rmSync(tmp, { recursive: true, force: true }); - }); - - it("flags steps that reexport noninteractive env", () => { - writeStep(tmp, "00-bad.sh", "export DEBIAN_FRONTEND=noninteractive\necho hi"); - const r = runTsx(LINT_BIN, ["--root", tmp]); - expect(r.status).not.toBe(0); - expect(r.stdout + r.stderr).toMatch(/00-bad\.sh/); - expect(r.stdout + r.stderr).toMatch(/DEBIAN_FRONTEND|non.?interactive/i); - }); - - it("flags steps that register their own trap", () => { - writeStep(tmp, "00-trap.sh", "trap cleanup EXIT"); - const r = runTsx(LINT_BIN, ["--root", tmp]); - expect(r.status).not.toBe(0); - expect(r.stdout + r.stderr).toMatch(/00-trap\.sh/); - expect(r.stdout + r.stderr).toMatch(/trap/i); - }); - - it("flags steps that call section", () => { - writeStep(tmp, "00-section.sh", 'section "Phase 3: X"'); - const r = runTsx(LINT_BIN, ["--root", tmp]); - expect(r.status).not.toBe(0); - expect(r.stdout + r.stderr).toMatch(/00-section\.sh/); - expect(r.stdout + r.stderr).toMatch(/section/i); - }); - - it("flags steps that write to a tmp log path", () => { - writeStep(tmp, "00-tmplog.sh", "echo hi > /tmp/foo.log"); - const r = runTsx(LINT_BIN, ["--root", tmp]); - expect(r.status).not.toBe(0); - expect(r.stdout + r.stderr).toMatch(/00-tmplog\.sh/); - expect(r.stdout + r.stderr).toMatch(/\/tmp.*\.log|E2E_CONTEXT_DIR/); - }); - - it("flags nonstandard repo root discovery patterns", () => { - writeStep(tmp, "00-reporoot.sh", 'REPO_ROOT="$(git rev-parse --show-toplevel)"'); - const r = runTsx(LINT_BIN, ["--root", tmp]); - expect(r.status).not.toBe(0); - expect(r.stdout + r.stderr).toMatch(/repo.?root|git rev-parse/i); - }); - - it("does not require legacy scripts to update the parity map", () => { - writeLegacy(tmp, "test-new-thing.sh", '# legacy script\npass "something"'); - const r = runTsx(LINT_BIN, ["--root", tmp]); - expect(r.status, r.stdout + r.stderr).toBe(0); - }); - - it("passes on the current repo state", () => { - const r = runTsx(LINT_BIN); - expect(r.status, r.stdout + r.stderr).toBe(0); - }); -}); diff --git a/test/e2e-scenario/framework-tests/e2e-expected-state.test.ts b/test/e2e-scenario/framework-tests/e2e-expected-state.test.ts index f89cf8a77ad..8d255b484c6 100644 --- a/test/e2e-scenario/framework-tests/e2e-expected-state.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-expected-state.test.ts @@ -2,35 +2,18 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it } from "vitest"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { compileRunPlans } from "../scenarios/compiler.ts"; import { getExpectedState, listExpectedStates, probesForState, requireExpectedState, } from "../scenarios/expected-states.ts"; -import { ScenarioRunner } from "../scenarios/orchestrators/runner.ts"; import { listScenarios } from "../scenarios/registry.ts"; -import type { - ExpectedState, - PhaseName, - PhaseResult, - RunContext, - RunPlanPhase, -} from "../scenarios/types.ts"; +import type { ExpectedState } from "../scenarios/types.ts"; -function freshCtx(): RunContext { - return { contextDir: fs.mkdtempSync(path.join(os.tmpdir(), "e2e-state-")) }; -} - -// The legacy parity tests against `nemoclaw_scenarios/expected-states.yaml` -// were retired alongside the YAML resolver path (see commit 9da75ac0a). // The typed registry in `scenarios/expected-states.ts` is the single source -// of truth; these id-coverage assertions replace the YAML-mirror checks. +// of truth for live Vitest state-validation fixtures. describe("typed expected-state registry id coverage", () => { it("exposes a non-empty list of registered expected-state ids", () => { const ids = listExpectedStates().map((s) => s.id); @@ -129,233 +112,6 @@ describe("probesForState maps typed expected-state into probe ids", () => { }); }); -describe("compiler emits state-validation phase actions from expected-state registry", () => { - it("positive scenario gets cli-installed + gateway-healthy + sandbox-running probe actions", () => { - const [plan] = compileRunPlans(["ubuntu-repo-cloud-openclaw"]); - const stateValidationPhase = plan.phases.find((p) => p.name === "state-validation"); - expect(stateValidationPhase).toBeTruthy(); - expect(stateValidationPhase!.actions.map((a) => a.id)).toEqual([ - "state-validation.cli-installed", - "state-validation.gateway-healthy", - "state-validation.sandbox-running", - ]); - // Probes are typed shell-fn actions that go through the shared - // dispatcher; the orchestrator owns timeouts and redaction. - for (const action of stateValidationPhase!.actions) { - expect(action.kind).toBe("shell-fn"); - expect(action.fn).toBe("e2e_state_probe"); - expect(action.scriptRef).toBe("test/e2e-scenario/nemoclaw_scenarios/probes/dispatch.sh"); - expect(action.timeoutSeconds).toBe(30); - } - }); - - it("negative scenario gets cli-installed + gateway-absent + sandbox-absent probe actions", () => { - const [plan] = compileRunPlans(["ubuntu-no-docker-preflight-negative"]); - const stateValidationPhase = plan.phases.find((p) => p.name === "state-validation"); - expect(stateValidationPhase).toBeTruthy(); - expect(stateValidationPhase!.actions.map((a) => a.id)).toEqual([ - "state-validation.cli-installed", - "state-validation.gateway-absent", - "state-validation.sandbox-absent", - ]); - }); - - it("compiler fails hard on a scenario referencing an unknown expected-state ID", () => { - expect(() => - compileRunPlans([ - { - id: "synthetic-unknown-state", - assertionGroups: [], - expectedStateId: "definitely-not-a-state", - }, - ]), - ).toThrow(/unknown expected_state/); - }); - - it("phase order is environment -> onboarding -> state-validation -> lifecycle -> runtime", () => { - const [plan] = compileRunPlans(["ubuntu-repo-cloud-openclaw"]); - // 'lifecycle' is the post-onboard state-mutation phase. Scenarios - // without a `environment.lifecycle` profile (e.g. this one) emit - // an empty action list for the phase but the phase still appears - // in the plan so phase-order invariants stay deterministic. - expect(plan.phases.map((p) => p.name)).toEqual([ - "environment", - "onboarding", - "state-validation", - "lifecycle", - "runtime", - ]); - }); -}); - -describe("ScenarioRunner short-circuit semantics around state-validation", () => { - it("onboarding action failure does NOT block state-validation (negative scenarios verify absent state)", async () => { - const ctx = freshCtx(); - try { - const [plan] = compileRunPlans(["ubuntu-no-docker-preflight-negative"]); - const phase = ( - name: PhaseName, - outcome: PhaseResult, - ): { run: (ctx: RunContext, p: RunPlanPhase) => Promise } => ({ - run: async () => outcome, - }); - - let stateValidationCalled = false; - let runtimeCalled = false; - const runner = new ScenarioRunner({ - environment: phase("environment", { - phase: "environment", - status: "passed", - actions: [], - assertions: [], - }), - onboarding: phase("onboarding", { - phase: "onboarding", - status: "failed", - actions: [ - { - id: "onboarding.profile.cloud-openclaw-no-docker", - status: "failed", - durationMs: 1, - message: "preflight detected docker-missing", - }, - ], - assertions: [], - }), - stateValidation: { - run: async () => { - stateValidationCalled = true; - return { - phase: "state-validation", - status: "passed", - actions: [], - assertions: [], - }; - }, - }, - runtime: { - run: async () => { - runtimeCalled = true; - return { phase: "runtime", status: "passed", actions: [], assertions: [] }; - }, - }, - }); - - const results = await runner.run(ctx, plan); - expect(stateValidationCalled).toBe(true); - expect(runtimeCalled).toBe(false); - // state-validation has its real result; runtime is skipped with - // the blocking-action message. - const stateRes = results.find((r) => r.phase === "state-validation")!; - expect(stateRes.status).toBe("passed"); - const runtimeRes = results.find((r) => r.phase === "runtime")!; - expect(runtimeRes.status).toBe("skipped"); - } finally { - fs.rmSync(ctx.contextDir, { recursive: true, force: true }); - } - }); - - it("environment action failure blocks state-validation AND runtime", async () => { - const ctx = freshCtx(); - try { - const [plan] = compileRunPlans(["ubuntu-repo-cloud-openclaw"]); - let stateValidationCalled = false; - let runtimeCalled = false; - const runner = new ScenarioRunner({ - environment: { - run: async () => ({ - phase: "environment", - status: "failed", - actions: [ - { - id: "environment.install.repo-current", - status: "failed", - durationMs: 1, - message: "install dispatcher exit 1", - }, - ], - assertions: [], - }), - }, - onboarding: { - run: async () => ({ phase: "onboarding", status: "passed", actions: [], assertions: [] }), - }, - stateValidation: { - run: async () => { - stateValidationCalled = true; - return { - phase: "state-validation", - status: "passed", - actions: [], - assertions: [], - }; - }, - }, - runtime: { - run: async () => { - runtimeCalled = true; - return { phase: "runtime", status: "passed", actions: [], assertions: [] }; - }, - }, - }); - await runner.run(ctx, plan); - expect(stateValidationCalled).toBe(false); - expect(runtimeCalled).toBe(false); - } finally { - fs.rmSync(ctx.contextDir, { recursive: true, force: true }); - } - }); - - it("state-validation action failure blocks runtime", async () => { - const ctx = freshCtx(); - try { - const [plan] = compileRunPlans(["ubuntu-repo-cloud-openclaw"]); - let runtimeCalled = false; - const runner = new ScenarioRunner({ - environment: { - run: async () => ({ - phase: "environment", - status: "passed", - actions: [], - assertions: [], - }), - }, - onboarding: { - run: async () => ({ phase: "onboarding", status: "passed", actions: [], assertions: [] }), - }, - stateValidation: { - run: async () => ({ - phase: "state-validation", - status: "failed", - actions: [ - { - id: "state-validation.gateway-healthy", - status: "failed", - durationMs: 1, - message: "gateway unreachable at http://127.0.0.1:18789", - }, - ], - assertions: [], - }), - }, - runtime: { - run: async () => { - runtimeCalled = true; - return { phase: "runtime", status: "passed", actions: [], assertions: [] }; - }, - }, - }); - const results = await runner.run(ctx, plan); - expect(runtimeCalled).toBe(false); - const runtimeRes = results.find((r) => r.phase === "runtime")!; - expect(runtimeRes.status).toBe("skipped"); - expect(runtimeRes.assertions[0].message).toMatch(/state-validation\.gateway-healthy/); - } finally { - fs.rmSync(ctx.contextDir, { recursive: true, force: true }); - } - }); -}); - describe("expected-state registry covers every scenario referenced in the typed registry", () => { it("every ScenarioDefinition.expectedStateId resolves in the typed expected-state registry", () => { const referenced = new Set(); diff --git a/test/e2e-scenario/framework-tests/e2e-fixture-context.test.ts b/test/e2e-scenario/framework-tests/e2e-fixture-context.test.ts index 960168e6319..c1f19c8e0b6 100644 --- a/test/e2e-scenario/framework-tests/e2e-fixture-context.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-fixture-context.test.ts @@ -7,7 +7,7 @@ import path from "node:path"; import { describe, expect, expectTypeOf, it } from "vitest"; -import { ArtifactSink } from "../framework/artifacts.ts"; +import { ArtifactSink, createArtifactSink } from "../framework/artifacts.ts"; import { assertCleanupPassed, CleanupRegistry } from "../framework/cleanup.ts"; import { test as e2eTest } from "../framework/e2e-test.ts"; import { SecretStore } from "../framework/secrets.ts"; @@ -52,6 +52,70 @@ describe("E2E fixture primitives", () => { } }); + it("live scenario artifacts match the workflow upload allowlist paths", async () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-e2e-live-artifacts-")); + const previousArtifactDir = process.env.E2E_ARTIFACT_DIR; + const scenarioId = "ubuntu-repo-cloud-openclaw"; + const artifactParent = path.join(tmp, "e2e-artifacts", "vitest"); + const allowlistedFiles = [ + "run-plan.json", + "scenario.json", + "scenario-result.json", + "environment.result.json", + "onboarding.result.json", + "state-validation.result.json", + ]; + const shellEvidenceFiles = [ + "shell/command-evidence.result.json", + "shell/command-evidence.stdout.txt", + "shell/command-evidence.stderr.txt", + ]; + + try { + process.env.E2E_ARTIFACT_DIR = artifactParent; + const artifacts = createArtifactSink(scenarioId, tmp); + await artifacts.ensureRoot(); + + expect(artifacts.rootDir).toBe(path.resolve(artifactParent, scenarioId)); + for (const file of allowlistedFiles) { + await artifacts.writeJson(file, { scenarioId, file }); + } + const controller = new AbortController(); + const shellProbe = new ShellProbe({ + artifacts, + redact: (text) => text, + signal: controller.signal, + }); + const shellResult = await shellProbe.run( + trustedShellCommand({ + command: process.execPath, + args: ["-e", "console.log('shell evidence')"], + reason: "verify workflow allowlist preserves command evidence", + }), + { artifactName: "command-evidence", timeoutMs: 5_000 }, + ); + + expect(shellResult.exitCode).toBe(0); + + for (const file of allowlistedFiles) { + expect(fs.existsSync(path.join(artifactParent, scenarioId, file))).toBe(true); + } + for (const file of shellEvidenceFiles) { + expect(fs.existsSync(path.join(artifactParent, scenarioId, file))).toBe(true); + } + expect( + fs.existsSync(path.join(artifactParent, scenarioId, scenarioId, "run-plan.json")), + ).toBe(false); + } finally { + if (previousArtifactDir === undefined) { + delete process.env.E2E_ARTIFACT_DIR; + } else { + process.env.E2E_ARTIFACT_DIR = previousArtifactDir; + } + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + it("cleanup registry runs callbacks in reverse order", async () => { const cleanup = new CleanupRegistry(); const order: string[] = []; diff --git a/test/e2e-scenario/framework-tests/e2e-lib-helpers.test.ts b/test/e2e-scenario/framework-tests/e2e-lib-helpers.test.ts deleted file mode 100644 index de43932a621..00000000000 --- a/test/e2e-scenario/framework-tests/e2e-lib-helpers.test.ts +++ /dev/null @@ -1,1434 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { describe, it, expect } from "vitest"; -import { spawnSync, type SpawnSyncReturns } from "node:child_process"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); -const RUNTIME_LIB = path.join(REPO_ROOT, "test/e2e-scenario/runtime/lib"); -const VALIDATION_SUITES = path.join(REPO_ROOT, "test/e2e-scenario/validation_suites"); -const VALIDATION_LIB = path.join(VALIDATION_SUITES, "lib"); -const ASSERT = path.join(VALIDATION_SUITES, "assert"); -const REBUILD_UPGRADE_LIB = path.join(VALIDATION_SUITES, "lib/rebuild_upgrade.sh"); -const FIXTURES = path.join(REPO_ROOT, "test/e2e-scenario/nemoclaw_scenarios/fixtures"); -const ONBOARD_DIR = path.join(REPO_ROOT, "test/e2e-scenario/nemoclaw_scenarios/onboard"); - -function runBash(script: string, env: Record = {}): SpawnSyncReturns { - return spawnSync("bash", ["--noprofile", "--norc"], { - env: { ...process.env, ...env }, - encoding: "utf8", - input: script, - timeout: Number(process.env.E2E_SPAWN_TIMEOUT_MS ?? 60_000), - cwd: REPO_ROOT, - }); -} - -// ────────────────────────────────────────────────────────────────────────── -// Phase 1 helpers (logging, sandbox-exec, fixtures, assertions, install -// splits) — extends the pre-existing e2e shell helper coverage. -// ────────────────────────────────────────────────────────────────────────── - -describe("E2E shell helpers", () => { - it("should source inference routing helpers under strict shell mode", () => { - const r = runBash(` - set -euo pipefail - . "${VALIDATION_SUITES}/lib/inference_routing.sh" - declare -F e2e_inference_routing_assert_chat_completion - `); - expect(r.status, r.stderr).toBe(0); - }); - - it("should fail clearly when required context is missing", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-inf-missing-")); - try { - const r = runBash( - ` - set -euo pipefail - . "${RUNTIME_LIB}/context.sh" - . "${VALIDATION_SUITES}/lib/inference_routing.sh" - e2e_context_init - e2e_inference_routing_assert_chat_completion "post-onboard.inference-routing.inference-local-chat-completion" - `, - { E2E_CONTEXT_DIR: tmp }, - ); - expect(r.status).not.toBe(0); - expect(r.stderr).toMatch(/E2E_SANDBOX_NAME|E2E_CONTEXT_DIR|context/i); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("no-Docker onboarding worker should preserve seeded context and redact the log", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-no-docker-context-")); - const fakeBin = path.join(tmp, "bin"); - fs.mkdirSync(fakeBin); - fs.writeFileSync( - path.join(fakeBin, "nemoclaw"), - `#!/usr/bin/env bash -if [[ "\${1:-}" = "onboard" ]]; then - expected='onboard --non-interactive --yes --yes-i-accept-third-party-software' - if [[ "$*" != "\${expected}" ]]; then - echo "unexpected nemoclaw args: $*" >&2 - exit 2 - fi - if [[ "\${NEMOCLAW_AGENT:-}" != "openclaw" || "\${NEMOCLAW_PROVIDER:-}" != "cloud" || "\${NEMOCLAW_SANDBOX_NAME:-}" != "e2e-preserved" ]]; then - echo "unexpected nemoclaw env: agent=\${NEMOCLAW_AGENT:-unset} provider=\${NEMOCLAW_PROVIDER:-unset} sandbox=\${NEMOCLAW_SANDBOX_NAME:-unset}" >&2 - exit 2 - fi - echo "NVIDIA_API_KEY=\${NVIDIA_API_KEY:-unset}" >&2 - echo "Docker is required before onboarding" >&2 - exit 42 -fi -echo "unexpected nemoclaw invocation: $*" >&2 -exit 2 -`, - { mode: 0o755 }, - ); - try { - fs.writeFileSync( - path.join(tmp, "context.env"), - "E2E_SCENARIO=ubuntu-no-docker-preflight-negative\nE2E_SANDBOX_NAME=e2e-preserved\n", - ); - const r = runBash( - ` - set -euo pipefail - test/e2e-scenario/nemoclaw_scenarios/dispatch-action.sh e2e_onboard cloud-openclaw-no-docker "${ONBOARD_DIR}/dispatch.sh" - `, - { - E2E_ACTION_ID: "onboarding.profile.cloud-openclaw-no-docker", - E2E_CONTEXT_DIR: tmp, - E2E_PHASE: "onboarding", - NVIDIA_API_KEY: "secret-token", - PATH: `${fakeBin}:${process.env.PATH ?? ""}`, - TMPDIR: tmp, - }, - ); - expect(r.status, `${r.stdout}\n${r.stderr}`).toBe(0); - const contextBody = fs.readFileSync(path.join(tmp, "context.env"), "utf8"); - expect(contextBody).toMatch(/^E2E_SANDBOX_NAME=e2e-preserved$/m); - const logBody = fs.readFileSync(path.join(tmp, "negative-preflight.log"), "utf8"); - expect(logBody).toContain("Docker is required before onboarding"); - expect(logBody).toContain("[REDACTED]"); - expect(logBody).not.toContain("secret-token"); - const tempEntries = fs.readdirSync(tmp, { recursive: true }).map(String).join("\n"); - expect(tempEntries).not.toContain("negative-preflight.raw.log"); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("no-Docker onboarding worker should fail on unrelated onboarding errors", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-no-docker-unrelated-")); - const fakeBin = path.join(tmp, "bin"); - fs.mkdirSync(fakeBin); - fs.writeFileSync( - path.join(fakeBin, "nemoclaw"), - `#!/usr/bin/env bash -if [[ "\${1:-}" = "onboard" ]]; then - expected='onboard --non-interactive --yes --yes-i-accept-third-party-software' - if [[ "$*" != "\${expected}" ]]; then - echo "unexpected nemoclaw args: $*" >&2 - exit 2 - fi - if [[ "\${NEMOCLAW_AGENT:-}" != "openclaw" || "\${NEMOCLAW_PROVIDER:-}" != "cloud" || "\${NEMOCLAW_SANDBOX_NAME:-}" != "e2e-preserved" ]]; then - echo "unexpected nemoclaw env: agent=\${NEMOCLAW_AGENT:-unset} provider=\${NEMOCLAW_PROVIDER:-unset} sandbox=\${NEMOCLAW_SANDBOX_NAME:-unset}" >&2 - exit 2 - fi - echo "provider rejected NVIDIA_API_KEY=\${NVIDIA_API_KEY:-unset}" >&2 - exit 42 -fi -echo "unexpected nemoclaw invocation: $*" >&2 -exit 2 -`, - { mode: 0o755 }, - ); - try { - fs.writeFileSync( - path.join(tmp, "context.env"), - "E2E_SCENARIO=ubuntu-no-docker-preflight-negative\nE2E_SANDBOX_NAME=e2e-preserved\n", - ); - const r = runBash( - ` - set -euo pipefail - test/e2e-scenario/nemoclaw_scenarios/dispatch-action.sh e2e_onboard cloud-openclaw-no-docker "${ONBOARD_DIR}/dispatch.sh" - `, - { - E2E_ACTION_ID: "onboarding.profile.cloud-openclaw-no-docker", - E2E_CONTEXT_DIR: tmp, - E2E_PHASE: "onboarding", - NVIDIA_API_KEY: "secret-token", - PATH: `${fakeBin}:${process.env.PATH ?? ""}`, - TMPDIR: tmp, - }, - ); - expect(r.status).toBe(42); - expect(`${r.stdout}\n${r.stderr}`).toContain( - "failed without Docker-missing preflight signature", - ); - const logBody = fs.readFileSync(path.join(tmp, "negative-preflight.log"), "utf8"); - expect(logBody).toContain("provider rejected"); - expect(logBody).toContain("[REDACTED]"); - expect(logBody).not.toContain("secret-token"); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("no-Docker onboarding worker should accept current preflight wording", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-no-docker-wording-")); - const fakeBin = path.join(tmp, "bin"); - fs.mkdirSync(fakeBin); - fs.writeFileSync( - path.join(fakeBin, "nemoclaw"), - `#!/usr/bin/env bash -if [[ "\${1:-}" = "onboard" ]]; then - echo "Docker is not reachable. Please fix Docker and try again." >&2 - exit 1 -fi -echo "unexpected nemoclaw invocation: $*" >&2 -exit 2 -`, - { mode: 0o755 }, - ); - try { - fs.writeFileSync( - path.join(tmp, "context.env"), - "E2E_SCENARIO=ubuntu-no-docker-preflight-negative\nE2E_SANDBOX_NAME=e2e-preserved\n", - ); - const r = runBash( - ` - set -euo pipefail - test/e2e-scenario/nemoclaw_scenarios/dispatch-action.sh e2e_onboard cloud-openclaw-no-docker "${ONBOARD_DIR}/dispatch.sh" - `, - { - E2E_ACTION_ID: "onboarding.profile.cloud-openclaw-no-docker", - E2E_CONTEXT_DIR: tmp, - E2E_PHASE: "onboarding", - NVIDIA_API_KEY: "secret-token", - PATH: `${fakeBin}:${process.env.PATH ?? ""}`, - TMPDIR: tmp, - }, - ); - expect(r.status, `${r.stdout}\n${r.stderr}`).toBe(0); - const logBody = fs.readFileSync(path.join(tmp, "negative-preflight.log"), "utf8"); - expect(logBody).toContain("Docker is not reachable"); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("no-Docker redactor fallback should redact sensitive env values without Python", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-no-docker-redactor-")); - const noPythonBin = path.join(tmp, "bin"); - const logPath = path.join(tmp, "negative-preflight.log"); - try { - const r = runBash( - ` - set -euo pipefail - mkdir -p "${noPythonBin}" - for cmd in rm mktemp sed env cat mv; do - ln -s "$(command -v "\${cmd}")" "${noPythonBin}/\${cmd}" - done - . "${ONBOARD_DIR}/cloud-openclaw-no-docker.sh" - export NVIDIA_API_KEY=plain-secret-value - PATH="${noPythonBin}" - printf 'plain-secret-value\\nDocker is required before onboarding\\n' | e2e_no_docker_write_redacted_preflight_log "${logPath}" - `, - { TMPDIR: tmp }, - ); - expect(r.status, `${r.stdout}\n${r.stderr}`).toBe(0); - const logBody = fs.readFileSync(logPath, "utf8"); - expect(logBody).toContain("[REDACTED]"); - expect(logBody).toContain("Docker is required before onboarding"); - expect(logBody).not.toContain("plain-secret-value"); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("security policy credentials helper should load with context library", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-context-")); - try { - fs.writeFileSync( - path.join(tmp, "context.env"), - "E2E_SCENARIO=test\nE2E_PROVIDER=nvidia\nE2E_CREDENTIALS_EXPECTED=present\n", - ); - const r = runBash( - ` - set -euo pipefail - . "${VALIDATION_SUITES}/lib/security_policy_credentials.sh" - spc_require_context E2E_SCENARIO E2E_PROVIDER - echo "provider=$(spc_context_get E2E_PROVIDER)" - `, - { E2E_CONTEXT_DIR: tmp }, - ); - expect(r.status, r.stderr).toBe(0); - expect(r.stdout).toContain("provider=nvidia"); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("security policy credentials helper should fail when required context is missing", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-context-missing-")); - try { - fs.writeFileSync(path.join(tmp, "context.env"), "E2E_SCENARIO=test\n"); - const r = runBash( - ` - set -euo pipefail - . "${VALIDATION_SUITES}/lib/security_policy_credentials.sh" - spc_require_context E2E_PROVIDER - `, - { E2E_CONTEXT_DIR: tmp }, - ); - expect(r.status).not.toBe(0); - expect(r.stderr).toContain("E2E_PROVIDER"); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("security policy credentials helper should not log secret values", () => { - const r = runBash(` - set -euo pipefail - . "${VALIDATION_SUITES}/lib/security_policy_credentials.sh" - spc_log_provider_metadata "nvidia" "primary" - printf 'token=nvapi-secret-value-1234567890 sk-abcdefghijklmnop\n' | spc_redact_secret_text - `); - expect(r.status, r.stderr).toBe(0); - expect(r.stdout).toContain("provider=nvidia name=primary"); - expect(r.stdout).not.toMatch(/nvapi-secret-value|sk-abcdefghijklmnop/); - expect(r.stdout).toMatch(/\[REDACTED\]/); - }); - - it("security policy credentials helper should reject empty gateway credentials", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-credentials-empty-")); - const fakeBin = path.join(tmp, "bin"); - fs.mkdirSync(fakeBin); - fs.writeFileSync( - path.join(fakeBin, "nemoclaw"), - `#!/usr/bin/env bash -if [ "$1 $2" = "credentials list" ]; then - echo " No provider credentials registered." - exit 0 -fi -exit 2 -`, - { mode: 0o755 }, - ); - try { - fs.writeFileSync( - path.join(tmp, "context.env"), - "E2E_SCENARIO=test\nE2E_PROVIDER=nvidia\nE2E_CREDENTIALS_EXPECTED=present\n", - ); - const r = runBash( - ` - set -euo pipefail - . "${VALIDATION_SUITES}/lib/security_policy_credentials.sh" - spc_assert_credentials_expected - `, - { E2E_CONTEXT_DIR: tmp, PATH: `${fakeBin}:${process.env.PATH ?? ""}` }, - ); - expect(r.status).not.toBe(0); - expect(r.stderr).toMatch(/no gateway credentials/i); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("security policy credentials helper should reject raw credential leaks", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-credentials-leak-")); - const fakeBin = path.join(tmp, "bin"); - fs.mkdirSync(fakeBin); - fs.writeFileSync( - path.join(fakeBin, "nemoclaw"), - `#!/usr/bin/env bash -if [ "$1 $2" = "credentials list" ]; then - echo " Providers registered with the OpenShell gateway:" - echo " nvidia token=nvapi-secret-value-1234567890" - exit 0 -fi -exit 2 -`, - { mode: 0o755 }, - ); - try { - fs.writeFileSync( - path.join(tmp, "context.env"), - "E2E_SCENARIO=test\nE2E_PROVIDER=nvidia\nE2E_CREDENTIALS_EXPECTED=present\n", - ); - const r = runBash( - ` - set -euo pipefail - . "${VALIDATION_SUITES}/lib/security_policy_credentials.sh" - spc_assert_credentials_expected - `, - { E2E_CONTEXT_DIR: tmp, PATH: `${fakeBin}:${process.env.PATH ?? ""}` }, - ); - expect(r.status).not.toBe(0); - expect(r.stderr).toMatch(/secret-looking raw output/i); - expect(r.stdout).not.toContain("nvapi-secret-value"); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("security policy credentials helper should reject raw credential leaks from failed list", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-credentials-failed-leak-")); - const fakeBin = path.join(tmp, "bin"); - fs.mkdirSync(fakeBin); - fs.writeFileSync( - path.join(fakeBin, "nemoclaw"), - `#!/usr/bin/env bash -if [ "$1 $2" = "credentials list" ]; then - echo "gateway error token=nvapi-secret-value-1234567890" >&2 - exit 1 -fi -exit 2 -`, - { mode: 0o755 }, - ); - try { - fs.writeFileSync( - path.join(tmp, "context.env"), - "E2E_SCENARIO=test\nE2E_PROVIDER=nvidia\nE2E_CREDENTIALS_EXPECTED=present\n", - ); - const r = runBash( - ` - set -euo pipefail - . "${VALIDATION_SUITES}/lib/security_policy_credentials.sh" - spc_assert_credentials_expected - `, - { E2E_CONTEXT_DIR: tmp, PATH: `${fakeBin}:${process.env.PATH ?? ""}` }, - ); - expect(r.status).not.toBe(0); - expect(r.stderr).toMatch(/secret-looking raw output/i); - expect(r.stderr).not.toMatch(/credentials list failed/); - expect(r.stdout).not.toContain("nvapi-secret-value"); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("security policy credentials helper should verify policy and shields state", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-policy-shields-")); - const fakeBin = path.join(tmp, "bin"); - fs.mkdirSync(fakeBin); - fs.writeFileSync( - path.join(fakeBin, "nemoclaw"), - `#!/usr/bin/env bash -if [ "$1 $2" = "sb policy-list" ]; then - echo " Policy presets for sandbox 'sb':" - echo " ● telegram — Telegram bridge egress" - echo " ○ slack — Slack bridge egress" - exit 0 -fi -if [ "$1 $2 $3" = "sb shields status" ]; then - echo " Shields: UP (lockdown active)" - exit 0 -fi -exit 2 -`, - { mode: 0o755 }, - ); - fs.writeFileSync( - path.join(fakeBin, "openshell"), - `#!/usr/bin/env bash -if [ "$1 $2 $3" = "sandbox exec --name" ]; then - echo "440 root:root" - exit 0 -fi -exit 2 -`, - { mode: 0o755 }, - ); - try { - fs.writeFileSync( - path.join(tmp, "context.env"), - "E2E_SCENARIO=test\nE2E_PROVIDER=nvidia\nE2E_SANDBOX_NAME=sb\nE2E_AGENT=openclaw\nE2E_SHIELDS_EXPECTED_STATE=up\n", - ); - const r = runBash( - ` - set -euo pipefail - . "${VALIDATION_SUITES}/lib/security_policy_credentials.sh" - spc_assert_policy_preset_present telegram - spc_assert_shields_config_consistent - `, - { E2E_CONTEXT_DIR: tmp, PATH: `${fakeBin}:${process.env.PATH ?? ""}` }, - ); - expect(r.status, r.stderr).toBe(0); - expect(r.stdout).toContain("telegram"); - expect(r.stdout).toContain("shields config state is consistent: up"); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("security policy credentials helper should fail on missing policy preset", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-policy-missing-")); - const fakeBin = path.join(tmp, "bin"); - fs.mkdirSync(fakeBin); - fs.writeFileSync( - path.join(fakeBin, "nemoclaw"), - `#!/usr/bin/env bash -echo " slack — Slack bridge egress" -exit 0 -`, - { mode: 0o755 }, - ); - try { - fs.writeFileSync( - path.join(tmp, "context.env"), - "E2E_SCENARIO=test\nE2E_PROVIDER=nvidia\nE2E_SANDBOX_NAME=sb\n", - ); - const r = runBash( - ` - set -euo pipefail - . "${VALIDATION_SUITES}/lib/security_policy_credentials.sh" - spc_assert_policy_preset_present telegram - `, - { E2E_CONTEXT_DIR: tmp, PATH: `${fakeBin}:${process.env.PATH ?? ""}` }, - ); - expect(r.status).not.toBe(0); - expect(r.stderr).toMatch(/expected policy preset 'telegram'/); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("security policy credentials helper should verify OpenShell rewrite markers", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-openshell-")); - const fakeBin = path.join(tmp, "bin"); - fs.mkdirSync(fakeBin); - fs.writeFileSync( - path.join(fakeBin, "openshell"), - `#!/usr/bin/env bash -# request-body-credential-rewrite websocket-credential-rewrite -if [ "$1" = "--version" ]; then - echo "openshell 0.0.39" - exit 0 -fi -exit 0 -`, - { mode: 0o755 }, - ); - try { - fs.writeFileSync(path.join(tmp, "context.env"), "E2E_SCENARIO=test\n"); - const r = runBash( - ` - set -euo pipefail - . "${VALIDATION_SUITES}/lib/security_policy_credentials.sh" - spc_assert_openshell_credential_rewrite_supported - `, - { E2E_CONTEXT_DIR: tmp, PATH: `${fakeBin}:${process.env.PATH ?? ""}` }, - ); - expect(r.status, r.stderr).toBe(0); - expect(r.stdout).toContain("OpenShell 0.0.39 credential rewrite capability markers present"); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("security policy credentials helper should reject below minimum OpenShell version", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-openshell-old-")); - const fakeBin = path.join(tmp, "bin"); - fs.mkdirSync(fakeBin); - fs.writeFileSync( - path.join(fakeBin, "openshell"), - `#!/usr/bin/env bash -# request-body-credential-rewrite websocket-credential-rewrite -if [ "$1" = "--version" ]; then - echo "openshell 0.0.38" - exit 0 -fi -exit 0 -`, - { mode: 0o755 }, - ); - try { - fs.writeFileSync(path.join(tmp, "context.env"), "E2E_SCENARIO=test\n"); - const r = runBash( - ` - set -euo pipefail - . "${VALIDATION_SUITES}/lib/security_policy_credentials.sh" - spc_assert_openshell_credential_rewrite_supported - `, - { E2E_CONTEXT_DIR: tmp, PATH: `${fakeBin}:${process.env.PATH ?? ""}` }, - ); - expect(r.status).not.toBe(0); - expect(r.stderr).toContain("below credential rewrite minimum"); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("env helper should set standard noninteractive env", () => { - const r = runBash(` - set -euo pipefail - . "${RUNTIME_LIB}/env.sh" - e2e_env_apply_noninteractive - echo "NEMOCLAW_NON_INTERACTIVE=\${NEMOCLAW_NON_INTERACTIVE:-}" - echo "DEBIAN_FRONTEND=\${DEBIAN_FRONTEND:-}" - echo "CI=\${CI:-}" - `); - expect(r.status, r.stderr).toBe(0); - expect(r.stdout).toContain("NEMOCLAW_NON_INTERACTIVE=1"); - expect(r.stdout).toContain("DEBIAN_FRONTEND=noninteractive"); - }); - - it("gateway helper should fail clearly when URL is unreachable", () => { - // Source the supported gateway helper and aim it at a port very - // unlikely to be bound on the runner. The helper should exit - // non-zero, name the gateway, and surface the URL/port so on-call - // engineers can grep for it in failure logs. - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-gw-")); - try { - const r = runBash( - ` - set -euo pipefail - . "${ASSERT}/gateway-alive.sh" - e2e_context_init - e2e_context_set E2E_SCENARIO test - e2e_gateway_assert_healthy "http://127.0.0.1:65531" - `, - { E2E_CONTEXT_DIR: tmp }, - ); - expect(r.status).not.toBe(0); - expect(r.stderr).toMatch(/gateway/i); - expect(r.stderr).toMatch(/65531/); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("sandbox helper should fail for missing sandbox name", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-sb-")); - try { - // Initialise a context file without E2E_SANDBOX_NAME. - const r = runBash( - ` - set -euo pipefail - . "${RUNTIME_LIB}/context.sh" - . "${ASSERT}/sandbox-alive.sh" - e2e_context_init - e2e_context_set E2E_SCENARIO test - e2e_sandbox_assert_running - `, - { E2E_CONTEXT_DIR: tmp }, - ); - expect(r.status).not.toBe(0); - expect(r.stderr).toMatch(/E2E_SANDBOX_NAME/); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); -}); - -// ───────────────────────────────────────────────────────────────────────────── -// Phase 1.A — Logging helpers (lib/logging.sh) -// ───────────────────────────────────────────────────────────────────────────── - -describe("rebuild/upgrade validation helpers", () => { - it("rebuild/upgrade library should source without side effects", () => { - const r = runBash(` - set -euo pipefail - . "${REBUILD_UPGRADE_LIB}" - declare -F rebuild_upgrade_require_context >/dev/null - `); - expect(r.status, r.stderr).toBe(0); - expect(r.stdout + r.stderr).not.toMatch(/install|onboard|rebuild/i); - }); - - it("rebuild/upgrade context should fail with a missing key name", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-ru-")); - try { - fs.writeFileSync(path.join(tmp, "context.env"), "E2E_SCENARIO=test\n"); - const r = runBash( - ` - . "${REBUILD_UPGRADE_LIB}" - rebuild_upgrade_require_context - `, - { E2E_CONTEXT_DIR: tmp }, - ); - expect(r.status).not.toBe(0); - expect(r.stderr).toMatch(/E2E_AGENT|E2E_SANDBOX_NAME|E2E_GATEWAY_URL/); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("rebuild/upgrade context should pass when required keys are present", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-ru-")); - try { - fs.writeFileSync( - path.join(tmp, "context.env"), - "E2E_SCENARIO=test\nE2E_AGENT=openclaw\nE2E_SANDBOX_NAME=sb\nE2E_GATEWAY_URL=http://127.0.0.1\n", - ); - const r = runBash( - ` - set -euo pipefail - . "${REBUILD_UPGRADE_LIB}" - rebuild_upgrade_require_context - `, - { E2E_CONTEXT_DIR: tmp }, - ); - expect(r.status, r.stderr).toBe(0); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("rebuild/upgrade checks should allow command fakes", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-ru-")); - try { - fs.writeFileSync( - path.join(tmp, "context.env"), - "E2E_SCENARIO=test\nE2E_AGENT=openclaw\nE2E_SANDBOX_NAME=sb\nE2E_GATEWAY_URL=http://127.0.0.1\n", - ); - const r = runBash( - ` - set -euo pipefail - fake_sandbox() { - case "$*" in - *cat*) printf 'marker' ;; - *version*) printf 'OpenClaw 2.0.0' ;; - *models*) printf '{"data":[]}' ;; - *) true ;; - esac - } - . "${REBUILD_UPGRADE_LIB}" - rebuild_upgrade_assert_marker_preserved - rebuild_upgrade_assert_agent_version_upgraded - rebuild_upgrade_assert_inference_works - `, - { - E2E_CONTEXT_DIR: tmp, - REBUILD_UPGRADE_SANDBOX_CMD: "fake_sandbox", - E2E_REBUILD_MARKER_EXPECTED: "marker", - E2E_OLD_AGENT_VERSION: "1.0.0", - }, - ); - expect(r.status, r.stderr).toBe(0); - expect(r.stdout).toContain("suite.rebuild.workspace_state_preserved"); - expect(r.stdout).toContain("suite.rebuild.agent_version_upgraded"); - expect(r.stdout).toContain("suite.rebuild.inference_still_works"); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("policy preset check should match endpoint URL when preset name absent", () => { - // The legacy assertion called `nemoclaw policy status` (a command - // that does not exist) and silently failed. The new assertion calls - // `openshell policy get --full ` and matches preset names - // OR their well-known endpoint hostnames. Verify both paths: a - // policy output containing only endpoint URLs (no bare preset name) - // still passes, mirroring the behavior of the live gateway policy - // dump in test/e2e/test-rebuild-openclaw.sh. - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-ru-policy-")); - try { - fs.writeFileSync( - path.join(tmp, "context.env"), - "E2E_SCENARIO=test\nE2E_AGENT=openclaw\nE2E_SANDBOX_NAME=sb\nE2E_GATEWAY_URL=http://127.0.0.1\n", - ); - const r = runBash( - ` - set -euo pipefail - fake_openshell() { - # Emit a minimal policy dump that contains the preset endpoint - # URLs but NOT the bare preset names. This is the realistic - # case: 'openshell policy get --full' renders network rules - # by hostname, not by preset label. - printf 'allow registry.npmjs.org\\nallow pypi.org\\n' - } - . "${REBUILD_UPGRADE_LIB}" - rebuild_upgrade_assert_policy_presets_preserved - `, - { - E2E_CONTEXT_DIR: tmp, - REBUILD_UPGRADE_OPENSHELL_CMD: "fake_openshell", - E2E_EXPECTED_POLICY_PRESETS: "npm pypi", - }, - ); - expect(r.status, r.stderr).toBe(0); - expect(r.stdout).toContain("suite.rebuild.policy_presets_preserved"); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("policy preset check should fail with diagnostic when preset missing", () => { - // Negative case: when a declared preset is absent from the live - // policy dump, the assertion must fail AND emit a diagnostic line - // identifying the missing preset and showing the policy head. The - // original implementation failed silently because the underlying - // `nemoclaw policy status` command did not exist; the new - // implementation must produce actionable evidence. - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-ru-policy-miss-")); - try { - fs.writeFileSync( - path.join(tmp, "context.env"), - "E2E_SCENARIO=test\nE2E_AGENT=openclaw\nE2E_SANDBOX_NAME=sb\nE2E_GATEWAY_URL=http://127.0.0.1\n", - ); - const r = runBash( - ` - fake_openshell() { - # Policy dump missing 'pypi' entirely. - printf 'allow registry.npmjs.org\\n' - } - . "${REBUILD_UPGRADE_LIB}" - rebuild_upgrade_assert_policy_presets_preserved - `, - { - E2E_CONTEXT_DIR: tmp, - REBUILD_UPGRADE_OPENSHELL_CMD: "fake_openshell", - E2E_EXPECTED_POLICY_PRESETS: "npm pypi", - }, - ); - expect(r.status).not.toBe(0); - expect(r.stdout + r.stderr).toMatch(/preset 'pypi' not in policy/); - expect(r.stdout + r.stderr).toMatch(/matchers: pypi/); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); -}); - -describe("Phase 1.A logging helpers", () => { - it("logging should emit stable pass marker when E2E pass called", () => { - const r = runBash(` - set -euo pipefail - . "${RUNTIME_LIB}/logging.sh" - e2e_pass "assertion X" - `); - expect(r.status, r.stderr).toBe(0); - expect(r.stdout).toMatch(/^PASS:.*assertion X/m); - }); - - it("logging should emit stable fail marker and nonzero exit when E2E fail called", () => { - const r = runBash(` - . "${RUNTIME_LIB}/logging.sh" - ( e2e_fail "assertion Y" ) - `); - expect(r.status).not.toBe(0); - expect(r.stdout + r.stderr).toMatch(/FAIL:.*assertion Y/); - }); - - it("logging should include phase prefix when E2E section called", () => { - const r = runBash(` - set -euo pipefail - . "${RUNTIME_LIB}/logging.sh" - e2e_section "Phase 2: onboarding" - `); - expect(r.status, r.stderr).toBe(0); - expect(r.stdout).toMatch(/^=== Phase 2:.*onboarding/m); - }); - - it("logging should autosource logging when env.sh is sourced", () => { - const r = runBash(` - set -euo pipefail - . "${RUNTIME_LIB}/env.sh" - # e2e_pass must be defined after sourcing env.sh alone. - e2e_pass "from env.sh" - `); - expect(r.status, r.stderr).toBe(0); - expect(r.stdout).toMatch(/^PASS:.*from env.sh/m); - }); -}); - -// ───────────────────────────────────────────────────────────────────────────── -// Phase 1.B — Sandbox exec helper (lib/sandbox-exec.sh) -// ───────────────────────────────────────────────────────────────────────────── - -describe("Phase 1.B sandbox-exec helper", () => { - it("sandbox exec should propagate exit code when command fails", () => { - // Use a fake openshell on PATH that executes the command after `--`. - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-sbex-fail-")); - try { - const bin = path.join(tmp, "bin"); - fs.mkdirSync(bin); - fs.writeFileSync( - path.join(bin, "openshell"), - `#!/usr/bin/env bash -set -euo pipefail -while [[ "$#" -gt 0 && "$1" != "--" ]]; do - shift -done -if [[ "$#" -gt 0 ]]; then - shift -fi -exec "$@" -`, - { mode: 0o755 }, - ); - const r = runBash( - ` - . "${VALIDATION_SUITES}/sandbox-exec.sh" - e2e_sandbox_exec sb1 -- false - echo "rc=$?" - `, - // Force the openshell-direct transport so the stubbed openshell - // (which has no `sandbox ssh-config` subcommand) is exercised. - { PATH: `${bin}:${process.env.PATH}`, E2E_SANDBOX_EXEC_VIA_OPENSHELL: "1" }, - ); - expect(r.stdout).toMatch(/rc=1/); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("sandbox exec stdin should quote args safely when input is piped", () => { - // Verify that $TOKEN is NOT expanded on the host side before being - // delivered to the sandbox. We stub openshell to echo back stdin. - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-sbex-stdin-")); - try { - const bin = path.join(tmp, "bin"); - fs.mkdirSync(bin); - // Fake openshell: when called as `openshell sandbox exec --name sb1 -- cat` - // read stdin and print it verbatim so the test can see what the sandbox - // would have received. - fs.writeFileSync(path.join(bin, "openshell"), "#!/usr/bin/env bash\ncat\n", { - mode: 0o755, - }); - const r = runBash( - ` - set -euo pipefail - . "${VALIDATION_SUITES}/sandbox-exec.sh" - printf 'hello $TOKEN' | e2e_sandbox_exec_stdin sb1 -- cat - `, - { - PATH: `${bin}:${process.env.PATH}`, - TOKEN: "SHOULD_NOT_EXPAND", - // Stub only handles the openshell-direct transport. - E2E_SANDBOX_EXEC_VIA_OPENSHELL: "1", - }, - ); - expect(r.status, r.stderr).toBe(0); - expect(r.stdout).toContain("hello $TOKEN"); - expect(r.stdout).not.toContain("SHOULD_NOT_EXPAND"); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("sandbox exec should prefer SSH config transport when OpenShell offers one", () => { - // Verify the new default: when `openshell sandbox ssh-config ` - // succeeds, the wrapper routes through `ssh -F ` instead of - // `openshell sandbox exec`. - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-sbex-ssh-")); - try { - const bin = path.join(tmp, "bin"); - fs.mkdirSync(bin); - const trace = path.join(tmp, "ssh.trace"); - fs.writeFileSync( - path.join(bin, "openshell"), - `#!/usr/bin/env bash -set -euo pipefail -if [[ "$1" == "sandbox" && "$2" == "ssh-config" ]]; then - printf 'Host openshell-%s\\n HostName 127.0.0.1\\n Port 2222\\n User sandbox\\n' "$3" - exit 0 -fi -echo "unexpected openshell call: $*" >&2 -exit 99 -`, - { mode: 0o755 }, - ); - fs.writeFileSync( - path.join(bin, "ssh"), - `#!/usr/bin/env bash -set -euo pipefail -printf '%s\\n' "ssh-args:$*" >> "${trace}" -remote="\${@: -1}" -printf '%s\\n' "remote-cmd:\${remote}" >> "${trace}" -echo ok-from-ssh -exit 0 -`, - { mode: 0o755 }, - ); - const ctxDir = path.join(tmp, "ctx"); - fs.mkdirSync(ctxDir); - const r = runBash( - ` - set -euo pipefail - . "${VALIDATION_SUITES}/sandbox-exec.sh" - e2e_sandbox_exec sb1 -- echo hello - `, - { - PATH: `${bin}:${process.env.PATH}`, - E2E_CONTEXT_DIR: ctxDir, - }, - ); - expect(r.status, r.stderr).toBe(0); - expect(r.stdout).toContain("ok-from-ssh"); - const traceContents = fs.readFileSync(trace, "utf8"); - expect(traceContents).toMatch(/ssh-args:.*-F /); - expect(traceContents).toContain("openshell-sb1"); - expect(traceContents).toMatch(/remote-cmd:echo hello$/m); - const cfg = path.join(ctxDir, ".ssh-config-cache", "sb1.cfg"); - expect(fs.existsSync(cfg)).toBe(true); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("sandbox exec should fall back to OpenShell when SSH config is unavailable", () => { - // If `openshell sandbox ssh-config` fails, the wrapper must fall - // back to `openshell sandbox exec`. - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-sbex-fb-")); - try { - const bin = path.join(tmp, "bin"); - fs.mkdirSync(bin); - fs.writeFileSync( - path.join(bin, "openshell"), - `#!/usr/bin/env bash -set -uo pipefail -if [[ "$1" == "sandbox" && "$2" == "ssh-config" ]]; then - exit 1 -fi -if [[ "$1" == "sandbox" && "$2" == "exec" ]]; then - shift 2 - while [[ "$#" -gt 0 && "$1" != "--" ]]; do shift; done - shift || true - exec "$@" -fi -exit 99 -`, - { mode: 0o755 }, - ); - const ctxDir = path.join(tmp, "ctx"); - fs.mkdirSync(ctxDir); - const r = runBash( - ` - set -euo pipefail - . "${VALIDATION_SUITES}/sandbox-exec.sh" - e2e_sandbox_exec sb1 -- echo fallback-ok - `, - { - PATH: `${bin}:${process.env.PATH}`, - E2E_CONTEXT_DIR: ctxDir, - }, - ); - expect(r.status, r.stderr).toBe(0); - expect(r.stdout).toContain("fallback-ok"); - expect(r.stderr).toMatch(/ssh-config unavailable for sb1/); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); -}); - -// ───────────────────────────────────────────────────────────────────────────── -// Phase 1.C — Fixtures (lib/fixtures/) -// ───────────────────────────────────────────────────────────────────────────── - -describe("Phase 1.C fixtures", () => { - it("fake OpenAI should start and stop cleanly and serve chat completions", () => { - const r = runBash(` - set -euo pipefail - . "${FIXTURES}/fake-openai.sh" - fake_openai_start - : "\${FAKE_OPENAI_PORT:?not exported}" - URL="http://127.0.0.1:\${FAKE_OPENAI_PORT}/v1/chat/completions" - body='{"model":"x","messages":[{"role":"user","content":"hi"}]}' - out=$(curl -fsS -H 'Content-Type: application/json' -d "$body" "$URL") - echo "$out" - fake_openai_stop - `); - expect(r.status, r.stderr).toBe(0); - expect(r.stdout).toMatch(/choices/); - expect(r.stdout).toMatch(/content/); - }); - - it("older base image should emit Dockerfile pointing at tagged base", () => { - const r = runBash(` - set -euo pipefail - . "${FIXTURES}/older-base-image.sh" - df="$(older_base_image_prepare v0.0.1-test)" - echo "DF=$df" - head -n1 "$df" - `); - expect(r.status, r.stderr).toBe(0); - expect(r.stdout).toMatch(/^FROM .*:v0\.0\.1-test/m); - }); - - it("fake messaging fixtures should bind a port and accept stub requests", () => { - for (const provider of ["telegram", "discord", "slack"]) { - const r = runBash(` - set -euo pipefail - . "${FIXTURES}/fake-${provider}.sh" - fake_${provider}_start - : "\${FAKE_${provider.toUpperCase()}_PORT:?port not exported}" - URL="http://127.0.0.1:\${FAKE_${provider.toUpperCase()}_PORT}/ping" - code=$(curl -fsS -o /dev/null -w '%{http_code}' "$URL" || echo failed) - echo "code=$code" - fake_${provider}_stop - `); - expect(r.status, `${provider}: ${r.stderr}`).toBe(0); - expect(r.stdout).toMatch(/code=200/); - } - }); -}); - -// ───────────────────────────────────────────────────────────────────────────── -// Phase 1.D — Assertion helpers (lib/assert/) -// ───────────────────────────────────────────────────────────────────────────── - -describe("Phase 1.D assertion helpers", () => { - it("inference works assertion should pass when the round trip returns ok", () => { - const r = runBash(` - set -euo pipefail - . "${FIXTURES}/fake-openai.sh" - . "${ASSERT}/inference-works.sh" - fake_openai_start - URL="http://127.0.0.1:\${FAKE_OPENAI_PORT}" - e2e_assert_inference_works "$URL" - rc=$? - fake_openai_stop - exit $rc - `); - expect(r.status, r.stderr).toBe(0); - }); - - it("no credentials leaked assertion should fail when a pattern leaks in the bundle", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-creds-")); - try { - const bundle = path.join(tmp, "bundle"); - fs.mkdirSync(bundle); - fs.writeFileSync( - path.join(bundle, "leak.txt"), - "token=sk-abc123DEADBEEFCAFE0000111122223333", - ); - const r = runBash(` - . "${ASSERT}/no-credentials-leaked.sh" - e2e_assert_no_credentials_leaked "${bundle}" - `); - expect(r.status).not.toBe(0); - expect(r.stdout + r.stderr).toMatch(/FAIL:/); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("policy preset applied assertion should pass when active presets match the declared set", () => { - // Stub `nemoclaw policies list` to emit a known set. - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-pol-")); - try { - const bin = path.join(tmp, "bin"); - fs.mkdirSync(bin); - fs.writeFileSync( - path.join(bin, "nemoclaw"), - '#!/usr/bin/env bash\nif [[ "$1" == "policies" && "$2" == "list" ]]; then\n printf "slack\\ndiscord\\n"\nfi\n', - { mode: 0o755 }, - ); - const r = runBash( - ` - set -euo pipefail - . "${ASSERT}/policy-preset-applied.sh" - e2e_assert_policy_preset_applied slack discord - `, - { PATH: `${bin}:${process.env.PATH}` }, - ); - expect(r.status, r.stderr).toBe(0); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("messaging bridge reachable assertion should pass when the provider endpoint is alive", () => { - const r = runBash(` - set -euo pipefail - . "${FIXTURES}/fake-telegram.sh" - . "${ASSERT}/messaging-bridge-reachable.sh" - fake_telegram_start - export MESSAGING_BRIDGE_URL="http://127.0.0.1:\${FAKE_TELEGRAM_PORT}" - e2e_assert_messaging_bridge_reachable telegram - rc=$? - fake_telegram_stop - exit $rc - `); - expect(r.status, r.stderr).toBe(0); - }); -}); - -// ───────────────────────────────────────────────────────────────────────────── -// Issue #3810 Phase 1 — Messaging provider primitive library -// ───────────────────────────────────────────────────────────────────────────── - -describe("Issue #3810 messaging provider helper library", () => { - function withContext(values: Record): string { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-msgctx-")); - fs.writeFileSync( - path.join(tmp, "context.env"), - Object.entries(values) - .map(([key, value]) => `${key}=${value}`) - .join("\n") + "\n", - ); - return tmp; - } - - it("should source messaging provider library in isolation", () => { - const r = runBash(` - set -euo pipefail - . "${VALIDATION_LIB}/messaging_providers.sh" - declare -F e2e_messaging_load_context - `); - expect(r.status, r.stderr).toBe(0); - expect(r.stdout).toContain("e2e_messaging_load_context"); - }); - - it("should fail with a clear diagnostic when context is missing", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-msgmissing-")); - fs.rmSync(tmp, { recursive: true, force: true }); - const r = runBash( - ` - set -euo pipefail - . "${VALIDATION_LIB}/messaging_providers.sh" - e2e_messaging_load_context - `, - { E2E_CONTEXT_DIR: tmp }, - ); - expect(r.status).not.toBe(0); - expect(r.stderr).toMatch(/E2E_CONTEXT_DIR|context\.env/); - }); - - it("should derive provider names for messaging channels", () => { - const cases: Array<[string, Record, string]> = [ - ["telegram", { E2E_AGENT: "openclaw", E2E_MESSAGING_PROVIDER: "telegram" }, "telegram"], - ["discord", { E2E_AGENT: "openclaw", E2E_MESSAGING_PROVIDER: "discord" }, "discord"], - [ - "slack-bot", - { E2E_AGENT: "openclaw", E2E_MESSAGING_PROVIDER: "slack", E2E_MESSAGING_CHANNEL: "bot" }, - "slack-bot", - ], - [ - "slack-app", - { E2E_AGENT: "openclaw", E2E_MESSAGING_PROVIDER: "slack", E2E_MESSAGING_CHANNEL: "app" }, - "slack-app", - ], - ["whatsapp", { E2E_AGENT: "openclaw", E2E_MESSAGING_PROVIDER: "whatsapp" }, "whatsapp-qr"], - ]; - for (const [name, values, expected] of cases) { - const ctx = withContext({ E2E_SANDBOX_NAME: "sb", ...values }); - try { - const r = runBash( - ` - set -euo pipefail - . "${VALIDATION_LIB}/messaging_providers.sh" - e2e_messaging_load_context >/dev/null - e2e_messaging_provider_name - `, - { E2E_CONTEXT_DIR: ctx }, - ); - expect(r.status, `${name}: ${r.stderr}`).toBe(0); - expect(r.stdout.trim()).toBe(expected); - } finally { - fs.rmSync(ctx, { recursive: true, force: true }); - } - } - }); - - it("should resolve agent config paths", () => { - const cases: Array<[string, string]> = [ - ["openclaw", "/sandbox/.openclaw/openclaw.json"], - ["hermes", "/sandbox/.hermes/.env"], - ]; - for (const [agent, expected] of cases) { - const ctx = withContext({ - E2E_SANDBOX_NAME: "sb", - E2E_AGENT: agent, - E2E_MESSAGING_PROVIDER: "discord", - }); - try { - const r = runBash( - ` - set -euo pipefail - . "${VALIDATION_LIB}/messaging_providers.sh" - e2e_messaging_load_context >/dev/null - e2e_messaging_agent_config_path - `, - { E2E_CONTEXT_DIR: ctx }, - ); - expect(r.status, r.stderr).toBe(0); - expect(r.stdout.trim()).toBe(expected); - } finally { - fs.rmSync(ctx, { recursive: true, force: true }); - } - } - }); - - it("should expose placeholder and secret leak interfaces without live secrets", () => { - const r = runBash(` - set -euo pipefail - . "${VALIDATION_LIB}/messaging_providers.sh" - e2e_messaging_assert_placeholder_configured 'token=\${TELEGRAM_BOT_TOKEN}' 'TELEGRAM_BOT_TOKEN' - e2e_messaging_assert_no_secret_leak 'safe placeholder \${TELEGRAM_BOT_TOKEN}' 'raw-secret-123' - if e2e_messaging_assert_no_secret_leak 'oops raw-secret-123' 'raw-secret-123'; then - echo unexpected-pass - exit 1 - fi - `); - expect(r.status, r.stderr).toBe(0); - expect(r.stdout).not.toContain("unexpected-pass"); - }); -}); - -describe("baseline onboarding validation helper", () => { - it("baseline helper should source under strict shell options", () => { - const r = runBash( - `set -euo pipefail; source "${VALIDATION_SUITES}/lib/baseline_onboarding.sh"`, - ); - expect(r.status, r.stderr).toBe(0); - }); - - it("baseline CLI assertions should use mocked binaries", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "baseline-cli-")); - try { - const bin = path.join(tmp, "bin"); - const ctx = path.join(tmp, "ctx"); - fs.mkdirSync(bin); - fs.mkdirSync(ctx); - fs.writeFileSync( - path.join(ctx, "context.env"), - "E2E_SANDBOX_NAME=sb1\nE2E_PROVIDER=nvidia\nE2E_INFERENCE_ROUTE=inference-local\n", - ); - fs.writeFileSync( - path.join(bin, "nemoclaw"), - `#!/usr/bin/env bash -case "$*" in - --help) echo help;; - "sb1 status") echo 'status running gateway healthy sandbox running';; - "sb1 logs") echo baseline-log;; - *) echo "unexpected nemoclaw args: $*" >&2; exit 64;; -esac -`, - { mode: 0o755 }, - ); - fs.writeFileSync(path.join(bin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { - mode: 0o755, - }); - const r = runBash( - ` - set -euo pipefail - source "${VALIDATION_SUITES}/lib/baseline_onboarding.sh" - baseline_onboarding_load_context - baseline_assert_nemoclaw_on_path - baseline_assert_openshell_on_path - baseline_assert_nemoclaw_help_exits_zero - baseline_assert_sandbox_status_exits_zero - baseline_assert_logs_produce_output - `, - { E2E_CONTEXT_DIR: ctx, PATH: `${bin}:${process.env.PATH}` }, - ); - expect(r.status, r.stderr).toBe(0); - expect(r.stdout).toContain("PASS: validation.baseline_onboarding.nemoclaw_on_path"); - expect(r.stdout).toContain("PASS: validation.baseline_onboarding.openshell_on_path"); - expect(r.stdout).toContain("PASS: validation.baseline_onboarding.nemoclaw_help_exits_zero"); - expect(r.stdout).toContain("PASS: validation.baseline_onboarding.sandbox_status"); - expect(r.stdout).toContain("PASS: validation.baseline_onboarding.logs_available"); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); -}); - -describe("sandbox lifecycle validation helper", () => { - it("should load context from E2E context dir", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-life-")); - try { - fs.writeFileSync( - path.join(tmp, "context.env"), - "E2E_SANDBOX_NAME=sb1\nE2E_GATEWAY_URL=http://127.0.0.1:1\n", - ); - const r = runBash( - `set -euo pipefail; . "${VALIDATION_SUITES}/lib/sandbox_lifecycle.sh"; sandbox_lifecycle_load_context; echo "$E2E_SANDBOX_NAME $E2E_GATEWAY_URL"`, - { E2E_CONTEXT_DIR: tmp }, - ); - expect(r.status, r.stderr).toBe(0); - expect(r.stdout).toContain("sb1 http://127.0.0.1:1"); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("should emit stable pass and fail IDs", () => { - const r = runBash( - `. "${VALIDATION_SUITES}/lib/sandbox_lifecycle.sh"; sandbox_lifecycle_pass validation.sandbox_lifecycle.gateway_health ok; sandbox_lifecycle_fail validation.sandbox_operations.logs_available nope`, - ); - expect(r.status).not.toBe(0); - expect(r.stdout).toMatch(/PASS: validation\.sandbox_lifecycle\.gateway_health/); - expect(r.stderr).toMatch(/FAIL: validation\.sandbox_operations\.logs_available/); - }); - - it("should apply timeout to command execution", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-life-timeout-")); - try { - const bin = path.join(tmp, "bin"); - fs.mkdirSync(bin); - fs.writeFileSync( - path.join(bin, "timeout"), - "#!/usr/bin/env bash\necho timed out >&2\nexit 124\n", - { mode: 0o755 }, - ); - const r = runBash( - `set -e; . "${VALIDATION_SUITES}/lib/sandbox_lifecycle.sh"; sandbox_lifecycle_run_with_timeout 1 bash -c 'sleep 5'`, - { PATH: `${bin}:${process.env.PATH}` }, - ); - expect(r.status).toBe(124); - expect(r.stderr).toMatch(/timed out/); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("should validate list status logs exec with mocked commands", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-life-mock-")); - try { - const bin = path.join(tmp, "bin"); - fs.mkdirSync(bin); - fs.writeFileSync( - path.join(bin, "nemoclaw"), - `#!/usr/bin/env bash -case "$*" in - list) echo sb1;; - "sb1 status") printf ' Sandbox: sb1\\n Model: nvidia/x\\n OpenShell: 0.0.44\\n Policies: npm\\n';; - "sb1 logs") echo logline;; - *) echo "unexpected nemoclaw args: $*" >&2; exit 64;; -esac -`, - { mode: 0o755 }, - ); - fs.writeFileSync( - path.join(bin, "openshell"), - `#!/usr/bin/env bash -echo lifecycle-ok -`, - { mode: 0o755 }, - ); - fs.writeFileSync( - path.join(tmp, "context.env"), - "E2E_SANDBOX_NAME=sb1\nE2E_GATEWAY_URL=http://127.0.0.1:1\n", - ); - // Force the wrapper's openshell-exec fallback transport: this - // stub openshell ignores its argv and always echoes 'lifecycle-ok', - // which would corrupt an ssh-config materialization. The opt-out - // env var keeps the test exercising openshell-exec directly while - // production callers still pick up ssh-config-preferred routing. - const r = runBash( - `set -euo pipefail; . "${VALIDATION_SUITES}/lib/sandbox_lifecycle.sh"; sandbox_lifecycle_load_context; sandbox_lifecycle_assert_nemoclaw_list_contains_sandbox; sandbox_lifecycle_assert_status_fields_present; sandbox_lifecycle_assert_logs_available; sandbox_lifecycle_assert_openshell_exec_ok`, - { - E2E_CONTEXT_DIR: tmp, - PATH: `${bin}:${process.env.PATH}`, - E2E_SANDBOX_EXEC_VIA_OPENSHELL: "1", - }, - ); - expect(r.status, r.stderr).toBe(0); - expect(r.stdout).toMatch(/validation\.sandbox_operations\.sandbox_listed/); - expect(r.stdout).toMatch(/validation\.sandbox_operations\.openshell_exec_ok/); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); -}); diff --git a/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts b/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts index 7158039d2c8..5f8968e33d1 100644 --- a/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest"; +import { buildLiveScenarioRunPlan } from "../live/run-plan.ts"; import { listScenarios } from "../scenarios/registry.ts"; import { liveScenarioSupport } from "../scenarios/runtime-support.ts"; @@ -29,6 +30,33 @@ describe("live Vitest registry discovery support", () => { ]); }); + it("builds the live run-plan artifact shape from registry metadata", () => { + const scenario = listScenarios().find((entry) => entry.id === "ubuntu-repo-cloud-openclaw"); + + expect(scenario).toBeTruthy(); + expect(buildLiveScenarioRunPlan(scenario!)).toEqual({ + scenarioId: "ubuntu-repo-cloud-openclaw", + manifestPath: "test/e2e-scenario/manifests/openclaw-nvidia.yaml", + expectedStateId: "cloud-openclaw-ready", + suiteIds: ["smoke", "inference", "credentials"], + phases: ["environment", "onboarding", "state-validation"], + }); + }); + + it("includes the lifecycle phase in live run-plan artifacts when a scenario mutates state", () => { + const scenario = listScenarios().find( + (entry) => entry.id === "ubuntu-repo-docker-post-reboot-recovery", + ); + + expect(scenario).toBeTruthy(); + expect(buildLiveScenarioRunPlan(scenario!).phases).toEqual([ + "environment", + "onboarding", + "lifecycle", + "state-validation", + ]); + }); + it("keeps unsupported onboarding profiles skipped with a concrete reason", () => { const scenario = listScenarios().find((entry) => entry.id === "ubuntu-repo-cloud-hermes"); diff --git a/test/e2e-scenario/framework-tests/e2e-manifests.test.ts b/test/e2e-scenario/framework-tests/e2e-manifests.test.ts index 65e562458ca..3b51eccd947 100644 --- a/test/e2e-scenario/framework-tests/e2e-manifests.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-manifests.test.ts @@ -4,7 +4,6 @@ import { describe, expect, it } from "vitest"; import path from "node:path"; -import { compileRunPlans } from "../scenarios/compiler.ts"; import { loadManifest, loadManifestsFromDir, validateManifest } from "../scenarios/manifests.ts"; import { listScenarios } from "../scenarios/registry.ts"; @@ -70,16 +69,14 @@ describe("NemoClawInstance manifests", () => { expect(missingManifests, `missing manifest files: ${missingManifests.join(", ")}`).toEqual([]); }); - it("plan only output should show resolved manifest setup and onboarding choices", () => { - const [plan] = compileRunPlans(["ubuntu-repo-cloud-openclaw"]); + it("registry scenario manifest paths resolve setup and onboarding choices", () => { + const scenario = listScenarios().find((entry) => entry.id === "ubuntu-repo-cloud-openclaw"); - expect(plan.manifestPath).toBe("test/e2e-scenario/manifests/openclaw-nvidia.yaml"); - expect(plan.manifestPath).toBeDefined(); - expect(plan.manifest).toEqual( - loadManifest(path.join(REPO_ROOT, plan.manifestPath as string)).document, - ); - expect(plan.manifest?.spec.setup.install.source).toBe("repo-current"); - expect(plan.manifest?.spec.onboarding.agent).toBe("openclaw"); - expect(plan.manifest?.spec.onboarding.provider).toBe("nvidia"); + expect(scenario).toBeTruthy(); + expect(scenario!.manifestPath).toBe("test/e2e-scenario/manifests/openclaw-nvidia.yaml"); + const manifest = loadManifest(path.join(REPO_ROOT, scenario!.manifestPath as string)).document; + expect(manifest.spec.setup.install.source).toBe("repo-current"); + expect(manifest.spec.onboarding.agent).toBe("openclaw"); + expect(manifest.spec.onboarding.provider).toBe("nvidia"); }); }); diff --git a/test/e2e-scenario/framework-tests/e2e-migration-inventory.test.ts b/test/e2e-scenario/framework-tests/e2e-migration-inventory.test.ts index 14998a690a5..b4b9e0b9cc8 100644 --- a/test/e2e-scenario/framework-tests/e2e-migration-inventory.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-migration-inventory.test.ts @@ -99,11 +99,13 @@ function isCoveredByInventoryPath(filePath: string, inventoryPath: string): bool return filePath === inventoryPath || filePath.startsWith(`${inventoryPath}/`); } -function expectPathListIsRepoRelative(paths: readonly string[]) { +function expectPathListIsRepoRelative(paths: readonly string[], options = { mustExist: true }) { expect(paths.length).toBeGreaterThan(0); for (const repoRelativePath of paths) { expect(repoRelativePath).not.toBe(""); - expect(repoPathExists(repoRelativePath)).toBe(true); + if (options.mustExist) { + expect(repoPathExists(repoRelativePath)).toBe(true); + } } } @@ -139,7 +141,7 @@ function expectMigrationRecordDeletionGate( if (record.deletionReady) { expect(["covered", "retired"]).toContain(record.status); - expect(record.deletionApprovalIssue).toBe("#4357"); + expect(["#4357", "#5098"]).toContain(record.deletionApprovalIssue); expect( record.status === "retired" ? record.retiredReason : record.targetVitestScenarios.length, ).toBeTruthy(); @@ -175,9 +177,9 @@ describe("E2E migration inventory deletion gates", () => { expect(surface.id).toMatch(/^[a-z0-9-]+$/); expect(internalSurfaceIds.has(surface.id)).toBe(false); internalSurfaceIds.add(surface.id); - expectPathListIsRepoRelative(surface.paths); + expectPathListIsRepoRelative(surface.paths, { mustExist: surface.status !== "retired" }); expect(surface.domain).not.toBe(""); - expect(surface.ownerIssue).toMatch(/^#(?:3588|434[7-9]|435[0-7]|4941)$/); + expect(surface.ownerIssue).toMatch(/^#(?:3588|434[7-9]|435[0-7]|4941|5098)$/); expect(surface.replacementSurface).not.toBe(""); expect(surface.notes).not.toBe(""); } @@ -198,8 +200,16 @@ describe("E2E migration inventory deletion gates", () => { const surfacePaths = inventory.internalSurfaces.flatMap((surface) => surface.paths); for (const root of INTERNAL_SURFACE_ROOTS) { + if (!repoPathExists(root)) { + const retiredSurface = inventory.internalSurfaces.find((surface) => + surface.paths.some((surfacePath) => isCoveredByInventoryPath(root, surfacePath)), + ); + expect(retiredSurface?.status).toBe("retired"); + expect(retiredSurface?.retiredReason).not.toBe(""); + continue; + } + const files = listRepoFilesUnder(root); - expect(files.length).toBeGreaterThan(0); for (const file of files) { expect( surfacePaths.some((surfacePath) => isCoveredByInventoryPath(file, surfacePath)), diff --git a/test/e2e-scenario/framework-tests/e2e-negative-matcher.test.ts b/test/e2e-scenario/framework-tests/e2e-negative-matcher.test.ts deleted file mode 100644 index a06447d66f4..00000000000 --- a/test/e2e-scenario/framework-tests/e2e-negative-matcher.test.ts +++ /dev/null @@ -1,547 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { describe, expect, it } from "vitest"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { compileRunPlans } from "../scenarios/compiler.ts"; -import { - evaluateNegativeContract, - negativeContractPhaseResult, -} from "../scenarios/orchestrators/negative-matcher.ts"; -import { ScenarioRunner } from "../scenarios/orchestrators/runner.ts"; -import { listScenarios } from "../scenarios/registry.ts"; -import { planFailed } from "../scenarios/run.ts"; -import type { - ExpectedFailureContract, - PhaseName, - PhaseResult, - RunContext, - RunPlan, - RunPlanPhase, -} from "../scenarios/types.ts"; - -function freshCtx(): RunContext { - return { contextDir: fs.mkdtempSync(path.join(os.tmpdir(), "e2e-neg-")) }; -} - -function planWithExpectedFailure(contract: ExpectedFailureContract): RunPlan { - return { - scenarioId: "synthetic-negative", - status: "compiled", - suiteIds: [], - onboardingAssertionIds: [], - phases: [ - { name: "environment", actions: [], assertionGroups: [] }, - { name: "onboarding", actions: [], assertionGroups: [] }, - { name: "runtime", actions: [], assertionGroups: [] }, - ], - runnerRequirements: [], - requiredSecrets: [], - skippedCapabilities: [], - expectedFailure: contract, - sutBoundaries: [{ id: "host-cli", client: "HostCliClient" }], - }; -} - -function phaseResult( - phase: PhaseName, - opts: { - status?: PhaseResult["status"]; - failedActionId?: string; - failedActionMessage?: string; - failedAssertionId?: string; - failedAssertionMessage?: string; - } = {}, -): PhaseResult { - return { - phase, - status: opts.status ?? "passed", - actions: opts.failedActionId - ? [ - { - id: opts.failedActionId, - status: "failed", - durationMs: 1, - message: opts.failedActionMessage, - }, - ] - : [], - assertions: opts.failedAssertionId - ? [ - { - id: opts.failedAssertionId, - status: "failed", - attempts: 1, - durationMs: 1, - message: opts.failedAssertionMessage, - }, - ] - : [], - }; -} - -function passedNegativeContractPhase(): PhaseResult { - return { - phase: "negative-contract", - status: "passed", - actions: [], - assertions: [ - { - id: "negative-contract.match", - status: "passed", - attempts: 1, - durationMs: 0, - message: "matched", - }, - ], - }; -} - -function stateValidationResult( - status: PhaseResult["status"], - actionIds: string[] = ["state-validation.gateway-absent", "state-validation.sandbox-absent"], -): PhaseResult { - return { - phase: "state-validation", - status, - actions: actionIds.map((id) => ({ id, status: "passed", durationMs: 1 })), - assertions: [], - }; -} - -describe("evaluateNegativeContract - phase + errorClass matching", () => { - it("matches when expected phase fails with the declared errorClass", () => { - const plan = planWithExpectedFailure({ - phase: "onboarding", - errorClass: "invalid-nvidia-api-key", - forbiddenSideEffects: ["gateway-started"], - }); - const results: PhaseResult[] = [ - phaseResult("environment", { status: "passed" }), - phaseResult("onboarding", { - status: "failed", - failedActionId: "onboarding.profile.cloud-openclaw-invalid-nvidia-key", - failedActionMessage: "phase action onboarding exit 1: invalid-nvidia-api-key auth failed", - }), - ]; - const result = evaluateNegativeContract(plan, results); - expect(result.matched).toBe(true); - expect(result.outcome).toBe("matched"); - expect(result.observed.failedPhase).toBe("onboarding"); - }); - - it("resolves preflight expected phase to onboarding orchestrator", () => { - const plan = planWithExpectedFailure({ - phase: "preflight", - errorClass: "docker-missing", - }); - const results: PhaseResult[] = [ - phaseResult("environment", { status: "passed" }), - phaseResult("onboarding", { - status: "failed", - failedActionId: "onboarding.profile.cloud-openclaw", - failedActionMessage: "preflight detected docker-missing on the runner host", - }), - ]; - const result = evaluateNegativeContract(plan, results); - expect(result.matched).toBe(true); - expect(result.outcome).toBe("matched"); - }); - - it("fails when no failure was observed at all", () => { - const plan = planWithExpectedFailure({ phase: "onboarding", errorClass: "docker-missing" }); - const results: PhaseResult[] = [ - phaseResult("environment", { status: "passed" }), - phaseResult("onboarding", { status: "passed" }), - phaseResult("runtime", { status: "passed" }), - ]; - const result = evaluateNegativeContract(plan, results); - expect(result.matched).toBe(false); - expect(result.outcome).toBe("no-failure-observed"); - expect(result.message).toMatch(/all phases passed/); - }); - - it("matches when a passed expected-failure assertion handled the failure", () => { - const plan = planWithExpectedFailure({ - phase: "preflight", - errorClass: "docker-missing", - forbiddenSideEffects: ["gateway-started", "sandbox-created"], - }); - const results: PhaseResult[] = [ - phaseResult("environment", { status: "passed" }), - { - phase: "onboarding", - status: "passed", - actions: [ - { - id: "onboarding.profile.cloud-openclaw-no-docker", - status: "passed", - durationMs: 1, - }, - ], - assertions: [ - { - id: "onboarding.preflight.expected-failed", - status: "passed", - attempts: 1, - durationMs: 1, - }, - ], - }, - phaseResult("state-validation", { status: "passed" }), - ]; - - const result = evaluateNegativeContract(plan, results); - expect(result.matched).toBe(true); - expect(result.outcome).toBe("matched"); - expect(result.observed).toMatchObject({ - failedPhase: "onboarding", - handledAssertionId: "onboarding.preflight.expected-failed", - }); - }); - - it("matches handled expected-failure actions using scenario error-class aliases", () => { - const plan = planWithExpectedFailure({ - phase: "onboarding", - errorClass: "invalid-nvidia-api-key", - }); - const results: PhaseResult[] = [ - { - phase: "onboarding", - status: "passed", - actions: [ - { - id: "onboarding.profile.cloud-openclaw-invalid-nvidia-key", - status: "passed", - durationMs: 1, - }, - ], - assertions: [], - }, - ]; - - const result = evaluateNegativeContract(plan, results); - expect(result.matched).toBe(true); - expect(result.observed.handledActionId).toBe( - "onboarding.profile.cloud-openclaw-invalid-nvidia-key", - ); - }); - - it("fails when the wrong phase failed", () => { - const plan = planWithExpectedFailure({ phase: "onboarding", errorClass: "docker-missing" }); - const results: PhaseResult[] = [ - phaseResult("environment", { - status: "failed", - failedActionId: "environment.install.ubuntu-repo-no-docker", - failedActionMessage: "install dispatcher exit 1: docker-missing", - }), - ]; - const result = evaluateNegativeContract(plan, results); - expect(result.matched).toBe(false); - expect(result.outcome).toBe("wrong-phase"); - expect(result.message).toMatch(/expected onboarding failure/); - expect(result.observed.failedPhase).toBe("environment"); - }); - - it("fails when the right phase failed for the wrong errorClass", () => { - const plan = planWithExpectedFailure({ - phase: "onboarding", - errorClass: "gateway-port-conflict", - }); - const results: PhaseResult[] = [ - phaseResult("onboarding", { - status: "failed", - failedActionId: "onboarding.profile.cloud-openclaw-gateway-port-conflict", - failedActionMessage: "onboard exit 1: invalid-nvidia-api-key authentication failed", - }), - ]; - const result = evaluateNegativeContract(plan, results); - expect(result.matched).toBe(false); - expect(result.outcome).toBe("wrong-error-class"); - expect(result.message).toMatch(/errorClass mismatch/); - }); - - it("ignores the runtime side-effect probe step when scanning for observed failure", () => { - const plan = planWithExpectedFailure({ phase: "onboarding", errorClass: "docker-missing" }); - const results: PhaseResult[] = [ - phaseResult("environment", { status: "passed" }), - phaseResult("onboarding", { - status: "failed", - failedActionId: "onboarding.profile.cloud-openclaw", - failedActionMessage: "onboard exit 1: docker-missing daemon unreachable", - }), - // runtime phase has only the required pending side-effect step - // that fails closed until the probe lands. The matcher must NOT - // treat that as the observed failure mode. - { - phase: "runtime", - status: "failed", - actions: [], - assertions: [ - { - id: "runtime.expected-failure.no-side-effects", - status: "failed", - attempts: 1, - durationMs: 0, - message: "required pending step not implemented: expectedFailureNoSideEffectsProbe", - }, - ], - }, - ]; - const result = evaluateNegativeContract(plan, results); - expect(result.matched).toBe(true); - expect(result.observed.failedActionId).toBe("onboarding.profile.cloud-openclaw"); - }); - - it("matches errorClass case-insensitively and across separator variants", () => { - const plan = planWithExpectedFailure({ phase: "onboarding", errorClass: "docker-missing" }); - const results: PhaseResult[] = [ - phaseResult("onboarding", { - status: "failed", - failedActionId: "onboarding", - failedActionMessage: "Onboard exit 1: Docker_Missing daemon socket unreachable", - }), - ]; - expect(evaluateNegativeContract(plan, results).matched).toBe(true); - }); - - it("throws if invoked for a plan without expectedFailure", () => { - const plan: RunPlan = { - ...planWithExpectedFailure({ phase: "onboarding", errorClass: "x" }), - expectedFailure: undefined, - }; - expect(() => evaluateNegativeContract(plan, [])).toThrow(/no expectedFailure declared/); - }); - - it("synthetic phase result reflects matched status", () => { - const plan = planWithExpectedFailure({ phase: "onboarding", errorClass: "docker-missing" }); - const results: PhaseResult[] = [ - phaseResult("onboarding", { - status: "failed", - failedActionId: "onboarding", - failedActionMessage: "docker-missing", - }), - ]; - const synthetic = negativeContractPhaseResult(evaluateNegativeContract(plan, results)); - expect(synthetic.phase).toBe("negative-contract"); - expect(synthetic.status).toBe("passed"); - expect(synthetic.assertions[0]).toEqual( - expect.objectContaining({ id: "negative-contract.match", status: "passed" }), - ); - }); -}); - -describe("negative plan exit-code contract", () => { - const plan = planWithExpectedFailure({ - phase: "preflight", - errorClass: "docker-missing", - forbiddenSideEffects: ["gateway-started", "sandbox-created"], - }); - - it("passes when negative contract and forbidden-side-effect probes pass", () => { - expect(planFailed(plan, [passedNegativeContractPhase(), stateValidationResult("passed")])).toBe( - false, - ); - }); - - it("fails when state-validation is missing", () => { - expect(planFailed(plan, [passedNegativeContractPhase()])).toBe(true); - }); - - it("fails when state-validation is skipped", () => { - expect( - planFailed(plan, [passedNegativeContractPhase(), stateValidationResult("skipped")]), - ).toBe(true); - }); - - it("fails when a declared forbidden-side-effect probe did not run", () => { - expect( - planFailed(plan, [ - passedNegativeContractPhase(), - stateValidationResult("passed", ["state-validation.gateway-absent"]), - ]), - ).toBe(true); - }); -}); - -describe("ScenarioRunner appends negative-contract phase", () => { - it("invokes matcher and appends a passing synthetic phase when contract matched", async () => { - const ctx = freshCtx(); - try { - const fakePhase = (phase: PhaseName, outcome: PhaseResult) => ({ - run: async ( - _ctx: RunContext, - _runPhase: RunPlanPhase, - _prior?: PhaseResult[], - ): Promise => outcome, - }); - - const runner = new ScenarioRunner({ - environment: fakePhase("environment", { - phase: "environment", - status: "passed", - actions: [], - assertions: [], - }), - onboarding: fakePhase("onboarding", { - phase: "onboarding", - status: "failed", - actions: [ - { - id: "onboarding.profile.cloud-openclaw", - status: "failed", - durationMs: 1, - message: "onboard exit 1: docker-missing daemon unreachable", - }, - ], - assertions: [], - }), - runtime: fakePhase("runtime", { - phase: "runtime", - status: "passed", - actions: [], - assertions: [], - }), - }); - - const plan = planWithExpectedFailure({ phase: "preflight", errorClass: "docker-missing" }); - const results = await runner.run(ctx, plan); - - const contractPhase = results[results.length - 1]; - expect(contractPhase.phase).toBe("negative-contract"); - expect(contractPhase.status).toBe("passed"); - - // Artifact emitted to ctx.contextDir/.e2e/negative-contract.json - const artifact = path.join(ctx.contextDir, ".e2e", "negative-contract.json"); - expect(fs.existsSync(artifact)).toBe(true); - const parsed = JSON.parse(fs.readFileSync(artifact, "utf8")); - expect(parsed.matched).toBe(true); - expect(parsed.outcome).toBe("matched"); - } finally { - fs.rmSync(ctx.contextDir, { recursive: true, force: true }); - } - }); - - it("emits a failed synthetic phase when the wrong phase failed", async () => { - const ctx = freshCtx(); - try { - const fakePhase = (outcome: PhaseResult) => ({ - run: async (): Promise => outcome, - }); - - const runner = new ScenarioRunner({ - environment: fakePhase({ - phase: "environment", - status: "failed", - actions: [ - { - id: "environment.install.ubuntu-repo-no-docker", - status: "failed", - durationMs: 1, - message: "install dispatcher exit 1: dns-resolution-error", - }, - ], - assertions: [], - }), - onboarding: fakePhase({ - phase: "onboarding", - status: "skipped", - actions: [], - assertions: [], - }), - runtime: fakePhase({ phase: "runtime", status: "skipped", actions: [], assertions: [] }), - }); - - const plan = planWithExpectedFailure({ phase: "onboarding", errorClass: "docker-missing" }); - const results = await runner.run(ctx, plan); - - const contractPhase = results[results.length - 1]; - expect(contractPhase.phase).toBe("negative-contract"); - expect(contractPhase.status).toBe("failed"); - expect(contractPhase.assertions[0].message).toMatch(/expected onboarding failure/); - } finally { - fs.rmSync(ctx.contextDir, { recursive: true, force: true }); - } - }); - - it("does NOT append negative-contract phase for positive scenarios", async () => { - const ctx = freshCtx(); - try { - const [plan] = compileRunPlans(["ubuntu-repo-cloud-openclaw"]); - expect(plan.expectedFailure).toBeUndefined(); - - const fakePhase = (phase: PhaseName) => ({ - run: async (): Promise => ({ - phase, - status: "passed", - actions: [], - assertions: [], - }), - }); - const runner = new ScenarioRunner({ - environment: fakePhase("environment"), - onboarding: fakePhase("onboarding"), - stateValidation: fakePhase("state-validation"), - lifecycle: fakePhase("lifecycle"), - runtime: fakePhase("runtime"), - }); - - const results = await runner.run(ctx, plan); - expect(results.map((r) => r.phase)).toEqual([ - "environment", - "onboarding", - "state-validation", - "lifecycle", - "runtime", - ]); - } finally { - fs.rmSync(ctx.contextDir, { recursive: true, force: true }); - } - }); -}); - -describe("registry contract: negative scenarios use typed state-validation side-effect probes", () => { - it("scenario.expectedFailure does not inject the legacy runtime no-side-effects pending step", () => { - const negatives = listScenarios().filter((scenario) => scenario.expectedFailure); - expect(negatives.length).toBeGreaterThan(0); - for (const scenario of negatives) { - const hasLegacyPendingStep = scenario.assertionGroups.some((group) => - group.steps.some((step) => step.id === "runtime.expected-failure.no-side-effects"), - ); - expect( - hasLegacyPendingStep, - `scenario ${scenario.id} must rely on state-validation, not the legacy pending step`, - ).toBe(false); - } - }); -}); - -describe("compiler validates the typed expected-failure contract", () => { - it("rejects an invalid phase value", () => { - expect(() => - compileRunPlans([ - { - id: "synthetic-bad-phase", - assertionGroups: [], - // Force the bad shape the compiler must reject. - expectedFailure: { phase: "bogus" as never, errorClass: "x" }, - }, - ]), - ).toThrow(/expectedFailure\.phase invalid/); - }); - - it("rejects an empty errorClass", () => { - expect(() => - compileRunPlans([ - { - id: "synthetic-empty-class", - assertionGroups: [], - expectedFailure: { phase: "onboarding", errorClass: "" }, - }, - ]), - ).toThrow(/errorClass must be a non-empty string/); - }); -}); diff --git a/test/e2e-scenario/framework-tests/e2e-phase-environment.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-environment.test.ts index a3e2e3c965a..59e10497355 100644 --- a/test/e2e-scenario/framework-tests/e2e-phase-environment.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-phase-environment.test.ts @@ -1,8 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + import { describe, expect, expectTypeOf, it } from "vitest"; +import { ArtifactSink } from "../framework/artifacts.ts"; import { HostCliClient, type CommandRunner } from "../framework/clients/index.ts"; import type { E2EScenarioFixtures } from "../framework/e2e-test.ts"; import { EnvironmentPhaseFixture, type DockerRuntimeReady } from "../framework/phases/index.ts"; @@ -35,6 +40,10 @@ function shellResult(exitCode: number, output = ""): ShellProbeResult { }; } +function readJson(filePath: string): unknown { + return JSON.parse(fs.readFileSync(filePath, "utf8")); +} + class FakeRunner implements CommandRunner { readonly calls: RunnerCall[] = []; private readonly responses: Array = []; @@ -309,6 +318,59 @@ describe("environment phase fixture", () => { ).rejects.toThrow(/Unsupported scenario runtime 'podman-running'/); }); + it("writes an environment phase result artifact on success", async () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-environment-artifacts-")); + try { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(0, "Docker is available\n")); + const artifacts = new ArtifactSink(tmp); + const environment = new EnvironmentPhaseFixture(new HostCliClient(runner), artifacts); + + await environment.assertReady(cloudOpenClawEnvironment); + + expect(readJson(path.join(tmp, "environment.result.json"))).toMatchObject({ + phase: "environment", + status: "passed", + environment: { + platform: "ubuntu-local", + install: "repo-current", + runtime: "docker-running", + onboarding: "cloud-openclaw", + cliPath: "nemoclaw", + }, + }); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("writes an environment phase result artifact on failure", async () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-environment-artifacts-")); + try { + const artifacts = new ArtifactSink(tmp); + const environment = new EnvironmentPhaseFixture( + new HostCliClient(new FakeRunner()), + artifacts, + ); + + await expect( + environment.assertReady({ ...cloudOpenClawEnvironment, install: "tarball" }), + ).rejects.toThrow(/Unsupported scenario install 'tarball'/); + + expect(readJson(path.join(tmp, "environment.result.json"))).toMatchObject({ + phase: "environment", + status: "failed", + environment: { + install: "tarball", + }, + error: "Unsupported scenario install 'tarball'.", + }); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + it("exposes the environment phase on the Vitest scenario context", () => { expectTypeOf().toEqualTypeOf(); }); diff --git a/test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts index 51ca4d7f0c4..7add981d6c5 100644 --- a/test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts @@ -6,6 +6,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { ArtifactSink } from "../framework/artifacts.ts"; import { HostCliClient, type CommandRunner } from "../framework/clients/index.ts"; import type { E2EScenarioFixtures } from "../framework/e2e-test.ts"; import { OnboardingPhaseFixture, type OnboardingSecrets } from "../framework/phases/index.ts"; @@ -43,6 +44,10 @@ function shellResult(exitCode: number, output = ""): ShellProbeResult { }; } +function readJson(filePath: string): unknown { + return JSON.parse(fs.readFileSync(filePath, "utf8")); +} + class FakeRunner implements CommandRunner { readonly calls: RunnerCall[] = []; private readonly responses: ShellProbeResult[] = []; @@ -477,6 +482,63 @@ describe("onboarding phase fixture", () => { ); }); + it("writes an onboarding phase result artifact on success", async () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-onboarding-artifacts-")); + try { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "onboarded\n")); + const onboard = new OnboardingPhaseFixture( + new HostCliClient(runner), + new FakeSecrets({ NVIDIA_API_KEY: "secret-token" }), + undefined, + new ArtifactSink(tmp), + ); + + await onboard.from(ready(), { sandboxName: "e2e-artifact-success" }); + + expect(readJson(path.join(tmp, "onboarding.result.json"))).toMatchObject({ + phase: "onboarding", + status: "passed", + onboarding: "cloud-openclaw", + sandboxName: "e2e-artifact-success", + agent: "openclaw", + provider: "nvidia", + providerEnv: "cloud", + }); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("writes an onboarding phase result artifact on failure", async () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-onboarding-artifacts-")); + try { + const onboard = new OnboardingPhaseFixture( + new HostCliClient(new FakeRunner()), + new FakeSecrets({ NVIDIA_API_KEY: "secret" }), + undefined, + new ArtifactSink(tmp), + ); + + await expect( + onboard.from( + ready({ + docker: { id: "docker-running", expectation: "required", available: false }, + }), + ), + ).rejects.toThrow(/requires an available Docker runtime/); + + expect(readJson(path.join(tmp, "onboarding.result.json"))).toMatchObject({ + phase: "onboarding", + status: "failed", + onboarding: "cloud-openclaw", + error: "cloud-openclaw onboarding requires an available Docker runtime.", + }); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + it("exposes the onboarding phase on the Vitest scenario context", () => { expectTypeOf().toEqualTypeOf(); }); diff --git a/test/e2e-scenario/framework-tests/e2e-phase-orchestrators.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-orchestrators.test.ts deleted file mode 100644 index 1dfd9248fd1..00000000000 --- a/test/e2e-scenario/framework-tests/e2e-phase-orchestrators.test.ts +++ /dev/null @@ -1,1041 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { describe, expect, it } from "vitest"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { HostCliClient } from "../scenarios/clients/host-cli.ts"; -import { compileRunPlans } from "../scenarios/compiler.ts"; -import { PhaseOrchestrator } from "../scenarios/orchestrators/phase.ts"; -import { ScenarioRunner } from "../scenarios/orchestrators/runner.ts"; -import type { - AssertionStep, - PhaseAction, - PhaseName, - PhaseResult, - RunContext, - RunPlanPhase, -} from "../scenarios/types.ts"; - -const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); - -function freshCtx(): RunContext { - return { contextDir: fs.mkdtempSync(path.join(os.tmpdir(), "e2e-phase-")) }; -} - -function shellStep( - id: string, - phase: PhaseName, - ref: string, - reliability?: AssertionStep["reliability"], -): AssertionStep { - return { - id, - phase, - implementation: { kind: "shell", ref }, - evidencePath: `.e2e/assertions/${id}.log`, - reliability, - }; -} - -function probeStep(id: string, phase: PhaseName, ref = "no-such-probe"): AssertionStep { - return { - id, - phase, - implementation: { kind: "probe", ref }, - evidencePath: `.e2e/assertions/${id}.json`, - }; -} - -function pendingStep(id: string, phase: PhaseName): AssertionStep { - return { - id, - phase, - implementation: { kind: "pending", ref: "not-yet" }, - }; -} - -function makePhase(steps: AssertionStep[]): RunPlanPhase { - return { - name: steps[0].phase, - actions: [], - assertionGroups: [ - { id: `group.${steps[0].id}`, phase: steps[0].phase, migrationStatus: "complete", steps }, - ], - }; -} - -function writeTempScript(dir: string, name: string, body: string): string { - const p = path.join(dir, name); - fs.writeFileSync(p, `#!/usr/bin/env bash\nset -euo pipefail\n${body}\n`, { mode: 0o755 }); - return p; -} - -function shellAction( - id: string, - phase: PhaseName, - scriptRef: string, - opts: { timeoutSeconds?: number; arg?: string } = {}, -): PhaseAction { - return { - id, - phase, - kind: "shell", - scriptRef, - arg: opts.arg, - timeoutSeconds: opts.timeoutSeconds, - }; -} - -function makePhaseWithActions( - phase: PhaseName, - actions: PhaseAction[], - steps: AssertionStep[], -): RunPlanPhase { - return { - name: phase, - actions, - assertionGroups: - steps.length > 0 - ? [{ id: `group.${steps[0].id}`, phase, migrationStatus: "complete", steps }] - : [], - }; -} - -describe("phase orchestrators - top-level delegation", () => { - it("should execute phase assertions from phase orchestrators, not the top-level runner", async () => { - const ctx = freshCtx(); - try { - const [plan] = compileRunPlans(["ubuntu-repo-cloud-openclaw"]); - const calls: string[] = []; - const fakeOrchestrator = (phase: PhaseName) => ({ - run: async ( - _ctx: RunContext, - runPhase: RunPlanPhase, - _prior?: PhaseResult[], - ): Promise => { - calls.push(runPhase.name); - return { phase, status: "passed", actions: [], assertions: [] }; - }, - }); - const runner = new ScenarioRunner({ - environment: fakeOrchestrator("environment"), - onboarding: fakeOrchestrator("onboarding"), - stateValidation: fakeOrchestrator("state-validation"), - lifecycle: fakeOrchestrator("lifecycle"), - runtime: fakeOrchestrator("runtime"), - }); - - const results = await runner.run(ctx, plan); - - expect(calls).toEqual([ - "environment", - "onboarding", - "state-validation", - "lifecycle", - "runtime", - ]); - expect(results.map((result) => result.phase)).toEqual([ - "environment", - "onboarding", - "state-validation", - "lifecycle", - "runtime", - ]); - } finally { - fs.rmSync(ctx.contextDir, { recursive: true, force: true }); - } - }); -}); - -describe("phase orchestrators - real shell execution", () => { - it("shell step passes when the script exits zero", async () => { - const ctx = freshCtx(); - try { - const script = writeTempScript(ctx.contextDir, "ok.sh", "echo hello-from-real-shell"); - const ref = path.relative(REPO_ROOT, script); - const step = shellStep("runtime.real-pass", "runtime", ref); - const orchestrator = new PhaseOrchestrator("runtime"); - - const result = await orchestrator.run(ctx, makePhase([step])); - - expect(result.status).toBe("passed"); - expect(result.assertions[0]).toEqual( - expect.objectContaining({ id: "runtime.real-pass", status: "passed", attempts: 1 }), - ); - const log = fs.readFileSync(result.assertions[0].evidence!, "utf8"); - expect(log).toContain("hello-from-real-shell"); - } finally { - fs.rmSync(ctx.contextDir, { recursive: true, force: true }); - } - }); - - it("shell step fails when the script exits nonzero and records the stderr tail", async () => { - const ctx = freshCtx(); - try { - const script = writeTempScript( - ctx.contextDir, - "fail.sh", - 'echo "boom: real failure" >&2; exit 7', - ); - const ref = path.relative(REPO_ROOT, script); - const step = shellStep("runtime.real-fail", "runtime", ref); - const orchestrator = new PhaseOrchestrator("runtime"); - - const result = await orchestrator.run(ctx, makePhase([step])); - - expect(result.status).toBe("failed"); - expect(result.assertions[0].status).toBe("failed"); - expect(result.assertions[0].message).toMatch(/exit 7/); - expect(result.assertions[0].message).toMatch(/boom: real failure/); - } finally { - fs.rmSync(ctx.contextDir, { recursive: true, force: true }); - } - }); - - it("shell step times out via orchestrator policy, not the script", async () => { - const ctx = freshCtx(); - try { - const script = writeTempScript(ctx.contextDir, "slow.sh", "sleep 30"); - const ref = path.relative(REPO_ROOT, script); - const step = shellStep("runtime.real-timeout", "runtime", ref, { timeoutSeconds: 1 }); - const orchestrator = new PhaseOrchestrator("runtime"); - - const started = Date.now(); - const result = await orchestrator.run(ctx, makePhase([step])); - const elapsed = Date.now() - started; - - expect(result.status).toBe("failed"); - expect(result.assertions[0].message).toMatch(/exceeded 1s/); - expect(elapsed).toBeLessThan(15_000); - } finally { - fs.rmSync(ctx.contextDir, { recursive: true, force: true }); - } - }, 20_000); - - it("shell step retries on a classified transient and then passes", async () => { - const ctx = freshCtx(); - try { - const counterFile = path.join(ctx.contextDir, "counter"); - fs.writeFileSync(counterFile, "0"); - const script = writeTempScript( - ctx.contextDir, - "gateway-flaky.sh", - `n=$(cat "${counterFile}"); n=$((n+1)); echo "$n" > "${counterFile}"; if [ "$n" -lt 2 ]; then echo "gateway-transient: try again" >&2; exit 1; fi; echo ok`, - ); - const ref = path.relative(REPO_ROOT, script); - const step = shellStep("runtime.gateway-retry", "runtime", ref, { - retry: { attempts: 2, on: ["gateway-transient"] }, - }); - const orchestrator = new PhaseOrchestrator("runtime"); - - const result = await orchestrator.run(ctx, makePhase([step])); - - expect(result.status).toBe("passed"); - expect(result.assertions[0].attempts).toBe(2); - expect(result.assertions[0].classifier).toBe("gateway-transient"); - } finally { - fs.rmSync(ctx.contextDir, { recursive: true, force: true }); - } - }); - - it("shell step fails with a clear message when the script is missing", async () => { - const ctx = freshCtx(); - try { - const step = shellStep("runtime.missing", "runtime", "test/e2e-scenario/does-not-exist.sh"); - const orchestrator = new PhaseOrchestrator("runtime"); - - const result = await orchestrator.run(ctx, makePhase([step])); - - expect(result.status).toBe("failed"); - expect(result.assertions[0].message).toMatch(/script not found/); - } finally { - fs.rmSync(ctx.contextDir, { recursive: true, force: true }); - } - }); - - it("probe step without a registered probe skips visibly and never passes falsely", async () => { - const ctx = freshCtx(); - try { - const step = probeStep("runtime.probe-pending", "runtime"); - const orchestrator = new PhaseOrchestrator("runtime"); - - const result = await orchestrator.run(ctx, makePhase([step])); - - expect(result.assertions[0].status).toBe("skipped"); - expect(result.assertions[0].message).toMatch(/probe not registered/); - } finally { - fs.rmSync(ctx.contextDir, { recursive: true, force: true }); - } - }); - - it("pending step skips visibly with a pending marker", async () => { - const ctx = freshCtx(); - try { - const step = pendingStep("runtime.pending", "runtime"); - const orchestrator = new PhaseOrchestrator("runtime"); - - const result = await orchestrator.run(ctx, makePhase([step])); - - expect(result.assertions[0].status).toBe("skipped"); - expect(result.assertions[0].message).toMatch(/^pending:/); - } finally { - fs.rmSync(ctx.contextDir, { recursive: true, force: true }); - } - }); -}); - -describe("phase orchestrators - actions execute before assertions", () => { - it("phase action runs before assertions and records evidence", async () => { - const ctx = freshCtx(); - try { - const actionScript = writeTempScript( - ctx.contextDir, - "setup.sh", - "echo phase-action-evidence", - ); - const action = shellAction( - "environment.setup-ok", - "environment", - path.relative(REPO_ROOT, actionScript), - ); - const stepScript = writeTempScript(ctx.contextDir, "after.sh", "echo after-action"); - const step = shellStep( - "environment.assert-ok", - "environment", - path.relative(REPO_ROOT, stepScript), - ); - const orchestrator = new PhaseOrchestrator("environment"); - - const result = await orchestrator.run( - ctx, - makePhaseWithActions("environment", [action], [step]), - ); - - expect(result.status).toBe("passed"); - expect(result.actions).toHaveLength(1); - expect(result.actions[0]).toEqual( - expect.objectContaining({ id: "environment.setup-ok", status: "passed" }), - ); - expect(result.actions[0].evidence).toBeTruthy(); - const actionLog = fs.readFileSync(result.actions[0].evidence!, "utf8"); - expect(actionLog).toContain("phase-action-evidence"); - expect(result.assertions).toHaveLength(1); - expect(result.assertions[0].status).toBe("passed"); - } finally { - fs.rmSync(ctx.contextDir, { recursive: true, force: true }); - } - }); - - it("phase action failure short-circuits assertions", async () => { - const ctx = freshCtx(); - try { - const failScript = writeTempScript( - ctx.contextDir, - "fail.sh", - 'echo "setup boom" >&2; exit 5', - ); - const action = shellAction( - "environment.setup-fail", - "environment", - path.relative(REPO_ROOT, failScript), - ); - const stepScript = writeTempScript(ctx.contextDir, "after.sh", "echo should-not-run"); - const step = shellStep( - "environment.never-runs", - "environment", - path.relative(REPO_ROOT, stepScript), - ); - const orchestrator = new PhaseOrchestrator("environment"); - - const result = await orchestrator.run( - ctx, - makePhaseWithActions("environment", [action], [step]), - ); - - expect(result.status).toBe("failed"); - expect(result.actions).toHaveLength(1); - expect(result.actions[0].status).toBe("failed"); - expect(result.actions[0].message).toMatch(/exit 5/); - // Assertions must NOT have run, so they must NOT show a misleading - // pass for an environment that was never set up. - expect(result.assertions).toEqual([]); - } finally { - fs.rmSync(ctx.contextDir, { recursive: true, force: true }); - } - }); - - it("phase action times out via orchestrator policy", async () => { - const ctx = freshCtx(); - try { - const slow = writeTempScript(ctx.contextDir, "slow.sh", "sleep 30"); - const action = shellAction( - "environment.setup-slow", - "environment", - path.relative(REPO_ROOT, slow), - { - timeoutSeconds: 1, - }, - ); - const orchestrator = new PhaseOrchestrator("environment"); - - const started = Date.now(); - const result = await orchestrator.run(ctx, makePhaseWithActions("environment", [action], [])); - - expect(result.status).toBe("failed"); - expect(result.actions[0].status).toBe("failed"); - expect(result.actions[0].message).toMatch(/exceeded 1s/); - // The orchestrator must enforce the timeout, not depend on the - // script self-killing. Allow some headroom but fail if we waited - // anywhere near the script's 30s sleep. - expect(Date.now() - started).toBeLessThan(15_000); - } finally { - fs.rmSync(ctx.contextDir, { recursive: true, force: true }); - } - }); - - it("phase action publishes alias path on success", async () => { - const ctx = freshCtx(); - try { - const actionScript = writeTempScript(ctx.contextDir, "alias.sh", "echo aliased-output"); - const action: PhaseAction = { - id: "onboarding.profile.alias-demo", - phase: "onboarding", - kind: "shell", - scriptRef: path.relative(REPO_ROOT, actionScript), - aliasPath: "onboard.log", - }; - const orchestrator = new PhaseOrchestrator("onboarding"); - - const result = await orchestrator.run(ctx, makePhaseWithActions("onboarding", [action], [])); - - expect(result.actions[0].status).toBe("passed"); - const aliasContents = fs.readFileSync(path.join(ctx.contextDir, "onboard.log"), "utf8"); - expect(aliasContents).toContain("aliased-output"); - } finally { - fs.rmSync(ctx.contextDir, { recursive: true, force: true }); - } - }); - - it("phase action evidence log is flushed before resolve", async () => { - const ctx = freshCtx(); - try { - const actionScript = writeTempScript( - ctx.contextDir, - "flush.sh", - "echo flushed-phase-action-output", - ); - const action = shellAction( - "environment.flush", - "environment", - path.relative(REPO_ROOT, actionScript), - ); - const orchestrator = new PhaseOrchestrator("environment"); - - const result = await orchestrator.run(ctx, makePhaseWithActions("environment", [action], [])); - - // Synchronous read must already see the output - the orchestrator - // must wait for the WriteStream's 'finish' before resolving. - const log = fs.readFileSync(result.actions[0].evidence!, "utf8"); - expect(log).toContain("flushed-phase-action-output"); - } finally { - fs.rmSync(ctx.contextDir, { recursive: true, force: true }); - } - }); -}); - -describe("plan compiler emits phase actions for canonical scenarios", () => { - it("compiler emits install and onboard actions for canonical scenarios", async () => { - const { compileRunPlans } = await import("../scenarios/compiler.ts"); - const ids = [ - "ubuntu-repo-cloud-openclaw", - "ubuntu-repo-cloud-hermes", - "gpu-repo-local-ollama-openclaw", - "macos-repo-cloud-openclaw", - "wsl-repo-cloud-openclaw", - "brev-launchable-cloud-openclaw", - "ubuntu-no-docker-preflight-negative", - ]; - const plans = compileRunPlans(ids); - expect(plans).toHaveLength(ids.length); - for (const plan of plans) { - const env = plan.phases.find((p) => p.name === "environment")!; - const onb = plan.phases.find((p) => p.name === "onboarding")!; - expect(env.actions.some((a) => a.id.startsWith("environment.install."))).toBe(true); - expect(onb.actions.some((a) => a.id.startsWith("onboarding.profile."))).toBe(true); - // context.env emission is framework infrastructure (ScenarioRunner), - // not a shell action. The compiler must NOT emit a shell context - // action - if it did we'd be coupling back to the old resolver's - // plan.json shape. - expect(env.actions.map((a) => a.id)).not.toContain("environment.context.emit"); - // Onboarding action must publish a stable alias path so legacy - // shell assertions referencing ${E2E_CONTEXT_DIR}/onboard.log - // keep working without coupling them to action ids. - const onboardingAction = onb.actions.find((a) => a.id.startsWith("onboarding.profile.")); - expect(onboardingAction?.aliasPath).toBe("onboard.log"); - // Every install/onboard action must be a typed shell-fn referencing - // the canonical dispatcher script - no free-form strings. - for (const action of [...env.actions, ...onb.actions]) { - if ( - action.id.startsWith("environment.install.") || - action.id.startsWith("onboarding.profile.") - ) { - expect(action.kind).toBe("shell-fn"); - expect(action.scriptRef).toMatch(/dispatch\.sh$/); - expect(action.fn).toMatch(/^e2e_(install|onboard)$/); - expect(action.arg).toBeTruthy(); - } - } - } - }); - - it("compiler routes Docker-missing runtime to the no-Docker onboarding profile", async () => { - const { compileRunPlans } = await import("../scenarios/compiler.ts"); - // Negative scenario declares runtime=docker-missing in scenarios.yaml. - // The compiler must substitute the onboarding profile id from the - // base 'cloud-openclaw' to 'cloud-openclaw-no-docker' so the - // dispatcher routes to the worker that installs the docker shim and - // captures negative-preflight.log. Without this routing, the - // 'onboarding.preflight.expected-failed' assertion has nothing to grep. - const [plan] = compileRunPlans(["ubuntu-no-docker-preflight-negative"]); - const onb = plan.phases.find((p) => p.name === "onboarding")!; - const action = onb.actions.find((a) => a.id.startsWith("onboarding.profile.")); - expect(action?.id).toBe("onboarding.profile.cloud-openclaw-no-docker"); - expect(action?.arg).toBe("cloud-openclaw-no-docker"); - expect(action?.evidencePath).toBe( - ".e2e/actions/onboarding.profile.cloud-openclaw-no-docker.log", - ); - // Secret env must still include NVIDIA_API_KEY so behavior matches - // a real user invocation (CLI loads creds even if preflight aborts). - expect(action?.secretEnv).toContain("NVIDIA_API_KEY"); - // Positive scenarios must NOT pick up the -no-docker suffix. - const [posPlan] = compileRunPlans(["ubuntu-repo-cloud-openclaw"]); - const posAction = posPlan.phases - .find((p) => p.name === "onboarding")! - .actions.find((a) => a.id.startsWith("onboarding.profile.")); - expect(posAction?.arg).toBe("cloud-openclaw"); - }); - - it("compiler emits lifecycle phase action when scenario declares lifecycle profile", async () => { - const { compileRunPlans } = await import("../scenarios/compiler.ts"); - // Rebuild scenario declares environment.lifecycle = - // 'rebuild-current-version'. The compiler must emit a single - // lifecycle phase action that dispatches to the canonical - // lifecycle dispatcher; without this, runtime-phase rebuild - // assertions run against a sandbox that was never rebuilt. - const [plan] = compileRunPlans(["ubuntu-rebuild-openclaw"]); - const lifecycle = plan.phases.find((p) => p.name === "lifecycle")!; - expect(lifecycle).toBeTruthy(); - expect(lifecycle.actions).toHaveLength(1); - const action = lifecycle.actions[0]; - expect(action.id).toBe("lifecycle.profile.rebuild-current-version"); - expect(action.arg).toBe("rebuild-current-version"); - expect(action.scriptRef).toMatch(/lifecycle\/dispatch\.sh$/); - expect(action.fn).toBe("e2e_lifecycle"); - expect(action.evidencePath).toBe(".e2e/actions/lifecycle.profile.rebuild-current-version.log"); - // Secret env: nemoclaw rebuild re-reads NVIDIA_API_KEY when the - // post-rebuild sandbox is brought back up. - expect(action.secretEnv).toContain("NVIDIA_API_KEY"); - }); - - it("compiler emits no lifecycle actions when scenario does not declare lifecycle", async () => { - const { compileRunPlans } = await import("../scenarios/compiler.ts"); - // Default scenarios omit environment.lifecycle. The lifecycle - // phase still appears in the plan (deterministic phase order) - // but emits zero actions and runs no assertions. - const [plan] = compileRunPlans(["ubuntu-repo-cloud-openclaw"]); - const lifecycle = plan.phases.find((p) => p.name === "lifecycle")!; - expect(lifecycle).toBeTruthy(); - expect(lifecycle.actions).toHaveLength(0); - expect(lifecycle.assertionGroups).toHaveLength(0); - }); - - it("compiler drops rebuild and upgrade supplemental suites from cloud OpenClaw", async () => { - const { compileRunPlans } = await import("../scenarios/compiler.ts"); - // The 'rebuild' and 'upgrade' suites used to be supplementally - // attached to ubuntu-repo-cloud-openclaw, which produced - // fake-failures (no rebuild ran -> nothing could be preserved). - // Coverage now lives on ubuntu-rebuild-openclaw, which actually - // runs the lifecycle phase. The cloud-openclaw scenario must NOT - // include those suites' assertion groups. - const [plan] = compileRunPlans(["ubuntu-repo-cloud-openclaw"]); - const runtime = plan.phases.find((p) => p.name === "runtime")!; - const groupIds = runtime.assertionGroups.map((g) => g.id); - expect(groupIds).not.toContain("suite.rebuild"); - expect(groupIds).not.toContain("suite.upgrade"); - }); - - it("compiler includes rebuild and upgrade groups on ubuntu-rebuild-openclaw", async () => { - const { compileRunPlans } = await import("../scenarios/compiler.ts"); - const [plan] = compileRunPlans(["ubuntu-rebuild-openclaw"]); - const runtime = plan.phases.find((p) => p.name === "runtime")!; - const groupIds = runtime.assertionGroups.map((g) => g.id); - expect(groupIds).toContain("suite.rebuild"); - expect(groupIds).toContain("suite.upgrade"); - }); -}); - -describe("ScenarioRunner seeds context.env and short-circuits across phases", () => { - it("seedContextEnv writes normalized keys at the top-level context env path", async () => { - const { compileRunPlans } = await import("../scenarios/compiler.ts"); - const { seedContextEnv } = await import("../scenarios/orchestrators/context.ts"); - const ctx = freshCtx(); - try { - const [plan] = compileRunPlans(["ubuntu-repo-cloud-openclaw"]); - const result = seedContextEnv(ctx, plan); - - // Path matches the shell helper's e2e_context_init: top-level, - // not under .e2e/. Runtime steps source ${E2E_CONTEXT_DIR}/context.env. - expect(result.path).toBe(path.join(ctx.contextDir, "context.env")); - const body = fs.readFileSync(result.path, "utf8"); - // Required keys downstream shell assertions look up. - expect(body).toMatch(/^E2E_SCENARIO=ubuntu-repo-cloud-openclaw$/m); - expect(body).toMatch(/^E2E_PLATFORM_OS=ubuntu$/m); - expect(body).toMatch(/^E2E_AGENT=openclaw$/m); - expect(body).toMatch(/^E2E_PROVIDER=nvidia$/m); - expect(body).toMatch(/^E2E_GATEWAY_URL=http:\/\/127\.0\.0\.1:18789$/m); - expect(body).toMatch(/^E2E_SANDBOX_NAME=e2e-ubuntu-repo-cloud-openclaw$/m); - } finally { - fs.rmSync(ctx.contextDir, { recursive: true, force: true }); - } - }); - - it("Hermes scenario seeds the Hermes gateway URL", async () => { - const { compileRunPlans } = await import("../scenarios/compiler.ts"); - const { seedContextEnv } = await import("../scenarios/orchestrators/context.ts"); - const ctx = freshCtx(); - try { - const [plan] = compileRunPlans(["ubuntu-repo-cloud-hermes"]); - const result = seedContextEnv(ctx, plan); - const body = fs.readFileSync(result.path, "utf8"); - expect(body).toMatch(/^E2E_AGENT=hermes$/m); - expect(body).toMatch(/^E2E_GATEWAY_URL=http:\/\/127\.0\.0\.1:8642$/m); - } finally { - fs.rmSync(ctx.contextDir, { recursive: true, force: true }); - } - }); - - it("runner skips downstream phases when a prior phase action fails", async () => { - const { ScenarioRunner } = await import("../scenarios/orchestrators/runner.ts"); - const { compileRunPlans } = await import("../scenarios/compiler.ts"); - const ctx = freshCtx(); - try { - const [plan] = compileRunPlans(["ubuntu-repo-cloud-openclaw"]); - // Inject a failing environment phase to simulate an install action - // failure. Onboarding and runtime must report skipped, not run - // their own actions or assertions. - const failingEnv = { - run: async () => ({ - phase: "environment" as const, - status: "failed" as const, - actions: [ - { - id: "environment.install.repo-current", - status: "failed" as const, - durationMs: 5, - message: "simulated install failure", - }, - ], - assertions: [], - }), - }; - let onboardingCalled = false; - let runtimeCalled = false; - const onboarding = { - run: async () => { - onboardingCalled = true; - return { - phase: "onboarding" as const, - status: "passed" as const, - actions: [], - assertions: [], - }; - }, - }; - const runtime = { - run: async () => { - runtimeCalled = true; - return { - phase: "runtime" as const, - status: "passed" as const, - actions: [], - assertions: [], - }; - }, - }; - let stateValidationCalled = false; - const stateValidation = { - run: async () => { - stateValidationCalled = true; - return { - phase: "state-validation" as const, - status: "passed" as const, - actions: [], - assertions: [], - }; - }, - }; - const runner = new ScenarioRunner({ - environment: failingEnv, - onboarding, - stateValidation, - runtime, - }); - - const results = await runner.run(ctx, plan); - - // Downstream orchestrators must NOT have been invoked. An - // environment failure means install never ran; there is nothing - // for state-validation to probe. - expect(onboardingCalled).toBe(false); - expect(stateValidationCalled).toBe(false); - expect(runtimeCalled).toBe(false); - // Each phase still has a result, and the downstream ones are - // skipped with a message that names the blocking action. - expect(results.map((r) => r.phase)).toEqual([ - "environment", - "onboarding", - "state-validation", - "lifecycle", - "runtime", - ]); - expect(results[1].status).toBe("skipped"); - expect(results[2].status).toBe("skipped"); - expect(results[3].status).toBe("skipped"); - expect(results[4].status).toBe("skipped"); - expect(results[1].assertions[0].message).toMatch(/blocked by prior failure/); - expect(results[1].assertions[0].message).toMatch(/environment.install.repo-current/); - } finally { - fs.rmSync(ctx.contextDir, { recursive: true, force: true }); - } - }); - - it("runner does not short-circuit on assertion failures alone", async () => { - // Assertion failures (as opposed to action failures) must not block - // downstream phases - reviewers need to see all failure layers. - const { ScenarioRunner } = await import("../scenarios/orchestrators/runner.ts"); - const { compileRunPlans } = await import("../scenarios/compiler.ts"); - const ctx = freshCtx(); - try { - const [plan] = compileRunPlans(["ubuntu-repo-cloud-openclaw"]); - const env = { - run: async () => ({ - phase: "environment" as const, - status: "failed" as const, - actions: [], - assertions: [ - { id: "environment.something", status: "failed" as const, attempts: 1, durationMs: 1 }, - ], - }), - }; - let onboardingCalled = false; - const onboarding = { - run: async () => { - onboardingCalled = true; - return { - phase: "onboarding" as const, - status: "passed" as const, - actions: [], - assertions: [], - }; - }, - }; - const runner = new ScenarioRunner({ - environment: env, - onboarding, - runtime: { - run: async () => ({ - phase: "runtime" as const, - status: "passed" as const, - actions: [], - assertions: [], - }), - }, - }); - - await runner.run(ctx, plan); - expect(onboardingCalled).toBe(true); - } finally { - fs.rmSync(ctx.contextDir, { recursive: true, force: true }); - } - }); -}); - -describe("required probe and pending steps fail closed", () => { - it("required probe step that is unregistered fails the phase", async () => { - const ctx = freshCtx(); - try { - const step: AssertionStep = { - id: "runtime.security.required-probe", - phase: "runtime", - implementation: { kind: "probe", ref: "unregisteredSecurityProbe" }, - evidencePath: ".e2e/assertions/runtime.security.required-probe.json", - required: true, - }; - const orchestrator = new PhaseOrchestrator("runtime"); - - const result = await orchestrator.run(ctx, makePhase([step])); - - expect(result.status).toBe("failed"); - expect(result.assertions[0].status).toBe("failed"); - expect(result.assertions[0].message).toMatch(/required probe not registered/); - expect(result.assertions[0].message).toContain("unregisteredSecurityProbe"); - } finally { - fs.rmSync(ctx.contextDir, { recursive: true, force: true }); - } - }); - - it("non-required probe step continues to skip visibly", async () => { - const ctx = freshCtx(); - try { - const step: AssertionStep = { - id: "runtime.diagnostics.non-required-probe", - phase: "runtime", - // Use an intentionally-unregistered ref so this test exercises - // the "missing probe" code path. `diagnosticsProbe` is now a - // real built-in registered at orchestrator import time, so - // referring to it here would actually invoke nemoclaw and the - // assertion would fail (or pass) on real CLI behavior — - // unrelated to what this test verifies. - implementation: { kind: "probe", ref: "unregisteredFakeProbe" }, - evidencePath: ".e2e/assertions/runtime.diagnostics.non-required-probe.json", - // required intentionally omitted (defaults to false) - }; - const orchestrator = new PhaseOrchestrator("runtime"); - - const result = await orchestrator.run(ctx, makePhase([step])); - - expect(result.assertions[0].status).toBe("skipped"); - expect(result.assertions[0].message).toMatch(/probe not registered/); - // Non-required skipped step does not fail the phase. - expect(result.status).not.toBe("failed"); - } finally { - fs.rmSync(ctx.contextDir, { recursive: true, force: true }); - } - }); - - it("required pending step fails closed", async () => { - const ctx = freshCtx(); - try { - const step: AssertionStep = { - id: "runtime.expected-failure.no-side-effects", - phase: "runtime", - implementation: { kind: "pending", ref: "expectedFailureNoSideEffectsProbe" }, - evidencePath: ".e2e/assertions/runtime.expected-failure.no-side-effects.json", - required: true, - }; - const orchestrator = new PhaseOrchestrator("runtime"); - - const result = await orchestrator.run(ctx, makePhase([step])); - - expect(result.status).toBe("failed"); - expect(result.assertions[0].status).toBe("failed"); - expect(result.assertions[0].message).toMatch(/required pending step not implemented/); - } finally { - fs.rmSync(ctx.contextDir, { recursive: true, force: true }); - } - }); - - it("security suite groups in registry mark their steps as required", async () => { - const { assertionGroupForSuite } = await import("../scenarios/assertions/registry.ts"); - for (const suiteId of ["security-shields", "security-policy", "security-injection"]) { - const group = assertionGroupForSuite(suiteId); - expect(group, `missing assertion group for suite ${suiteId}`).toBeDefined(); - for (const step of group?.steps ?? []) { - expect( - step.required, - `${suiteId} step ${step.id} must be required so it fails closed`, - ).toBe(true); - } - } - }); - - it("expected-failure no-side-effects step is not in the active registry", async () => { - const { assertionRegistry } = await import("../scenarios/assertions/registry.ts"); - const group = assertionRegistry.groups.find( - (g) => g.id === "runtime.expected-failure.no-side-effects", - ); - expect(group).toBeUndefined(); - }); -}); - -describe("framework-owned secret hygiene at the spawn boundary", () => { - it("should not persist secret-shaped child output into evidence", async () => { - const ctx = freshCtx(); - try { - // Child writes secret-shaped tokens (NVIDIA, GitHub, OpenAI, - // Slack, Bearer-prefixed) on both stdout and stderr, then exits - // non-zero so stderrTail also flows into result.message. None of - // those literal tokens may persist anywhere in the evidence. - const body = [ - 'echo "step prints nvapi-1234567890abcdef0123456789"', - 'echo "and ghp_abcdefghijklmnopqrstuvwxyz0123456789"', - 'echo "and sk-abcdefghijklmnopqrstuvwxyz0123456789"', - 'echo "and xoxb-9876543210-fake-bot-token-abc"', - 'echo "Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.payload.signature" 1>&2', - "exit 7", - ].join("\n"); - const script = writeTempScript(ctx.contextDir, "leak.sh", body); - const ref = path.relative(REPO_ROOT, script); - const step = shellStep("runtime.leak", "runtime", ref); - const orchestrator = new PhaseOrchestrator("runtime"); - - const result = await orchestrator.run(ctx, makePhase([step])); - const assertion = result.assertions[0]; - const logBody = fs.readFileSync( - path.join(ctx.contextDir, ".e2e", "logs", `${step.id}.log`), - "utf8", - ); - const phaseResultJson = fs.readFileSync( - path.join(ctx.contextDir, ".e2e", "runtime.result.json"), - "utf8", - ); - const surfaces = [logBody, assertion.message ?? "", phaseResultJson]; - - // Every secret-shaped token canonicalized in - // src/lib/security/secret-patterns.ts must be redacted on the - // way to disk, regardless of which surface is read. - const forbiddenPatterns = [ - /nvapi-[A-Za-z0-9_-]{10,}/, - /ghp_[A-Za-z0-9_-]{10,}/, - /sk-[A-Za-z0-9_-]{20,}/, - /(?:xox[bpas]|xapp)-[A-Za-z0-9-]{10,}/, - /Bearer\s+[A-Za-z0-9_.+\/=-]{10,}/i, - ]; - for (const surface of surfaces) { - for (const pat of forbiddenPatterns) { - expect(surface, `evidence surface must not contain ${pat}`).not.toMatch(pat); - } - expect(surface).toMatch(//); - } - } finally { - fs.rmSync(ctx.contextDir, { recursive: true, force: true }); - } - }); - - it("should drop non-allowlisted parent env unless declared in secretEnv", async () => { - const ctx = freshCtx(); - const sentinelKey = "SECRET_LEAK_PROBE_TOKEN"; - const previous = process.env[sentinelKey]; - process.env[sentinelKey] = "sentinel-value-that-must-not-leak"; - try { - const script = writeTempScript(ctx.contextDir, "env-leak.sh", `printenv | sort\n`); - const ref = path.relative(REPO_ROOT, script); - // Step does NOT declare SECRET_LEAK_PROBE_TOKEN in secretEnv, - // so the framework must drop it before spawn. - const step = shellStep("runtime.env-drop", "runtime", ref); - const orchestrator = new PhaseOrchestrator("runtime"); - - const result = await orchestrator.run(ctx, makePhase([step])); - const logBody = fs.readFileSync( - path.join(ctx.contextDir, ".e2e", "logs", `${step.id}.log`), - "utf8", - ); - - expect(result.assertions[0].status).toBe("passed"); - expect(logBody, "non-allowlisted parent env must not reach the child").not.toContain( - sentinelKey, - ); - expect(logBody).not.toContain("sentinel-value-that-must-not-leak"); - // Framework allowlist + overlay still arrive: PATH and E2E_PHASE. - expect(logBody).toMatch(/^PATH=/m); - expect(logBody).toMatch(/^E2E_PHASE=runtime$/m); - } finally { - if (previous === undefined) delete process.env[sentinelKey]; - else process.env[sentinelKey] = previous; - fs.rmSync(ctx.contextDir, { recursive: true, force: true }); - } - }); - - it("should pass declared secretEnv through to child", async () => { - const ctx = freshCtx(); - const declaredKey = "NEMOCLAW_TEST_API_KEY"; // matches SECRET_ENV_KEY_SHAPE - const previous = process.env[declaredKey]; - process.env[declaredKey] = "declared-secret-value-passes-through"; - try { - const script = writeTempScript( - ctx.contextDir, - "declared.sh", - `printenv ${declaredKey} || echo MISSING\n`, - ); - const ref = path.relative(REPO_ROOT, script); - const step: AssertionStep = { - ...shellStep("runtime.env-declared", "runtime", ref), - secretEnv: [declaredKey], - }; - const orchestrator = new PhaseOrchestrator("runtime"); - - const result = await orchestrator.run(ctx, makePhase([step])); - const logBody = fs.readFileSync( - path.join(ctx.contextDir, ".e2e", "logs", `${step.id}.log`), - "utf8", - ); - - expect(result.assertions[0].status).toBe("passed"); - // Declared secret reaches the child verbatim (printenv would - // print MISSING otherwise), but the orchestrator scrubs the - // value at the I/O boundary before any byte reaches evidence: - // the explicit secretEnv values are passed to redactString - // alongside the canonical token-shape patterns, so secrets - // without a recognised shape still get sanitised. - expect(logBody).not.toContain("MISSING"); - expect(logBody).not.toContain("declared-secret-value-passes-through"); - expect(logBody).toContain("[REDACTED]"); - } finally { - if (previous === undefined) delete process.env[declaredKey]; - else process.env[declaredKey] = previous; - fs.rmSync(ctx.contextDir, { recursive: true, force: true }); - } - }); - - it("should reject non-secret-shaped keys in secretEnv at runtime", async () => { - const { buildChildEnv } = await import("../scenarios/orchestrators/redaction.ts"); - expect(() => - buildChildEnv(process.env, { secretEnv: ["FOO_VAR"], frameworkOverlay: {} }), - ).toThrow(/secret-key shape/); - }); - - it("should declare NVIDIA API key only for cloud onboarding actions", async () => { - const { compileRunPlans } = await import("../scenarios/compiler.ts"); - const plans = compileRunPlans(["ubuntu-repo-cloud-openclaw", "gpu-repo-local-ollama-openclaw"]); - const cloudOnboard = plans[0].phases - .find((p) => p.name === "onboarding") - ?.actions.find((a) => a.id.startsWith("onboarding.profile.")); - const localOnboard = plans[1].phases - .find((p) => p.name === "onboarding") - ?.actions.find((a) => a.id.startsWith("onboarding.profile.")); - expect(cloudOnboard?.secretEnv).toEqual(["NVIDIA_API_KEY"]); - expect(localOnboard?.secretEnv).toEqual([]); - }); -}); - -describe("clients are pass/fail/policy free", () => { - it("should keep clients free of pass/fail and retry semantics", () => { - const observation = new HostCliClient().observeVersion(); - - // The client returns a raw act/observe shape only: the command it would - // run. It must NOT decide pass/fail, attach retry policy, surface a - // classifier, or expose AssertionResult/PhaseResult-shaped fields. - expect(observation).toEqual(expect.objectContaining({ command: ["nemoclaw", "--version"] })); - // Raw act/observe fields are allowed (exitCode/stdout/stderr/timing). - // Pass/fail and reliability-policy fields are not. - const forbiddenKeys = [ - "status", - "attempts", - "classifier", - "evidence", - "retry", - "timeout", - "timeoutSeconds", - "phase", - "assertions", - "passed", - "failed", - ]; - for (const key of forbiddenKeys) { - expect(observation).not.toHaveProperty(key); - } - }); -}); diff --git a/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts index 2d4011061b7..ca69fc221b4 100644 --- a/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts @@ -1,8 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + import { describe, expect, expectTypeOf, it } from "vitest"; +import { ArtifactSink } from "../framework/artifacts.ts"; import { GatewayClient, HostCliClient, @@ -70,6 +75,10 @@ function shellResult(exitCode: number, output = ""): ShellProbeResult { }; } +function readJson(filePath: string): unknown { + return JSON.parse(fs.readFileSync(filePath, "utf8")); +} + class FakeRunner implements CommandRunner { readonly calls: RunnerCall[] = []; private readonly responses: Array = []; @@ -111,6 +120,7 @@ function instance(overrides: Partial = {}): NemoClawInstance { function fixture( runner: FakeRunner, io: ConstructorParameters[3] = {}, + artifacts?: ArtifactSink, ): StateValidationPhaseFixture { const host = new HostCliClient(runner); return new StateValidationPhaseFixture( @@ -118,6 +128,7 @@ function fixture( new GatewayClient(host), new SandboxClient(runner), io, + artifacts, ); } @@ -459,6 +470,45 @@ describe("state-validation phase fixture", () => { ); }); + it("writes a state-validation phase result artifact on success", async () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-state-validation-artifacts-")); + try { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + const fx = fixture(runner, {}, new ArtifactSink(tmp)); + + await fx.from("macos-cli-ready-docker-optional"); + + expect(readJson(path.join(tmp, "state-validation.result.json"))).toMatchObject({ + phase: "state-validation", + status: "passed", + expectedStateId: "macos-cli-ready-docker-optional", + probes: ["cli-installed"], + }); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("writes a state-validation phase result artifact on failure", async () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-state-validation-artifacts-")); + try { + const fx = fixture(new FakeRunner(), {}, new ArtifactSink(tmp)); + + await expect(fx.from("missing-state", instance())).rejects.toThrow(/Unknown expected_state/); + + expect(readJson(path.join(tmp, "state-validation.result.json"))).toMatchObject({ + phase: "state-validation", + status: "failed", + expectedStateId: "missing-state", + probes: [], + error: expect.stringContaining("Unknown expected_state"), + }); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + it("exposes the state-validation phase on the Vitest scenario context", () => { expectTypeOf< E2EScenarioFixtures["stateValidation"] diff --git a/test/e2e-scenario/framework-tests/e2e-plan-compiler.test.ts b/test/e2e-scenario/framework-tests/e2e-plan-compiler.test.ts deleted file mode 100644 index 62498888514..00000000000 --- a/test/e2e-scenario/framework-tests/e2e-plan-compiler.test.ts +++ /dev/null @@ -1,108 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { describe, expect, it } from "vitest"; -import { spawnSync } from "node:child_process"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { compileRunPlans } from "../scenarios/compiler.ts"; -import { listScenarios } from "../scenarios/registry.ts"; -import type { ScenarioDefinition } from "../scenarios/types.ts"; - -const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); -const RUN_SCENARIOS = path.join(REPO_ROOT, "test/e2e-scenario/scenarios/run.ts"); -const TSX = path.join(REPO_ROOT, "node_modules/.bin/tsx"); - -function runScenarioCli(args: string[], env: Record = {}) { - return spawnSync(TSX, [RUN_SCENARIOS, ...args], { - cwd: REPO_ROOT, - env: { ...process.env, ...env }, - encoding: "utf8", - timeout: Number(process.env.E2E_SPAWN_TIMEOUT_MS ?? 60_000), - }); -} - -describe("plan compiler", () => { - it("should emit machine and human plan artifacts under context dir", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-plan-")); - try { - const result = runScenarioCli(["--scenarios", "ubuntu-repo-cloud-openclaw", "--plan-only"], { - E2E_CONTEXT_DIR: tmp, - }); - - expect(result.status, result.stderr).toBe(0); - const planPath = path.join(tmp, ".e2e", "run-plan.json"); - const summaryPath = path.join(tmp, ".e2e", "plan.txt"); - expect(fs.existsSync(planPath)).toBe(true); - expect(fs.existsSync(summaryPath)).toBe(true); - const plans = JSON.parse(fs.readFileSync(planPath, "utf8")); - expect(plans[0].scenarioId).toBe("ubuntu-repo-cloud-openclaw"); - expect(fs.readFileSync(summaryPath, "utf8")).toContain( - "Scenario: ubuntu-repo-cloud-openclaw", - ); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("should include expanded assertion steps by phase", () => { - const [plan] = compileRunPlans(["ubuntu-repo-cloud-openclaw"]); - const onboarding = plan.phases.find((phase) => phase.name === "onboarding"); - const runtime = plan.phases.find((phase) => phase.name === "runtime"); - - expect(onboarding?.assertionGroups.map((group) => group.id)).toContain( - "onboarding.base-installed", - ); - expect(runtime?.assertionGroups.map((group) => group.id)).toContain("suite.smoke"); - expect( - runtime?.assertionGroups.flatMap((group) => group.steps.map((step) => step.id)), - ).toContain("runtime.smoke.gateway-health"); - }); - - it("should show timeout and retry policy in plan", () => { - const summary = runScenarioCli(["--scenarios", "ubuntu-repo-cloud-openclaw", "--plan-only"]); - - expect(summary.status, summary.stderr).toBe(0); - expect(summary.stdout).toContain("timeout=30s"); - expect(summary.stdout).toContain("retry=2 on gateway-transient"); - }); - - it("should reject incompatible manifest scenario combination", () => { - const badScenario: ScenarioDefinition = { - id: "bad-platform", - manifestPath: "test/e2e-scenario/manifests/openclaw-nvidia-macos.yaml", - environment: { - platform: "ubuntu-local", - install: "repo-current", - runtime: "docker-running", - onboarding: "cloud-openclaw", - }, - assertionGroups: [], - expectedStateId: "cloud-openclaw-ready", - suiteIds: [], - onboardingAssertionIds: [], - }; - - expect(() => compileRunPlans([badScenario])).toThrow( - /incompatible.*platform|platform.*incompatible/i, - ); - }); - - it("should reject suite filter", () => { - const result = runScenarioCli(["--scenarios", "ubuntu-repo-cloud-openclaw", "--plan-only"], { - E2E_SUITE_FILTER: "smoke", - }); - - expect(result.status).not.toBe(0); - expect(`${result.stdout}${result.stderr}`).toMatch(/E2E_SUITE_FILTER|scenario builders/i); - }); - - it("plan only should work for every canonical scenario ID", () => { - const ids = listScenarios().map((scenario) => scenario.id); - const plans = compileRunPlans(ids); - - expect(plans.map((plan) => plan.scenarioId)).toEqual(ids); - }); -}); diff --git a/test/e2e-scenario/framework-tests/e2e-probes.test.ts b/test/e2e-scenario/framework-tests/e2e-probes.test.ts deleted file mode 100644 index 13f1c25c3fd..00000000000 --- a/test/e2e-scenario/framework-tests/e2e-probes.test.ts +++ /dev/null @@ -1,698 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { describe, it, expect, beforeEach, afterEach } from "vitest"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; -import { - listRegisteredProbes, - lookupProbe, - registerProbe, - resetProbeRegistry, -} from "../scenarios/probes/registry.ts"; -import type { ProbeContext, ProbeOutcome } from "../scenarios/probes/types.ts"; -import { registerBuiltinProbes } from "../scenarios/probes/builtin.ts"; -import { writeProbeEvidence } from "../scenarios/probes/util.ts"; - -const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../.."); - -describe("probe registry", () => { - // The orchestrator side-effect-imports builtin.ts at module load, - // so the registry already contains the built-ins. Each test resets - // and re-registers explicitly so order independence holds. - beforeEach(() => { - resetProbeRegistry(); - }); - - afterEach(() => { - // Restore the production wiring so subsequent test files don't - // see an empty registry (vitest shares module state across files - // within a worker). - resetProbeRegistry(); - registerBuiltinProbes(); - }); - - it("round-trips registerProbe through lookupProbe", () => { - const fn = async (): Promise => ({ status: "passed" }); - registerProbe("myProbe", fn); - expect(lookupProbe("myProbe")).toBe(fn); - }); - - it("lookupProbe returns undefined for an unknown ref", () => { - expect(lookupProbe("nonexistent")).toBeUndefined(); - }); - - it("registerProbe rejects duplicate registration", () => { - const fn = async (): Promise => ({ status: "passed" }); - registerProbe("dup", fn); - expect(() => registerProbe("dup", fn)).toThrow(/already registered/); - }); - - it("registerProbe rejects empty name", () => { - const fn = async (): Promise => ({ status: "passed" }); - expect(() => registerProbe("", fn)).toThrow(/name is required/); - }); - - it("listRegisteredProbes returns names sorted", () => { - registerProbe("zeta", async () => ({ status: "passed" })); - registerProbe("alpha", async () => ({ status: "passed" })); - registerProbe("mu", async () => ({ status: "passed" })); - expect(listRegisteredProbes()).toEqual(["alpha", "mu", "zeta"]); - }); - - it("registerBuiltinProbes is idempotent", () => { - registerBuiltinProbes(); - const first = listRegisteredProbes(); - expect(first).toContain("diagnosticsProbe"); - expect(first).toContain("docsValidationProbe"); - // Calling again must not throw on duplicate names. - expect(() => registerBuiltinProbes()).not.toThrow(); - expect(listRegisteredProbes()).toEqual(first); - }); - - it("registerBuiltinProbes registers security probes", () => { - // shieldsConfig / networkPolicy / injectionBlocked are marked - // `required: true` in scenarios/assertions/registry.ts. The - // orchestrator fails closed when a required probe is missing, - // so registering all three turns the security suites from - // 'silently skipped' into 'actually verified'. - registerBuiltinProbes(); - const registered = listRegisteredProbes(); - expect(registered).toContain("shieldsConfigProbe"); - expect(registered).toContain("networkPolicyProbe"); - expect(registered).toContain("injectionBlockedProbe"); - }); -}); - -describe("probe evidence writer", () => { - it("writes evidence under the context dir and ignores escape paths", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "probe-evidence-root-")); - const contextDir = path.join(tmp, "ctx"); - fs.mkdirSync(contextDir, { recursive: true }); - try { - const insidePath = path.join(contextDir, "nested", "evidence.json"); - const insideCtx: ProbeContext = { - contextDir, - evidencePath: insidePath, - contextEnv: {}, - sandboxName: null, - gatewayUrl: null, - repoRoot: REPO_ROOT, - }; - writeProbeEvidence(insideCtx, { ok: true }); - expect(JSON.parse(fs.readFileSync(insidePath, "utf8"))).toEqual({ ok: true }); - - const outsidePath = path.join(tmp, "escape.json"); - const escapingCtx: ProbeContext = { - ...insideCtx, - evidencePath: outsidePath, - }; - writeProbeEvidence(escapingCtx, { ok: false }); - expect(fs.existsSync(outsidePath)).toBe(false); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); -}); - -// ───────────────────────────────────────────────────────────────────────────── -// diagnosticsProbe — uses a fake `nemoclaw` on PATH so this test runs -// reproducibly without depending on a real nemoclaw install. -// ───────────────────────────────────────────────────────────────────────────── - -function makeProbeCtx(tmp: string, evidenceFile = "diag-evidence.json"): ProbeContext { - // contextDir doubles as the parent of the evidence file when the - // step does not specify an explicit path. Tests pass an explicit - // path here to keep the file under tmp. - return { - contextDir: tmp, - evidencePath: path.join(tmp, evidenceFile), - contextEnv: {}, - sandboxName: null, - gatewayUrl: null, - repoRoot: REPO_ROOT, - }; -} - -function installFakeOnPath(binDir: string, name: string, script: string): { restore: () => void } { - fs.mkdirSync(binDir, { recursive: true }); - fs.writeFileSync(path.join(binDir, name), script, { mode: 0o755 }); - const oldPath = process.env.PATH; - process.env.PATH = `${binDir}:${oldPath ?? ""}`; - return { - restore: () => { - process.env.PATH = oldPath; - }, - }; -} - -describe("diagnosticsProbe", () => { - it("passes when NemoClaw debug quick writes a non-empty archive", async () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "diag-probe-pass-")); - const fake = installFakeOnPath( - path.join(tmp, "bin"), - "nemoclaw", - `#!/usr/bin/env bash -# Stub: locate the --output value and write a small non-empty archive there. -out="" -while [[ "$#" -gt 0 ]]; do - case "$1" in - --output) out="$2"; shift 2 ;; - *) shift ;; - esac -done -[[ -n "$out" ]] || { echo "no --output" >&2; exit 2; } -printf 'fake-archive-bytes' > "$out" -exit 0 -`, - ); - try { - const { diagnosticsProbe } = await import("../scenarios/probes/diagnostics.ts"); - const outcome = await diagnosticsProbe(makeProbeCtx(tmp)); - expect(outcome.status).toBe("passed"); - expect(outcome.message).toMatch(/bundle ok/); - // Evidence JSON must exist and parse. - const ev = JSON.parse(fs.readFileSync(path.join(tmp, "diag-evidence.json"), "utf8")); - expect(ev.exitCode).toBe(0); - expect(ev.archiveSize).toBeGreaterThan(0); - } finally { - fake.restore(); - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("fails when NemoClaw exits nonzero", async () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "diag-probe-fail-")); - const fake = installFakeOnPath( - path.join(tmp, "bin"), - "nemoclaw", - `#!/usr/bin/env bash\necho "boom" >&2\nexit 7\n`, - ); - try { - const { diagnosticsProbe } = await import("../scenarios/probes/diagnostics.ts"); - const outcome = await diagnosticsProbe(makeProbeCtx(tmp)); - expect(outcome.status).toBe("failed"); - expect(outcome.message).toMatch(/exited 7/); - const ev = JSON.parse(fs.readFileSync(path.join(tmp, "diag-evidence.json"), "utf8")); - expect(ev.exitCode).toBe(7); - expect(ev.stderrTail).toContain("boom"); - } finally { - fake.restore(); - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("fails when archive is empty", async () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "diag-probe-empty-")); - const fake = installFakeOnPath( - path.join(tmp, "bin"), - "nemoclaw", - `#!/usr/bin/env bash -out="" -while [[ "$#" -gt 0 ]]; do - case "$1" in --output) out="$2"; shift 2 ;; *) shift ;; esac -done -: > "$out" # zero-byte archive -exit 0 -`, - ); - try { - const { diagnosticsProbe } = await import("../scenarios/probes/diagnostics.ts"); - const outcome = await diagnosticsProbe(makeProbeCtx(tmp)); - expect(outcome.status).toBe("failed"); - expect(outcome.message).toMatch(/empty/); - } finally { - fake.restore(); - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); -}); - -// ───────────────────────────────────────────────────────────────────────────── -// docsValidationProbe — substitutes a fake check-docs.sh by overriding -// the repoRoot in the ProbeContext so the resolved path points at a -// scratch dir we control. -// ───────────────────────────────────────────────────────────────────────────── - -describe("docsValidationProbe", () => { - function setupFakeCheckDocs( - tmp: string, - cliExit: number, - linksExit: number, - ): { ctx: ProbeContext } { - const scriptDir = path.join(tmp, "test/e2e/e2e-cloud-experimental"); - fs.mkdirSync(scriptDir, { recursive: true }); - fs.writeFileSync( - path.join(scriptDir, "check-docs.sh"), - `#!/usr/bin/env bash -case "$1" in - --only-cli) exit ${cliExit} ;; - --only-links) exit ${linksExit} ;; - *) echo "unknown: $*" >&2; exit 99 ;; -esac -`, - { mode: 0o755 }, - ); - return { - ctx: { - contextDir: tmp, - evidencePath: path.join(tmp, "docs-evidence.json"), - contextEnv: {}, - sandboxName: null, - gatewayUrl: null, - repoRoot: tmp, // probe resolves check-docs.sh against this - }, - }; - } - - it("passes when both CLI and links checks exit zero", async () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "docs-probe-pass-")); - try { - const { ctx } = setupFakeCheckDocs(tmp, 0, 0); - const { docsValidationProbe } = await import("../scenarios/probes/docs-validation.ts"); - const outcome = await docsValidationProbe(ctx); - expect(outcome.status).toBe("passed"); - const ev = JSON.parse(fs.readFileSync(ctx.evidencePath, "utf8")); - expect(ev.results).toHaveLength(2); - expect(ev.results[0].phase).toBe("cli-parity"); - expect(ev.results[0].exitCode).toBe(0); - expect(ev.results[1].phase).toBe("links-local"); - expect(ev.results[1].exitCode).toBe(0); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("fails when CLI parity check exits nonzero", async () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "docs-probe-cli-fail-")); - try { - const { ctx } = setupFakeCheckDocs(tmp, 3, 0); - const { docsValidationProbe } = await import("../scenarios/probes/docs-validation.ts"); - const outcome = await docsValidationProbe(ctx); - expect(outcome.status).toBe("failed"); - expect(outcome.message).toMatch(/CLI\/docs parity failed.*exit 3/); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("fails when links check exits nonzero", async () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "docs-probe-links-fail-")); - try { - const { ctx } = setupFakeCheckDocs(tmp, 0, 5); - const { docsValidationProbe } = await import("../scenarios/probes/docs-validation.ts"); - const outcome = await docsValidationProbe(ctx); - expect(outcome.status).toBe("failed"); - expect(outcome.message).toMatch(/markdown link check failed.*exit 5/); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("fails with actionable message when check docs script missing", async () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "docs-probe-missing-")); - try { - const { docsValidationProbe } = await import("../scenarios/probes/docs-validation.ts"); - const ctx: ProbeContext = { - contextDir: tmp, - evidencePath: path.join(tmp, "docs-evidence.json"), - contextEnv: {}, - sandboxName: null, - gatewayUrl: null, - repoRoot: tmp, // no test/e2e/... tree under tmp - }; - const outcome = await docsValidationProbe(ctx); - expect(outcome.status).toBe("failed"); - expect(outcome.message).toMatch(/check-docs\.sh not found/); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); -}); - -// ────────────────────────────────────────────────────────────────────────── -// Security probes — stub `nemoclaw` (host CLI) and `openshell` so the -// canonical sandbox-exec wrapper resolves through the stub. The -// wrapper's openshell-fallback path is exercised because the stub -// does not implement `sandbox ssh-config`. -// ────────────────────────────────────────────────────────────────────────── - -function makeProbeCtxFor( - tmp: string, - sandboxName: string, - contextEnv: Record = {}, -): ProbeContext { - // Write context.env so spawned bash scripts that source the - // wrapper can pick up E2E_SANDBOX_NAME if needed. - const lines = Object.entries({ E2E_SANDBOX_NAME: sandboxName, ...contextEnv }) - .map(([k, v]) => `${k}=${v}`) - .join("\n"); - fs.writeFileSync(path.join(tmp, "context.env"), lines + "\n"); - return { - contextDir: tmp, - evidencePath: path.join(tmp, "probe-evidence.json"), - contextEnv: { E2E_SANDBOX_NAME: sandboxName, ...contextEnv }, - sandboxName, - gatewayUrl: null, - repoRoot: REPO_ROOT, - }; -} - -describe("shieldsConfigProbe", () => { - it("passes when shields status matches expected and perms match state", async () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "shields-probe-pass-")); - const fakeBin = path.join(tmp, "bin"); - fs.mkdirSync(fakeBin); - fs.writeFileSync( - path.join(fakeBin, "nemoclaw"), - `#!/usr/bin/env bash -# nemoclaw shields status -if [[ "$2" == "shields" && "$3" == "status" ]]; then - echo "Shields: DOWN" - exit 0 -fi -exit 99 -`, - { mode: 0o755 }, - ); - fs.writeFileSync( - path.join(fakeBin, "openshell"), - `#!/usr/bin/env bash -# Stub openshell. Reject ssh-config so wrapper falls back to sandbox exec. -# Then implement 'sandbox exec --name -- ' by stripping args -# until '--' and running what's left. -if [[ "$1" == "sandbox" && "$2" == "ssh-config" ]]; then - exit 1 -fi -if [[ "$1" == "sandbox" && "$2" == "exec" ]]; then - shift 2 - while [[ "$#" -gt 0 && "$1" != "--" ]]; do shift; done - shift || true - # The 'stat -c %a %U:%G ' invocation: emit a fake permissions - # line that matches a DOWN-state sandbox config (sandbox-owned). - if [[ "$1" == "stat" ]]; then - echo "644 sandbox:sandbox" - exit 0 - fi - exit 0 -fi -exit 99 -`, - { mode: 0o755 }, - ); - const oldPath = process.env.PATH; - process.env.PATH = `${fakeBin}:${oldPath ?? ""}`; - try { - const { shieldsConfigProbe } = await import("../scenarios/probes/shields-config.ts"); - const ctx = makeProbeCtxFor(tmp, "sb1", { - E2E_AGENT: "openclaw", - E2E_SHIELDS_EXPECTED_STATE: "down", - }); - const outcome = await shieldsConfigProbe(ctx); - expect(outcome.status).toBe("passed"); - expect(outcome.message).toMatch(/shields=down/); - const ev = JSON.parse(fs.readFileSync(ctx.evidencePath, "utf8")); - expect(ev.observed).toBe("down"); - expect(ev.expected).toBe("down"); - expect(ev.permissionsLine).toBe("644 sandbox:sandbox"); - } finally { - process.env.PATH = oldPath; - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("fails when observed state disagrees with expected", async () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "shields-probe-mismatch-")); - const fakeBin = path.join(tmp, "bin"); - fs.mkdirSync(fakeBin); - fs.writeFileSync( - path.join(fakeBin, "nemoclaw"), - `#!/usr/bin/env bash -if [[ "$2" == "shields" && "$3" == "status" ]]; then - echo "Shields: UP" - exit 0 -fi -exit 99 -`, - { mode: 0o755 }, - ); - const oldPath = process.env.PATH; - process.env.PATH = `${fakeBin}:${oldPath ?? ""}`; - try { - const { shieldsConfigProbe } = await import("../scenarios/probes/shields-config.ts"); - const ctx = makeProbeCtxFor(tmp, "sb1", { - E2E_AGENT: "openclaw", - E2E_SHIELDS_EXPECTED_STATE: "down", - }); - const outcome = await shieldsConfigProbe(ctx); - expect(outcome.status).toBe("failed"); - expect(outcome.message).toMatch(/expected shields 'down', observed 'up'/); - } finally { - process.env.PATH = oldPath; - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("fails when permissions do not match observed state", async () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "shields-probe-perms-")); - const fakeBin = path.join(tmp, "bin"); - fs.mkdirSync(fakeBin); - fs.writeFileSync( - path.join(fakeBin, "nemoclaw"), - `#!/usr/bin/env bash -if [[ "$2" == "shields" && "$3" == "status" ]]; then - # Shields claim UP, but the stub openshell will report sandbox-owned - # perms below — a mismatch the probe must catch. - echo "Shields: UP" - exit 0 -fi -exit 99 -`, - { mode: 0o755 }, - ); - fs.writeFileSync( - path.join(fakeBin, "openshell"), - `#!/usr/bin/env bash -if [[ "$1" == "sandbox" && "$2" == "ssh-config" ]]; then exit 1; fi -if [[ "$1" == "sandbox" && "$2" == "exec" ]]; then - shift 2 - while [[ "$#" -gt 0 && "$1" != "--" ]]; do shift; done - shift || true - # Sandbox-owned perms: would pass for DOWN, must FAIL for UP. - echo "644 sandbox:sandbox" - exit 0 -fi -exit 99 -`, - { mode: 0o755 }, - ); - const oldPath = process.env.PATH; - process.env.PATH = `${fakeBin}:${oldPath ?? ""}`; - try { - const { shieldsConfigProbe } = await import("../scenarios/probes/shields-config.ts"); - // Don't declare expected state — the probe should still fail on - // perms-vs-observed mismatch alone. - const ctx = makeProbeCtxFor(tmp, "sb1", { E2E_AGENT: "openclaw" }); - const outcome = await shieldsConfigProbe(ctx); - expect(outcome.status).toBe("failed"); - expect(outcome.message).toMatch(/shields are 'up' but .* permissions are/); - } finally { - process.env.PATH = oldPath; - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); -}); - -describe("networkPolicyProbe", () => { - function fakeOpenshellEmittingHttpStatus( - binDir: string, - httpStatus: string, - curlExitCode: number = 0, - ): void { - fs.mkdirSync(binDir, { recursive: true }); - fs.writeFileSync( - path.join(binDir, "openshell"), - `#!/usr/bin/env bash -# Opt out of ssh-config; force wrapper to use 'sandbox exec' fallback. -if [[ "$1" == "sandbox" && "$2" == "ssh-config" ]]; then exit 1; fi -if [[ "$1" == "sandbox" && "$2" == "exec" ]]; then - shift 2 - while [[ "$#" -gt 0 && "$1" != "--" ]]; do shift; done - shift || true - # We're being asked to run curl inside the sandbox. Emit the test's - # chosen status to stdout (mirrors curl -w '%{http_code}') and exit - # with the test's chosen curl exit code. - printf '%s' "${httpStatus}" - exit ${curlExitCode} -fi -exit 99 -`, - { mode: 0o755 }, - ); - } - - it("passes when blocked URL returns 403", async () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "netpolicy-probe-403-")); - fakeOpenshellEmittingHttpStatus(path.join(tmp, "bin"), "403", 0); - const oldPath = process.env.PATH; - process.env.PATH = `${path.join(tmp, "bin")}:${oldPath ?? ""}`; - try { - const { networkPolicyProbe } = await import("../scenarios/probes/network-policy.ts"); - const ctx = makeProbeCtxFor(tmp, "sb1"); - const outcome = await networkPolicyProbe(ctx); - expect(outcome.status).toBe("passed"); - expect(outcome.message).toMatch(/blocked .*http_code=403/); - } finally { - process.env.PATH = oldPath; - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("passes when curl exits nonzero and no HTTP response", async () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "netpolicy-probe-conn-")); - // curl exit 7 = couldn't connect; status '000' = no HTTP response. - fakeOpenshellEmittingHttpStatus(path.join(tmp, "bin"), "000", 7); - const oldPath = process.env.PATH; - process.env.PATH = `${path.join(tmp, "bin")}:${oldPath ?? ""}`; - try { - const { networkPolicyProbe } = await import("../scenarios/probes/network-policy.ts"); - const ctx = makeProbeCtxFor(tmp, "sb1"); - const outcome = await networkPolicyProbe(ctx); - expect(outcome.status).toBe("passed"); - expect(outcome.message).toMatch(/curl exit 7/); - } finally { - process.env.PATH = oldPath; - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("fails when blocked URL returns 200", async () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "netpolicy-probe-200-")); - fakeOpenshellEmittingHttpStatus(path.join(tmp, "bin"), "200", 0); - const oldPath = process.env.PATH; - process.env.PATH = `${path.join(tmp, "bin")}:${oldPath ?? ""}`; - try { - const { networkPolicyProbe } = await import("../scenarios/probes/network-policy.ts"); - const ctx = makeProbeCtxFor(tmp, "sb1"); - const outcome = await networkPolicyProbe(ctx); - expect(outcome.status).toBe("failed"); - expect(outcome.message).toMatch(/reachable from sandbox.*http_code=200/); - } finally { - process.env.PATH = oldPath; - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("fails when blocked URL returns 401 indicating policy bypass", async () => { - // 401 means the request reached upstream auth, NOT that gateway - // dropped it. The probe must classify this as a policy bypass. - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "netpolicy-probe-401-")); - fakeOpenshellEmittingHttpStatus(path.join(tmp, "bin"), "401", 0); - const oldPath = process.env.PATH; - process.env.PATH = `${path.join(tmp, "bin")}:${oldPath ?? ""}`; - try { - const { networkPolicyProbe } = await import("../scenarios/probes/network-policy.ts"); - const ctx = makeProbeCtxFor(tmp, "sb1"); - const outcome = await networkPolicyProbe(ctx); - expect(outcome.status).toBe("failed"); - expect(outcome.message).toMatch(/reachable from sandbox.*http_code=401/); - } finally { - process.env.PATH = oldPath; - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); -}); - -describe("injectionBlockedProbe", () => { - // For the injection probe we need a stub openshell that simulates a - // sandbox shell honestly: pre-clean, echo back stdin, and respond - // SAFE/EXPLOITED based on whether the marker file exists. We give - // each test its own tmp dir and stub script. - function setupInjectionStub(tmp: string, exploited: boolean): { restore: () => void } { - const binDir = path.join(tmp, "bin"); - fs.mkdirSync(binDir, { recursive: true }); - // Use a state file in tmp to track whether the 'exploit' branch - // should claim the marker exists. - const stateFile = path.join(tmp, "exploit.state"); - fs.writeFileSync(stateFile, exploited ? "yes" : "no"); - fs.writeFileSync( - path.join(binDir, "openshell"), - `#!/usr/bin/env bash -state=\$(cat "${stateFile}" 2>/dev/null || echo no) -if [[ "\$1" == "sandbox" && "\$2" == "ssh-config" ]]; then exit 1; fi -if [[ "\$1" == "sandbox" && "\$2" == "exec" ]]; then - shift 2 - while [[ "\$#" -gt 0 && "\$1" != "--" ]]; do shift; done - shift || true - # Recognize the three operations the probe issues: - # 1. sh -c 'rm -f ' — cleanup; always succeeds - # 2. sh -c 'MSG=\$(cat); printf %s\\n "\$MSG"' — echo back stdin - # 3. sh -c 'test -f && echo EXPLOITED || echo SAFE' - cmd="\$*" - case "\$cmd" in - *"MSG="*"printf"*) - cat - ;; - *"test -f"*"EXPLOITED"*"SAFE"*) - if [[ "\$state" == "yes" ]]; then echo EXPLOITED; else echo SAFE; fi - ;; - *"rm -f"*) - : - ;; - *) - echo "unrecognized cmd: \$cmd" >&2 - exit 64 - ;; - esac - exit 0 -fi -exit 99 -`, - { mode: 0o755 }, - ); - const oldPath = process.env.PATH; - process.env.PATH = `${binDir}:${oldPath ?? ""}`; - return { - restore: () => { - process.env.PATH = oldPath; - }, - }; - } - - it("passes when the payload is preserved and the marker is absent", async () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "inj-probe-pass-")); - const stub = setupInjectionStub(tmp, false); - try { - const { injectionBlockedProbe } = await import("../scenarios/probes/injection-blocked.ts"); - const ctx = makeProbeCtxFor(tmp, "sb1"); - const outcome = await injectionBlockedProbe(ctx); - expect(outcome.status).toBe("passed"); - const ev = JSON.parse(fs.readFileSync(ctx.evidencePath, "utf8")); - expect(ev.payloadPreservedLiterally).toBe(true); - expect(ev.markerAbsent).toBe(true); - } finally { - stub.restore(); - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("fails when marker file creation indicates command substitution executed", async () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "inj-probe-fail-")); - const stub = setupInjectionStub(tmp, true); - try { - const { injectionBlockedProbe } = await import("../scenarios/probes/injection-blocked.ts"); - const ctx = makeProbeCtxFor(tmp, "sb1"); - const outcome = await injectionBlockedProbe(ctx); - expect(outcome.status).toBe("failed"); - expect(outcome.message).toMatch(/marker file .* present/); - expect(outcome.message).toMatch(/command substitution executed/); - const ev = JSON.parse(fs.readFileSync(ctx.evidencePath, "utf8")); - expect(ev.markerAbsent).toBe(false); - } finally { - stub.restore(); - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); -}); diff --git a/test/e2e-scenario/framework-tests/e2e-redaction-entry.test.ts b/test/e2e-scenario/framework-tests/e2e-redaction-entry.test.ts index 03d5a766b93..28505531997 100644 --- a/test/e2e-scenario/framework-tests/e2e-redaction-entry.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-redaction-entry.test.ts @@ -18,7 +18,7 @@ import { describe, expect, it } from "vitest"; import { SecretStore } from "../framework/secrets.ts"; -import { redactString } from "../scenarios/orchestrators/redaction.ts"; +import { redactString } from "../framework/redaction.ts"; describe("framework redaction entry point", () => { it("redacts explicit values with [REDACTED] and canonical shapes with ", () => { diff --git a/test/e2e-scenario/framework-tests/e2e-redaction-parity.test.ts b/test/e2e-scenario/framework-tests/e2e-redaction-parity.test.ts index 3451759bf57..8c90f313520 100644 --- a/test/e2e-scenario/framework-tests/e2e-redaction-parity.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-redaction-parity.test.ts @@ -3,7 +3,7 @@ /** * Parity test: the framework's local secret-pattern set - * (test/e2e-scenario/scenarios/orchestrators/redaction.ts) must stay in + * (test/e2e-scenario/framework/redaction.ts) must stay in * lockstep with the canonical product source * (src/lib/security/secret-patterns.ts). * @@ -24,7 +24,7 @@ import { describe, expect, it } from "vitest"; import { CONTEXT_PATTERNS as FRAMEWORK_CONTEXT_PATTERNS, TOKEN_PREFIX_PATTERNS as FRAMEWORK_TOKEN_PREFIX_PATTERNS, -} from "../scenarios/orchestrators/redaction.ts"; +} from "../framework/redaction.ts"; import { CONTEXT_PATTERNS as PRODUCT_CONTEXT_PATTERNS, TOKEN_PREFIX_PATTERNS as PRODUCT_TOKEN_PREFIX_PATTERNS, diff --git a/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts b/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts index 78eb216d53d..b46c1e88d36 100644 --- a/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts @@ -6,22 +6,13 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import { scenario } from "../scenarios/builder.ts"; -import { listScenarios } from "../scenarios/registry.ts"; -import { buildLiveScenarioMatrix, buildScenarioMatrix } from "../scenarios/run.ts"; +import { buildLiveScenarioMatrix } from "../scenarios/run.ts"; import { resolveRunnerForScenario } from "../scenarios/runner-routing.ts"; const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); const RUN_SCENARIOS = path.join(REPO_ROOT, "test/e2e-scenario/scenarios/run.ts"); const TSX = path.join(REPO_ROOT, "node_modules/.bin/tsx"); -function runEmitMatrix() { - return spawnSync(TSX, [RUN_SCENARIOS, "--emit-matrix"], { - cwd: REPO_ROOT, - encoding: "utf8", - timeout: Number(process.env.E2E_SPAWN_TIMEOUT_MS ?? 60_000), - }); -} - function runEmitLiveMatrix(args: string[] = []) { return spawnSync(TSX, [RUN_SCENARIOS, "--emit-live-matrix", ...args], { cwd: REPO_ROOT, @@ -30,32 +21,7 @@ function runEmitLiveMatrix(args: string[] = []) { }); } -describe("typed scenario matrix", () => { - it("emits one matrix entry per registered scenario", () => { - const matrix = buildScenarioMatrix(); - const ids = listScenarios().map((s) => s.id); - expect(matrix.map((entry) => entry.id).sort()).toEqual([...ids].sort()); - }); - - it("resolves a runner label for every scenario", () => { - const matrix = buildScenarioMatrix(); - expect(matrix.length).toBeGreaterThan(0); - for (const entry of matrix) { - expect(entry.runner, `runner missing for ${entry.id}`).toMatch(/[A-Za-z0-9-]/); - expect(entry.label, `label missing for ${entry.id}`).toContain(entry.id); - } - }); - - it("routes platforms to their canonical runners", () => { - const byId = new Map(buildScenarioMatrix().map((entry) => [entry.id, entry])); - expect(byId.get("ubuntu-repo-cloud-openclaw")?.runner).toBe("ubuntu-latest"); - expect(byId.get("macos-repo-cloud-openclaw")?.runner).toBe("macos-26"); - expect(byId.get("wsl-repo-cloud-openclaw")?.runner).toBe("windows-latest"); - expect(byId.get("gpu-repo-local-ollama-openclaw")?.runner).toBe( - "linux-amd64-gpu-rtxpro6000-latest-1", - ); - }); - +describe("live Vitest scenario matrix", () => { it("honors an explicit runs-on: