From fa31049ce8ce793fd2c9a3b2e6f83c2ef4d88f2e Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 18 May 2026 11:01:01 -0400 Subject: [PATCH 1/5] ci(e2e): surface scenario report in logs --- .github/workflows/e2e-scenarios.yaml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-scenarios.yaml b/.github/workflows/e2e-scenarios.yaml index 5fd1e0cf7ad..67e8956100e 100644 --- a/.github/workflows/e2e-scenarios.yaml +++ b/.github/workflows/e2e-scenarios.yaml @@ -88,8 +88,13 @@ jobs: run: | mkdir -p .e2e bash test/e2e/runtime/coverage-report.sh > .e2e/coverage.md - echo '## E2E scenario coverage' >> "$GITHUB_STEP_SUMMARY" - cat .e2e/coverage.md >> "$GITHUB_STEP_SUMMARY" + { + echo '# E2E Scenario Report' + echo '' + echo '**Scenario:** `${{ github.event.inputs.scenario }}`' + echo '' + cat .e2e/coverage.md + } | tee -a "$GITHUB_STEP_SUMMARY" - name: Run scenario if: ${{ !startsWith(github.event.inputs.scenario, 'wsl-') }} @@ -98,6 +103,18 @@ jobs: E2E_SUITE_FILTER: ${{ github.event.inputs.suite_filter }} run: | bash test/e2e/runtime/run-scenario.sh "${{ github.event.inputs.scenario }}" + { + echo '' + echo '## Scenario execution result' + echo '' + echo '- Scenario `${{ github.event.inputs.scenario }}` completed successfully.' + if grep -R '^SKIP:' .e2e test/e2e/logs >/tmp/e2e-skips.txt 2>/dev/null; then + echo '' + echo '### Runtime skips observed' + echo '' + sed 's/^/- `/' /tmp/e2e-skips.txt | sed 's/$/`/' + fi + } | tee -a "$GITHUB_STEP_SUMMARY" - name: Resolve workspace paths for WSL if: startsWith(github.event.inputs.scenario, 'wsl-') From 89d028e48881a84f6c313c1d78227fc7e9aad849 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 20 May 2026 13:53:29 -0400 Subject: [PATCH 2/5] ci(e2e): avoid scenario shell interpolation --- .github/workflows/e2e-scenarios.yaml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e-scenarios.yaml b/.github/workflows/e2e-scenarios.yaml index 67e8956100e..cc3d93f7059 100644 --- a/.github/workflows/e2e-scenarios.yaml +++ b/.github/workflows/e2e-scenarios.yaml @@ -85,13 +85,17 @@ jobs: - name: Render coverage report if: ${{ !startsWith(github.event.inputs.scenario, 'wsl-') }} + env: + SCENARIO: ${{ github.event.inputs.scenario }} run: | mkdir -p .e2e bash test/e2e/runtime/coverage-report.sh > .e2e/coverage.md { echo '# E2E Scenario Report' echo '' - echo '**Scenario:** `${{ github.event.inputs.scenario }}`' + # Keep workflow_dispatch input in an env var so untrusted scenario text + # is data, not YAML-interpolated shell source. + printf '**Scenario:** `%s`\n' "$SCENARIO" echo '' cat .e2e/coverage.md } | tee -a "$GITHUB_STEP_SUMMARY" @@ -101,13 +105,16 @@ jobs: env: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} E2E_SUITE_FILTER: ${{ github.event.inputs.suite_filter }} + SCENARIO: ${{ github.event.inputs.scenario }} run: | - bash test/e2e/runtime/run-scenario.sh "${{ github.event.inputs.scenario }}" + # Keep workflow_dispatch input in an env var so untrusted scenario text + # is data, not YAML-interpolated shell source. + bash test/e2e/runtime/run-scenario.sh "$SCENARIO" { echo '' echo '## Scenario execution result' echo '' - echo '- Scenario `${{ github.event.inputs.scenario }}` completed successfully.' + printf -- '- Scenario `%s` completed successfully.\n' "$SCENARIO" if grep -R '^SKIP:' .e2e test/e2e/logs >/tmp/e2e-skips.txt 2>/dev/null; then echo '' echo '### Runtime skips observed' From 4f3e260ac3ac6be606c688cd251ed71ffaaff027 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 20 May 2026 14:26:32 -0400 Subject: [PATCH 3/5] ci(e2e): harden scenario workflow inputs --- .github/workflows/e2e-scenarios.yaml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/e2e-scenarios.yaml b/.github/workflows/e2e-scenarios.yaml index cc3d93f7059..92e6c085792 100644 --- a/.github/workflows/e2e-scenarios.yaml +++ b/.github/workflows/e2e-scenarios.yaml @@ -74,7 +74,7 @@ jobs: - name: Set up Node if: ${{ !startsWith(github.event.inputs.scenario, 'wsl-') }} - uses: actions/setup-node@v6 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: node-version: 22 cache: npm @@ -214,15 +214,17 @@ jobs: env: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} E2E_SUITE_FILTER: ${{ github.event.inputs.suite_filter }} + SCENARIO: ${{ github.event.inputs.scenario }} run: | - $script = @" + $env:WSLENV = "NVIDIA_API_KEY:E2E_SUITE_FILTER:NEMOCLAW_RECREATE_SANDBOX:SCENARIO:WSL_WORKDIR" + $script = @' set -euo pipefail - cd '$env:WSL_WORKDIR' - export NVIDIA_API_KEY='$env:NVIDIA_API_KEY' - export E2E_SUITE_FILTER='$env:E2E_SUITE_FILTER' - export NEMOCLAW_RECREATE_SANDBOX='$env:NEMOCLAW_RECREATE_SANDBOX' - bash test/e2e/runtime/run-scenario.sh '${{ github.event.inputs.scenario }}' - "@ + cd "$WSL_WORKDIR" + export NVIDIA_API_KEY + export E2E_SUITE_FILTER + export NEMOCLAW_RECREATE_SANDBOX + bash test/e2e/runtime/run-scenario.sh "$SCENARIO" + '@ $tmp = "$env:RUNNER_TEMP\wsl-step.sh" [IO.File]::WriteAllText($tmp, ($script -replace "`r",""), (New-Object System.Text.UTF8Encoding $false)) $wslTmp = wsl -d $env:WSL_DISTRO -- wslpath -u ($tmp -replace '\\','/') @@ -245,7 +247,7 @@ jobs: - name: Upload scenario artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: e2e-scenario-${{ github.event.inputs.scenario }} path: | From 04c828a9c319dc96e1654270e0d48a8e29a4a86e Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 20 May 2026 15:45:45 -0400 Subject: [PATCH 4/5] test(e2e): allow pinned upload artifact action --- .../e2e/scenario-framework-tests/e2e-scenarios-workflow.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/scenario-framework-tests/e2e-scenarios-workflow.test.ts b/test/e2e/scenario-framework-tests/e2e-scenarios-workflow.test.ts index c3cd09420af..43f90afa605 100644 --- a/test/e2e/scenario-framework-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e/scenario-framework-tests/e2e-scenarios-workflow.test.ts @@ -51,7 +51,7 @@ function namedStep(workflow: AnyRecord, jobId: string, stepName: string): Workfl function uploadArtifactStep(workflow: AnyRecord, jobId: string, stepName: string): WorkflowStep { const step = namedStep(workflow, jobId, stepName); - expect(step.uses).toBe("actions/upload-artifact@v4"); + expect(step.uses).toMatch(/^actions\/upload-artifact@(?:v4|[a-f0-9]{40})$/); return step; } From cc3e2f37e1f731325b84fec0d8e79c21e76c0397 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 20 May 2026 15:58:49 -0400 Subject: [PATCH 5/5] ci(e2e): summarize failed scenario runs --- .github/workflows/e2e-scenarios.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e-scenarios.yaml b/.github/workflows/e2e-scenarios.yaml index 92e6c085792..1cc20b41710 100644 --- a/.github/workflows/e2e-scenarios.yaml +++ b/.github/workflows/e2e-scenarios.yaml @@ -109,12 +109,19 @@ jobs: run: | # Keep workflow_dispatch input in an env var so untrusted scenario text # is data, not YAML-interpolated shell source. + set +e bash test/e2e/runtime/run-scenario.sh "$SCENARIO" + rc=$? + set -e { echo '' echo '## Scenario execution result' echo '' - printf -- '- Scenario `%s` completed successfully.\n' "$SCENARIO" + if [ "$rc" -eq 0 ]; then + printf -- '- Scenario `%s` completed successfully.\n' "$SCENARIO" + else + printf -- '- Scenario `%s` failed with exit code `%s`.\n' "$SCENARIO" "$rc" + fi if grep -R '^SKIP:' .e2e test/e2e/logs >/tmp/e2e-skips.txt 2>/dev/null; then echo '' echo '### Runtime skips observed' @@ -122,6 +129,7 @@ jobs: sed 's/^/- `/' /tmp/e2e-skips.txt | sed 's/$/`/' fi } | tee -a "$GITHUB_STEP_SUMMARY" + exit "$rc" - name: Resolve workspace paths for WSL if: startsWith(github.event.inputs.scenario, 'wsl-')