diff --git a/.github/workflows/regression-e2e.yaml b/.github/workflows/regression-e2e.yaml index 5687c6da718..6058f8cba7a 100644 --- a/.github/workflows/regression-e2e.yaml +++ b/.github/workflows/regression-e2e.yaml @@ -148,14 +148,16 @@ jobs: needs.select_regression_jobs.outputs.gateway == 'true' runs-on: ubuntu-latest timeout-minutes: 20 + permissions: + contents: read steps: - name: Checkout uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - - name: Setup Node - uses: actions/setup-node@v6 with: - node-version: "22" + persist-credentials: false + + - name: Prepare E2E workspace + uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 - name: Run gateway health-honesty E2E test env: @@ -187,9 +189,18 @@ jobs: needs.select_regression_jobs.outputs.openshell_version_pin == 'true' runs-on: ubuntu-latest timeout-minutes: 15 + permissions: + contents: read steps: - name: Checkout uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Prepare E2E workspace + uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 + with: + build-cli: "false" - name: Run OpenShell version-pin E2E test env: @@ -218,20 +229,22 @@ jobs: needs.select_regression_jobs.outputs.gateway_drift_preflight == 'true' runs-on: ubuntu-latest timeout-minutes: 15 + permissions: + contents: read steps: - name: Checkout uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - - name: Setup Node - uses: actions/setup-node@v6 with: - node-version: "22" + persist-credentials: false + + - name: Prepare E2E workspace + uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 - name: Run gateway drift preflight E2E test env: NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" - run: npx vitest run --project cli test/gateway-drift-preflight.test.ts --silent=false --reporter=default + run: npx vitest run --project integration test/gateway-drift-preflight.test.ts --silent=false --reporter=default # ── Model Router provider-routed inference E2E ───────────────── # Coverage guard for #3255. Model Router onboard must generate a routed @@ -244,9 +257,16 @@ jobs: needs.select_regression_jobs.outputs.model_router_provider_routed_inference == 'true' runs-on: ubuntu-latest timeout-minutes: 45 + permissions: + contents: read steps: - name: Checkout uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Prepare E2E workspace + uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 - name: Run Model Router provider-routed inference E2E test env: diff --git a/package.json b/package.json index cfc1ef92d2e..8555ed054e5 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "test:package": "npm run clean:cli && npm --prefix nemoclaw run clean && npm run build:cli && npm --prefix nemoclaw run build && vitest run --project package-contract", "test:coverage:cli": "npm run clean:cli && npm run build:cli && tsx scripts/check-dist-sourcemaps.ts dist && vitest run --project cli --project integration --coverage --coverage.reporter=text-summary --coverage.reporter=json-summary --coverage.reportsDirectory=coverage/cli --coverage.include=\"bin/**/*.js\" --coverage.include=\"src/**/*.ts\" --coverage.exclude=\"test/**/*.js\" --coverage.exclude=\"test/**/*.ts\" && tsx scripts/check-coverage-ratchet.ts coverage/cli/coverage-summary.json ci/coverage-threshold-cli.json \"CLI coverage\"", "test:coverage:plugin": "vitest run --project plugin --coverage --coverage.reporter=text-summary --coverage.reporter=json-summary --coverage.reportsDirectory=coverage/plugin --coverage.include=\"nemoclaw/src/**/*.ts\" --coverage.include=\"nemoclaw/src/**/*.cts\" --coverage.exclude=\"**/*.test.ts\" && tsx scripts/check-coverage-ratchet.ts coverage/plugin/coverage-summary.json ci/coverage-threshold-plugin.json \"Plugin coverage\"", - "test:live-e2e": "NEMOCLAW_RUN_LIVE_E2E=1 vitest run --project e2e-live", + "test:live-e2e": "npm run clean:cli && npm run build:cli && NEMOCLAW_RUN_LIVE_E2E=1 vitest run --project e2e-live", "test:imports:check": "tsx scripts/checks/no-test-dist-imports.ts", "test:projects:check": "tsx scripts/checks/vitest-project-overlap.ts", "test:titles:check": "tsx scripts/checks/test-title-style.ts", diff --git a/test/e2e/docs/MIGRATION.md b/test/e2e/docs/MIGRATION.md index bfb0e96ab1e..c5d53fcddbe 100644 --- a/test/e2e/docs/MIGRATION.md +++ b/test/e2e/docs/MIGRATION.md @@ -103,8 +103,11 @@ npx tsx test/e2e/registry/run.ts --emit-live-matrix --targets ubuntu-repo-cloud- npx vitest run --project e2e-support --silent=false --reporter=default # Opt-in live E2E targets -npm run build:cli -NEMOCLAW_RUN_LIVE_E2E=1 npx vitest run --project e2e-live --silent=false --reporter=default +npm run test:live-e2e -- --silent=false --reporter=default ``` +The aggregate live command rebuilds the CLI before Vitest starts and runs live +test files serially. Whole-test retries stay disabled because live targets +mutate shared host, Docker, gateway, and sandbox state. + The old `--emit-matrix` and `--plan-only` interfaces are retired. diff --git a/test/e2e/docs/README.md b/test/e2e/docs/README.md index f4f155f0334..4fe3ef0d65e 100644 --- a/test/e2e/docs/README.md +++ b/test/e2e/docs/README.md @@ -72,15 +72,17 @@ npx tsx test/e2e/registry/run.ts --emit-live-matrix --targets ubuntu-repo-cloud- npx vitest run --project e2e-support --silent=false --reporter=default # Opt-in live E2E targets -npm run build:cli -NEMOCLAW_RUN_LIVE_E2E=1 npx vitest run --project e2e-live --silent=false --reporter=default +npm run test:live-e2e -- --silent=false --reporter=default ``` -Live E2E projects do not retry an entire failed test. These tests mutate host, -Docker, gateway, and sandbox state, so re-entering one on the same runner can -replace the original failure with stale-lock, storage-exhaustion, or ownership -noise. A target may retry a transient operation only inside its own cleanup -boundary. Retry a full target by starting a fresh workflow run and runner. +The aggregate live command rebuilds the CLI before Vitest starts and runs live +test files serially. +Live E2E projects do not retry an entire failed test. +These tests mutate host, Docker, gateway, and sandbox state, so re-entering one +on the same runner can replace the original failure with stale-lock, +storage-exhaustion, or ownership noise. A target may retry a transient operation +only inside its own cleanup boundary. +Retry a full target by starting a fresh workflow run and runner. The retired `--emit-matrix` and `--plan-only` paths must not be reintroduced. diff --git a/test/e2e/support/e2e-live-project-config.test.ts b/test/e2e/support/e2e-live-project-config.test.ts index f9de3bc564f..5b80cc522a5 100644 --- a/test/e2e/support/e2e-live-project-config.test.ts +++ b/test/e2e/support/e2e-live-project-config.test.ts @@ -1,6 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawnSync } from "node:child_process"; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; + import { describe, expect, it } from "vitest"; import config from "../../../vitest.config.ts"; import { readYaml, type WorkflowStep } from "../../helpers/e2e-workflow-contract.ts"; @@ -12,6 +17,8 @@ import { interface ProjectConfig { test?: { + env?: Record; + fileParallelism?: boolean; name?: string; include?: string[]; retry?: number; @@ -20,6 +27,7 @@ interface ProjectConfig { interface RootConfig { test?: { + env?: Record; projects?: ProjectConfig[]; }; } @@ -33,6 +41,16 @@ const INSTALLER_INTEGRATION_TESTS = [ ]; const LIVE_E2E_TARGET_TESTS = ["test/e2e/live/**/*.test.ts"]; const BRANCH_VALIDATION_E2E_TESTS = ["test/e2e/brev-e2e.test.ts"]; +const DRIFT_PREFLIGHT_BYPASS = "NEMOCLAW_DISABLE_GATEWAY_DRIFT_PREFLIGHT"; +const NON_LIVE_PROJECTS = [ + "cli", + "integration", + "installer-integration", + "package-contract", + "plugin", + "e2e-support", +]; +const LIVE_PROJECTS = ["e2e-live", "e2e-branch-validation"]; type BranchValidationWorkflow = { jobs?: { @@ -90,13 +108,76 @@ describe("gated E2E Vitest projects", () => { expect(shouldRunBranchValidationE2E({ NEMOCLAW_RUN_BRANCH_VALIDATION_E2E: "1" })).toBe(true); }); - it("keeps both stateful E2E projects single-shot", () => { + it("keeps stateful E2E retries disabled and aggregate live files serial (#6692)", () => { expect(projectConfig("cli").test?.retry).toBeUndefined(); expect(projectConfig("e2e-support").test?.retry).toBeUndefined(); expect(projectConfig("e2e-live").test?.retry).toBe(0); + expect(projectConfig("e2e-live").test?.fileParallelism).toBe(false); expect(projectConfig("e2e-branch-validation").test?.retry).toBe(0); }); + it("keeps the drift-preflight bypass out of live projects (#6692)", () => { + expect((config as RootConfig).test?.env?.[DRIFT_PREFLIGHT_BYPASS]).toBeUndefined(); + for (const name of NON_LIVE_PROJECTS) { + expect(projectConfig(name).test?.env?.[DRIFT_PREFLIGHT_BYPASS], name).toBe("1"); + } + for (const name of LIVE_PROJECTS) { + expect(projectConfig(name).test?.env?.[DRIFT_PREFLIGHT_BYPASS], name).toBeUndefined(); + } + }); + + it("cleans and rebuilds the CLI before aggregate live E2E execution (#6692)", () => { + const npmCli = process.env.npm_execpath ?? ""; + expect(npmCli).not.toBe(""); + + const fixtureRoot = mkdtempSync(join(tmpdir(), "nemoclaw-live-e2e-script-")); + const fakeBin = join(fixtureRoot, "bin"); + const commandLog = join(fixtureRoot, "commands.log"); + const scriptShell = join(fixtureRoot, "script-shell"); + const npmStub = join(fakeBin, "npm"); + const vitestStub = join(fakeBin, "vitest"); + + try { + mkdirSync(fakeBin); + writeFileSync( + scriptShell, + `#!/bin/sh\nPATH="$FAKE_BIN:${dirname(process.execPath)}:/usr/bin:/bin"\nexport PATH\nexec /bin/sh "$@"\n`, + { mode: 0o755 }, + ); + writeFileSync(npmStub, '#!/bin/sh\nprintf \'npm %s\\n\' "$*" >> "$COMMAND_LOG"\n', { + mode: 0o755, + }); + writeFileSync( + vitestStub, + '#!/bin/sh\nprintf \'vitest %s | live=%s\\n\' "$*" "$NEMOCLAW_RUN_LIVE_E2E" >> "$COMMAND_LOG"\n', + { mode: 0o755 }, + ); + chmodSync(scriptShell, 0o755); + chmodSync(npmStub, 0o755); + chmodSync(vitestStub, 0o755); + + const result = spawnSync(process.execPath, [npmCli, "run", "test:live-e2e"], { + cwd: process.cwd(), + encoding: "utf8", + env: { + ...process.env, + COMMAND_LOG: commandLog, + FAKE_BIN: fakeBin, + npm_config_script_shell: scriptShell, + }, + }); + + expect(result.status, result.stderr || result.stdout).toBe(0); + expect(readFileSync(commandLog, "utf8").trim().split("\n")).toEqual([ + "npm run clean:cli", + "npm run build:cli", + "vitest run --project e2e-live | live=1", + ]); + } finally { + rmSync(fixtureRoot, { force: true, recursive: true }); + } + }); + it("sets the branch-validation sentinel in the reusable workflow live E2E step", () => { const workflow = readYaml( ".github/workflows/e2e-branch-validation.yaml", diff --git a/test/regression-e2e-workflow.test.ts b/test/regression-e2e-workflow.test.ts index ed7b1179dd5..8214f7c21d2 100644 --- a/test/regression-e2e-workflow.test.ts +++ b/test/regression-e2e-workflow.test.ts @@ -3,6 +3,10 @@ import { describe, expect, it } from "vitest"; +import { + PREPARE_E2E_ACTION, + PREPARE_E2E_STEP, +} from "../tools/e2e/prepare-e2e-workflow-boundary.mts"; import { readYaml, type WorkflowStep } from "./helpers/e2e-workflow-contract"; type RegressionWorkflow = { @@ -26,6 +30,16 @@ type RegressionWorkflow = { }; const FULL_SHA_ACTION = /@[0-9a-f]{40}$/i; +const PREPARED_VITEST_JOBS = [ + ["gateway-health-honest-e2e", "Run gateway health-honesty E2E test", undefined], + ["openshell-version-pin-e2e", "Run OpenShell version-pin E2E test", { "build-cli": "false" }], + ["gateway-drift-preflight-e2e", "Run gateway drift preflight E2E test", undefined], + [ + "model-router-provider-routed-inference-e2e", + "Run Model Router provider-routed inference E2E test", + undefined, + ], +] as const; describe("Regression E2E workflow contract", () => { const workflow = readYaml(".github/workflows/regression-e2e.yaml"); @@ -72,6 +86,43 @@ describe("Regression E2E workflow contract", () => { expect(runStep?.env?.NVIDIA_INFERENCE_API_KEY).toBeUndefined(); }); + it.each( + PREPARED_VITEST_JOBS, + )("prepares %s before invoking Vitest (#6692)", (jobName, runStepName, prepareInputs) => { + const job = workflow.jobs?.[jobName]; + const steps = job?.steps ?? []; + const checkoutIndex = steps.findIndex((step) => step.uses?.startsWith("actions/checkout@")); + const prepareIndex = steps.findIndex((step) => step.name === PREPARE_E2E_STEP); + const runIndex = steps.findIndex((step) => step.name === runStepName); + const checkout = steps[checkoutIndex]; + const prepare = steps[prepareIndex]; + + expect(job?.permissions).toEqual({ contents: "read" }); + expect(checkout?.uses).toMatch(FULL_SHA_ACTION); + expect(checkout?.with?.["persist-credentials"]).toBe(false); + expect(prepare?.uses).toBe(PREPARE_E2E_ACTION); + expect(prepare?.with).toEqual(prepareInputs); + expect(prepare?.env).toBeUndefined(); + expect(prepareIndex).toBeGreaterThan(checkoutIndex); + expect(runIndex).toBeGreaterThan(prepareIndex); + expect(steps.filter((step) => step.uses === PREPARE_E2E_ACTION)).toHaveLength(1); + expect(steps.map((step) => step.name)).not.toContain("Setup Node"); + expect(steps.map((step) => step.name)).not.toContain("Install root dependencies"); + expect(steps.map((step) => step.name)).not.toContain("Build CLI"); + }); + + it("collects the gateway drift regression from its integration project (#6692)", () => { + const job = workflow.jobs?.["gateway-drift-preflight-e2e"]; + const runStep = job?.steps?.find( + (step) => step.name === "Run gateway drift preflight E2E test", + ); + + expect(runStep?.run).toContain( + "vitest run --project integration test/gateway-drift-preflight.test.ts", + ); + expect(runStep?.run).not.toContain("vitest run --project cli"); + }); + it("runs the OpenClaw custom-plugin lifecycle and EXDEV guard in a secret-free lane", () => { const job = workflow.jobs?.["openclaw-plugin-runtime-exdev-e2e"]; const steps = job?.steps ?? []; diff --git a/vitest.config.ts b/vitest.config.ts index 93806b312ab..38c8977d0cf 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -35,6 +35,9 @@ const typedSourceTransform = { }, }; const sourceNodeOptions = sourceLoaderNodeOptions(process.env.NODE_OPTIONS); +const controlledNonLiveEnv = { + NEMOCLAW_DISABLE_GATEWAY_DRIFT_PREFLIGHT: "1", +}; // Pin the file-creation umask of every non-live test worker to exactly 0o022 — // the conventional CI baseline — so Hermes/OpenClaw guard fixtures are created // with deterministic modes regardless of the developer's ambient umask (e.g. a @@ -59,9 +62,6 @@ export default defineConfig({ description: "Runs without external credentials in the shared E2E job", }, ], - env: { - NEMOCLAW_DISABLE_GATEWAY_DRIFT_PREFLIGHT: "1", - }, // CI logs are easiest to scan when test chatter stays quiet and failures // surface as GitHub annotations at the relevant file and line. reporters: isGithubActions ? ["github-actions"] : ["default"], @@ -74,6 +74,7 @@ export default defineConfig({ test: { name: "cli", alias: canonicalOpenShellPolicyAlias, + env: controlledNonLiveEnv, testTimeout: testTimeout(), setupFiles: [fixtureUmaskSetup, "test/helpers/onboard-script-mocks.cjs"], include: ["src/**/*.test.ts"], @@ -95,6 +96,7 @@ export default defineConfig({ // project as a bounded four-worker phase after the other projects. ...integrationProjectScheduling, env: { + ...controlledNonLiveEnv, NODE_OPTIONS: sourceNodeOptions, // Integration fixtures exercise onboarding against controlled fake // Docker state. Keep a base-image Dockerfile change in the PR from @@ -122,6 +124,7 @@ export default defineConfig({ test: { name: "installer-integration", alias: canonicalOpenShellPolicyAlias, + env: controlledNonLiveEnv, setupFiles: [fixtureUmaskSetup], include: [ "test/install-express-prompt.test.ts", @@ -139,6 +142,7 @@ export default defineConfig({ test: { name: "package-contract", alias: canonicalOpenShellPolicyAlias, + env: controlledNonLiveEnv, setupFiles: [fixtureUmaskSetup], include: ["test/package-contract/**/*.test.ts"], }, @@ -148,6 +152,7 @@ export default defineConfig({ test: { name: "plugin", alias: canonicalOpenShellPolicyAlias, + env: controlledNonLiveEnv, setupFiles: [fixtureUmaskSetup], include: ["nemoclaw/src/**/*.test.ts"], }, @@ -159,6 +164,7 @@ export default defineConfig({ // only harness; this project does not define a separate runner. name: "e2e-support", alias: canonicalOpenShellPolicyAlias, + env: controlledNonLiveEnv, testTimeout: testTimeout(), setupFiles: [fixtureUmaskSetup, "test/helpers/onboard-script-mocks.cjs"], include: ["test/e2e/support/**/*.test.ts"], @@ -185,6 +191,7 @@ export default defineConfig({ // whole-test retry reuses that state and can hide the first failure // behind stale locks or exhausted storage. Transient operations must // retry inside the target after proving their cleanup boundary. + fileParallelism: false, retry: 0, include: runLiveE2E ? ["test/e2e/live/**/*.test.ts"] : [], // Live E2E tests are opt-in because they install, onboard, and