diff --git a/test/e2e-scenario/framework-tests/e2e-assertion-modules.test.ts b/test/e2e-scenario/framework-tests/e2e-assertion-modules.test.ts index f2aa4ad9f58..d33f9c40079 100644 --- a/test/e2e-scenario/framework-tests/e2e-assertion-modules.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-assertion-modules.test.ts @@ -60,6 +60,19 @@ describe("assertion modules", () => { } }); + it("test_should_keep_snapshot_suite_distinct_from_snapshot_lifecycle", () => { + const snapshot = assertionGroupForSuite("snapshot"); + const snapshotLifecycle = assertionGroupForSuite("snapshot-lifecycle"); + + expect(snapshot?.steps.map((step) => step.id)).toEqual(["runtime.snapshot.sandbox-listed"]); + expect(snapshot?.steps.map((step) => step.implementation?.ref)).toEqual([ + "test/e2e-scenario/validation_suites/smoke/02-sandbox-listed.sh", + ]); + expect(snapshotLifecycle?.steps.map((step) => step.implementation?.ref)).toEqual([ + "test/e2e-scenario/validation_suites/sandbox/snapshot/00-create-list-restore.sh", + ]); + }); + it("test_should_require_each_assertion_group_to_have_steps", () => { const emptyGroup: AssertionGroup = { id: "empty", phase: "runtime", steps: [] }; diff --git a/test/e2e-scenario/framework-tests/e2e-lib-helpers.test.ts b/test/e2e-scenario/framework-tests/e2e-lib-helpers.test.ts index 83d8ef741e1..d8840fca016 100644 --- a/test/e2e-scenario/framework-tests/e2e-lib-helpers.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-lib-helpers.test.ts @@ -15,6 +15,7 @@ const ASSERT = path.join(VALIDATION_SUITES, "assert"); const REBUILD_UPGRADE_LIB = path.join(VALIDATION_SUITES, "lib/rebuild_upgrade.sh"); const FIXTURES = path.join(REPO_ROOT, "test/e2e-scenario/nemoclaw_scenarios/fixtures"); const INSTALL_DIR = path.join(REPO_ROOT, "test/e2e-scenario/nemoclaw_scenarios/install"); +const ONBOARD_DIR = path.join(REPO_ROOT, "test/e2e-scenario/nemoclaw_scenarios/onboard"); function runBash(script: string, env: Record = {}): SpawnSyncReturns { return spawnSync("bash", ["-c", script], { @@ -60,6 +61,192 @@ describe("E2E shell helpers", () => { } }); + it("no_docker_onboarding_worker_should_preserve_seeded_context_and_redact_log", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-no-docker-context-")); + const fakeBin = path.join(tmp, "bin"); + fs.mkdirSync(fakeBin); + fs.writeFileSync( + path.join(fakeBin, "nemoclaw"), + `#!/usr/bin/env bash +if [[ "\${1:-}" = "onboard" ]]; then + expected='onboard --non-interactive --yes --yes-i-accept-third-party-software' + if [[ "$*" != "\${expected}" ]]; then + echo "unexpected nemoclaw args: $*" >&2 + exit 2 + fi + if [[ "\${NEMOCLAW_AGENT:-}" != "openclaw" || "\${NEMOCLAW_PROVIDER:-}" != "cloud" || "\${NEMOCLAW_SANDBOX_NAME:-}" != "e2e-preserved" ]]; then + echo "unexpected nemoclaw env: agent=\${NEMOCLAW_AGENT:-unset} provider=\${NEMOCLAW_PROVIDER:-unset} sandbox=\${NEMOCLAW_SANDBOX_NAME:-unset}" >&2 + exit 2 + fi + echo "NVIDIA_API_KEY=\${NVIDIA_API_KEY:-unset}" >&2 + echo "Docker is required before onboarding" >&2 + exit 42 +fi +echo "unexpected nemoclaw invocation: $*" >&2 +exit 2 +`, + { mode: 0o755 }, + ); + try { + fs.writeFileSync( + path.join(tmp, "context.env"), + "E2E_SCENARIO=ubuntu-no-docker-preflight-negative\nE2E_SANDBOX_NAME=e2e-preserved\n", + ); + const r = runBash( + ` + set -euo pipefail + test/e2e-scenario/nemoclaw_scenarios/dispatch-action.sh e2e_onboard cloud-openclaw-no-docker "${ONBOARD_DIR}/dispatch.sh" + `, + { + E2E_ACTION_ID: "onboarding.profile.cloud-openclaw-no-docker", + E2E_CONTEXT_DIR: tmp, + E2E_PHASE: "onboarding", + NVIDIA_API_KEY: "secret-token", + PATH: `${fakeBin}:${process.env.PATH ?? ""}`, + TMPDIR: tmp, + }, + ); + expect(r.status, `${r.stdout}\n${r.stderr}`).toBe(0); + const contextBody = fs.readFileSync(path.join(tmp, "context.env"), "utf8"); + expect(contextBody).toMatch(/^E2E_SANDBOX_NAME=e2e-preserved$/m); + const logBody = fs.readFileSync(path.join(tmp, "negative-preflight.log"), "utf8"); + expect(logBody).toContain("Docker is required before onboarding"); + expect(logBody).toContain("[REDACTED]"); + expect(logBody).not.toContain("secret-token"); + const tempEntries = fs.readdirSync(tmp, { recursive: true }).map(String).join("\n"); + expect(tempEntries).not.toContain("negative-preflight.raw.log"); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("no_docker_onboarding_worker_should_fail_on_unrelated_onboarding_errors", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-no-docker-unrelated-")); + const fakeBin = path.join(tmp, "bin"); + fs.mkdirSync(fakeBin); + fs.writeFileSync( + path.join(fakeBin, "nemoclaw"), + `#!/usr/bin/env bash +if [[ "\${1:-}" = "onboard" ]]; then + expected='onboard --non-interactive --yes --yes-i-accept-third-party-software' + if [[ "$*" != "\${expected}" ]]; then + echo "unexpected nemoclaw args: $*" >&2 + exit 2 + fi + if [[ "\${NEMOCLAW_AGENT:-}" != "openclaw" || "\${NEMOCLAW_PROVIDER:-}" != "cloud" || "\${NEMOCLAW_SANDBOX_NAME:-}" != "e2e-preserved" ]]; then + echo "unexpected nemoclaw env: agent=\${NEMOCLAW_AGENT:-unset} provider=\${NEMOCLAW_PROVIDER:-unset} sandbox=\${NEMOCLAW_SANDBOX_NAME:-unset}" >&2 + exit 2 + fi + echo "provider rejected NVIDIA_API_KEY=\${NVIDIA_API_KEY:-unset}" >&2 + exit 42 +fi +echo "unexpected nemoclaw invocation: $*" >&2 +exit 2 +`, + { mode: 0o755 }, + ); + try { + fs.writeFileSync( + path.join(tmp, "context.env"), + "E2E_SCENARIO=ubuntu-no-docker-preflight-negative\nE2E_SANDBOX_NAME=e2e-preserved\n", + ); + const r = runBash( + ` + set -euo pipefail + test/e2e-scenario/nemoclaw_scenarios/dispatch-action.sh e2e_onboard cloud-openclaw-no-docker "${ONBOARD_DIR}/dispatch.sh" + `, + { + E2E_ACTION_ID: "onboarding.profile.cloud-openclaw-no-docker", + E2E_CONTEXT_DIR: tmp, + E2E_PHASE: "onboarding", + NVIDIA_API_KEY: "secret-token", + PATH: `${fakeBin}:${process.env.PATH ?? ""}`, + TMPDIR: tmp, + }, + ); + expect(r.status).toBe(42); + expect(`${r.stdout}\n${r.stderr}`).toContain("failed without Docker-missing preflight signature"); + const logBody = fs.readFileSync(path.join(tmp, "negative-preflight.log"), "utf8"); + expect(logBody).toContain("provider rejected"); + expect(logBody).toContain("[REDACTED]"); + expect(logBody).not.toContain("secret-token"); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("no_docker_onboarding_worker_should_accept_current_preflight_wording", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-no-docker-wording-")); + const fakeBin = path.join(tmp, "bin"); + fs.mkdirSync(fakeBin); + fs.writeFileSync( + path.join(fakeBin, "nemoclaw"), + `#!/usr/bin/env bash +if [[ "\${1:-}" = "onboard" ]]; then + echo "Docker is not reachable. Please fix Docker and try again." >&2 + exit 1 +fi +echo "unexpected nemoclaw invocation: $*" >&2 +exit 2 +`, + { mode: 0o755 }, + ); + try { + fs.writeFileSync( + path.join(tmp, "context.env"), + "E2E_SCENARIO=ubuntu-no-docker-preflight-negative\nE2E_SANDBOX_NAME=e2e-preserved\n", + ); + const r = runBash( + ` + set -euo pipefail + test/e2e-scenario/nemoclaw_scenarios/dispatch-action.sh e2e_onboard cloud-openclaw-no-docker "${ONBOARD_DIR}/dispatch.sh" + `, + { + E2E_ACTION_ID: "onboarding.profile.cloud-openclaw-no-docker", + E2E_CONTEXT_DIR: tmp, + E2E_PHASE: "onboarding", + NVIDIA_API_KEY: "secret-token", + PATH: `${fakeBin}:${process.env.PATH ?? ""}`, + TMPDIR: tmp, + }, + ); + expect(r.status, `${r.stdout}\n${r.stderr}`).toBe(0); + const logBody = fs.readFileSync(path.join(tmp, "negative-preflight.log"), "utf8"); + expect(logBody).toContain("Docker is not reachable"); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("no_docker_redactor_fallback_should_redact_sensitive_env_values_without_python", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-no-docker-redactor-")); + const noPythonBin = path.join(tmp, "bin"); + const logPath = path.join(tmp, "negative-preflight.log"); + try { + const r = runBash( + ` + set -euo pipefail + mkdir -p "${noPythonBin}" + for cmd in rm mktemp sed env cat mv; do + ln -s "$(command -v "\${cmd}")" "${noPythonBin}/\${cmd}" + done + . "${ONBOARD_DIR}/cloud-openclaw-no-docker.sh" + export NVIDIA_API_KEY=plain-secret-value + PATH="${noPythonBin}" + printf 'plain-secret-value\\nDocker is required before onboarding\\n' | e2e_no_docker_write_redacted_preflight_log "${logPath}" + `, + { TMPDIR: tmp }, + ); + expect(r.status, `${r.stdout}\n${r.stderr}`).toBe(0); + const logBody = fs.readFileSync(logPath, "utf8"); + expect(logBody).toContain("[REDACTED]"); + expect(logBody).toContain("Docker is required before onboarding"); + expect(logBody).not.toContain("plain-secret-value"); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + it("security_policy_credentials_helper_should_load_with_context_library", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-context-")); try { diff --git a/test/e2e-scenario/framework-tests/e2e-negative-matcher.test.ts b/test/e2e-scenario/framework-tests/e2e-negative-matcher.test.ts index 363cb3fcc99..dc432759c68 100644 --- a/test/e2e-scenario/framework-tests/e2e-negative-matcher.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-negative-matcher.test.ts @@ -13,6 +13,7 @@ import { } from "../scenarios/orchestrators/negative-matcher.ts"; import { ScenarioRunner } from "../scenarios/orchestrators/runner.ts"; import { listScenarios } from "../scenarios/registry.ts"; +import { planFailed } from "../scenarios/run.ts"; import type { ExpectedFailureContract, PhaseName, @@ -75,6 +76,35 @@ function phaseResult( }; } +function passedNegativeContractPhase(): PhaseResult { + return { + phase: "negative-contract", + status: "passed", + actions: [], + assertions: [ + { + id: "negative-contract.match", + status: "passed", + attempts: 1, + durationMs: 0, + message: "matched", + }, + ], + }; +} + +function stateValidationResult( + status: PhaseResult["status"], + actionIds: string[] = ["state-validation.gateway-absent", "state-validation.sandbox-absent"], +): PhaseResult { + return { + phase: "state-validation", + status, + actions: actionIds.map((id) => ({ id, status: "passed", durationMs: 1 })), + assertions: [], + }; +} + describe("evaluateNegativeContract - phase + errorClass matching", () => { it("matches when expected phase fails with the declared errorClass", () => { const plan = planWithExpectedFailure({ @@ -127,6 +157,70 @@ describe("evaluateNegativeContract - phase + errorClass matching", () => { expect(result.message).toMatch(/all phases passed/); }); + it("matches when a passed expected-failure assertion handled the failure", () => { + const plan = planWithExpectedFailure({ + phase: "preflight", + errorClass: "docker-missing", + forbiddenSideEffects: ["gateway-started", "sandbox-created"], + }); + const results: PhaseResult[] = [ + phaseResult("environment", { status: "passed" }), + { + phase: "onboarding", + status: "passed", + actions: [ + { + id: "onboarding.profile.cloud-openclaw-no-docker", + status: "passed", + durationMs: 1, + }, + ], + assertions: [ + { + id: "onboarding.preflight.expected-failed", + status: "passed", + attempts: 1, + durationMs: 1, + }, + ], + }, + phaseResult("state-validation", { status: "passed" }), + ]; + + const result = evaluateNegativeContract(plan, results); + expect(result.matched).toBe(true); + expect(result.outcome).toBe("matched"); + expect(result.observed).toMatchObject({ + failedPhase: "onboarding", + handledAssertionId: "onboarding.preflight.expected-failed", + }); + }); + + it("matches handled expected-failure actions using scenario error-class aliases", () => { + const plan = planWithExpectedFailure({ + phase: "onboarding", + errorClass: "invalid-nvidia-api-key", + }); + const results: PhaseResult[] = [ + { + phase: "onboarding", + status: "passed", + actions: [ + { + id: "onboarding.profile.cloud-openclaw-invalid-nvidia-key", + status: "passed", + durationMs: 1, + }, + ], + assertions: [], + }, + ]; + + const result = evaluateNegativeContract(plan, results); + expect(result.matched).toBe(true); + expect(result.observed.handledActionId).toBe("onboarding.profile.cloud-openclaw-invalid-nvidia-key"); + }); + it("fails when the wrong phase failed", () => { const plan = planWithExpectedFailure({ phase: "onboarding", errorClass: "docker-missing" }); const results: PhaseResult[] = [ @@ -228,6 +322,35 @@ describe("evaluateNegativeContract - phase + errorClass matching", () => { }); }); +describe("negative plan exit-code contract", () => { + const plan = planWithExpectedFailure({ + phase: "preflight", + errorClass: "docker-missing", + forbiddenSideEffects: ["gateway-started", "sandbox-created"], + }); + + it("passes when negative contract and forbidden-side-effect probes pass", () => { + expect(planFailed(plan, [passedNegativeContractPhase(), stateValidationResult("passed")])).toBe(false); + }); + + it("fails when state-validation is missing", () => { + expect(planFailed(plan, [passedNegativeContractPhase()])).toBe(true); + }); + + it("fails when state-validation is skipped", () => { + expect(planFailed(plan, [passedNegativeContractPhase(), stateValidationResult("skipped")])).toBe(true); + }); + + it("fails when a declared forbidden-side-effect probe did not run", () => { + expect( + planFailed(plan, [ + passedNegativeContractPhase(), + stateValidationResult("passed", ["state-validation.gateway-absent"]), + ]), + ).toBe(true); + }); +}); + describe("ScenarioRunner appends negative-contract phase", () => { it("invokes matcher and appends a passing synthetic phase when contract matched", async () => { const ctx = freshCtx(); @@ -352,21 +475,15 @@ describe("ScenarioRunner appends negative-contract phase", () => { }); }); -describe("registry contract: every negative scenario opts into the side-effect probe", () => { - it("scenario.expectedFailure implies the runtime no-side-effects required pending step", () => { +describe("registry contract: negative scenarios use typed state-validation side-effect probes", () => { + it("scenario.expectedFailure does not inject the legacy runtime no-side-effects pending step", () => { const negatives = listScenarios().filter((scenario) => scenario.expectedFailure); expect(negatives.length).toBeGreaterThan(0); for (const scenario of negatives) { - const runtimeGroups = scenario.assertionGroups.filter((group) => group.phase === "runtime"); - const hasProbeStep = runtimeGroups.some((group) => - group.steps.some( - (step) => - step.id === "runtime.expected-failure.no-side-effects" && - step.implementation?.kind === "pending" && - step.required === true, - ), + const hasLegacyPendingStep = scenario.assertionGroups.some((group) => + group.steps.some((step) => step.id === "runtime.expected-failure.no-side-effects"), ); - expect(hasProbeStep, `scenario ${scenario.id} must include the required side-effect pending step`).toBe(true); + expect(hasLegacyPendingStep, `scenario ${scenario.id} must rely on state-validation, not the legacy pending step`).toBe(false); } }); }); diff --git a/test/e2e-scenario/framework-tests/e2e-phase-environment.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-environment.test.ts index 524bad16670..81a9fe91721 100644 --- a/test/e2e-scenario/framework-tests/e2e-phase-environment.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-phase-environment.test.ts @@ -132,19 +132,23 @@ describe("environment phase fixture", () => { }); }); - it("fails if a no-Docker negative scenario unexpectedly has Docker", async () => { + it("records Docker availability for no-Docker negative scenarios without blocking simulation", async () => { const runner = new FakeRunner(); runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); runner.enqueue(shellResult(0, "Docker is available\n")); const environment = new EnvironmentPhaseFixture(new HostCliClient(runner)); - await expect( - environment.assertReady({ - ...cloudOpenClawEnvironment, - runtime: "docker-missing", - onboarding: "cloud-openclaw-no-docker", - }), - ).rejects.toThrow(/expected Docker to be unavailable/); + const ready = await environment.assertReady({ + ...cloudOpenClawEnvironment, + runtime: "docker-missing", + onboarding: "cloud-openclaw-no-docker", + }); + + expect(ready.docker).toMatchObject({ + id: "docker-missing", + expectation: "missing", + available: true, + }); }); it("records optional Docker as unavailable without failing", async () => { diff --git a/test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts new file mode 100644 index 00000000000..95a93f35514 --- /dev/null +++ b/test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts @@ -0,0 +1,453 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, expectTypeOf, it } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { HostCliClient, type CommandRunner } from "../framework/clients/index.ts"; +import type { E2EScenarioFixtures } from "../framework/e2e-test.ts"; +import { OnboardingPhaseFixture, type OnboardingSecrets } from "../framework/phases/index.ts"; +import type { EnvironmentReady } from "../framework/phases/index.ts"; +import type { ShellProbeResult, ShellProbeRunOptions, TrustedShellCommand } from "../framework/shell-probe.ts"; + +interface RunnerCall { + command: string; + args: string[]; + options?: ShellProbeRunOptions; +} + +interface CleanupCall { + name: string; + run: () => Promise | void; +} + +function shellResult(exitCode: number, output = ""): ShellProbeResult { + return { + command: [], + exitCode, + signal: null, + timedOut: false, + stdout: output, + stderr: exitCode === 0 ? "" : output, + artifacts: { + stdout: "/tmp/stdout.txt", + stderr: "/tmp/stderr.txt", + result: "/tmp/result.json", + }, + }; +} + +class FakeRunner implements CommandRunner { + readonly calls: RunnerCall[] = []; + private readonly responses: ShellProbeResult[] = []; + + enqueue(response: ShellProbeResult): void { + this.responses.push(response); + } + + async run(command: TrustedShellCommand, options?: ShellProbeRunOptions): Promise { + this.calls.push({ command: command.command, args: [...command.args], options }); + const response = this.responses.shift(); + if (!response) { + throw new Error(`FakeRunner response missing for command: ${command.command} ${command.args.join(" ")}`); + } + return response; + } +} + +class FakeCleanup { + readonly calls: CleanupCall[] = []; + + add(name: string, run: () => Promise | void): void { + this.calls.push({ name, run }); + } +} + +class FakeSecrets implements OnboardingSecrets { + readonly requiredCalls: string[] = []; + + constructor(private readonly values: Record = {}) {} + + required(name: string): string { + this.requiredCalls.push(name); + const value = this.values[name]; + if (!value) throw new Error(`skip: missing required E2E secret: ${name}`); + return value; + } + + redact(text: string, extraValues: string[] = []): string { + const values = [...Object.values(this.values), ...extraValues].filter((value): value is string => Boolean(value)); + return values.reduce((redacted, value) => redacted.split(value).join("[REDACTED]"), text); + } +} + +function ready(overrides: Partial = {}): EnvironmentReady { + return { + platform: "ubuntu-local", + install: "repo-current", + runtime: "docker-running", + onboarding: "cloud-openclaw", + cliPath: "nemoclaw", + docker: { + id: "docker-running", + expectation: "required", + available: true, + result: shellResult(0), + }, + ...overrides, + }; +} + +describe("onboarding phase fixture", () => { + it("runs cloud OpenClaw onboarding with explicit non-interactive inputs", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "onboarded\n")); + const secrets = new FakeSecrets({ NVIDIA_API_KEY: "secret-token" }); + const onboard = new OnboardingPhaseFixture(new HostCliClient(runner), secrets); + + const instance = await onboard.from(ready(), { sandboxName: "e2e-ubuntu-repo-cloud-openclaw" }); + + expect(instance).toMatchObject({ + onboarding: "cloud-openclaw", + sandboxName: "e2e-ubuntu-repo-cloud-openclaw", + agent: "openclaw", + provider: "nvidia", + providerEnv: "cloud", + gatewayUrl: "http://127.0.0.1:18789", + }); + expect(secrets.requiredCalls).toEqual(["NVIDIA_API_KEY"]); + expect(runner.calls).toEqual([ + { + command: "nemoclaw", + args: ["onboard", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"], + options: { + artifactName: "onboard-cloud-openclaw", + env: expect.objectContaining({ + NEMOCLAW_AGENT: "openclaw", + NEMOCLAW_PROVIDER: "cloud", + NEMOCLAW_SANDBOX_NAME: "e2e-ubuntu-repo-cloud-openclaw", + NVIDIA_API_KEY: "secret-token", + PATH: expect.any(String), + }), + redactionValues: ["secret-token"], + timeoutMs: 900_000, + }, + }, + ]); + }); + + it("fails cloud OpenClaw onboarding on non-zero exit", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(42, "provider rejected credential")); + const onboard = new OnboardingPhaseFixture(new HostCliClient(runner), new FakeSecrets({ NVIDIA_API_KEY: "secret" })); + + await expect(onboard.from(ready())).rejects.toThrow(/cloud-openclaw onboarding failed: provider rejected/); + }); + + it("keeps sandbox cleanup registered when cloud OpenClaw onboarding fails", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(42, "provider rejected credential")); + const cleanup = new FakeCleanup(); + const onboard = new OnboardingPhaseFixture( + new HostCliClient(runner), + new FakeSecrets({ NVIDIA_API_KEY: "secret" }), + cleanup, + ); + + await expect(onboard.from(ready(), { sandboxName: "e2e-partial-onboard" })).rejects.toThrow( + /cloud-openclaw onboarding failed: provider rejected/, + ); + + expect(cleanup.calls).toHaveLength(1); + expect(cleanup.calls[0]?.name).toBe("destroy NemoClaw sandbox e2e-partial-onboard"); + runner.enqueue(shellResult(1, "Error: sandbox e2e-partial-onboard not found")); + await cleanup.calls[0]?.run(); + expect(runner.calls[1]).toMatchObject({ + command: "nemoclaw", + args: ["e2e-partial-onboard", "destroy", "--yes"], + options: { + artifactName: "cleanup-destroy-e2e-partial-onboard", + timeoutMs: 900_000, + }, + }); + }); + + it("requires NVIDIA_API_KEY before spawning cloud OpenClaw onboarding", async () => { + const runner = new FakeRunner(); + const onboard = new OnboardingPhaseFixture(new HostCliClient(runner), new FakeSecrets()); + + await expect(onboard.from(ready())).rejects.toThrow(/missing required E2E secret: NVIDIA_API_KEY/); + expect(runner.calls).toEqual([]); + }); + + it("requires Docker for cloud OpenClaw onboarding", async () => { + const onboard = new OnboardingPhaseFixture(new HostCliClient(new FakeRunner()), new FakeSecrets({ NVIDIA_API_KEY: "secret" })); + + await expect( + onboard.from( + ready({ + docker: { id: "docker-running", expectation: "required", available: false }, + }), + ), + ).rejects.toThrow(/requires an available Docker runtime/); + }); + + it("rejects invalid sandbox names before cloud OpenClaw side effects", async () => { + const runner = new FakeRunner(); + const cleanup = new FakeCleanup(); + const secrets = new FakeSecrets({ NVIDIA_API_KEY: "secret" }); + const onboard = new OnboardingPhaseFixture(new HostCliClient(runner), secrets, cleanup); + + await expect(onboard.from(ready(), { sandboxName: "bad name" })).rejects.toThrow( + /sandbox name is invalid for fixture client/, + ); + + expect(secrets.requiredCalls).toEqual([]); + expect(runner.calls).toEqual([]); + expect(cleanup.calls).toEqual([]); + }); + + it("registers sandbox cleanup after successful cloud OpenClaw onboarding", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "onboarded\n")); + const cleanup = new FakeCleanup(); + const onboard = new OnboardingPhaseFixture( + new HostCliClient(runner), + new FakeSecrets({ NVIDIA_API_KEY: "secret-token" }), + cleanup, + ); + + await onboard.from(ready(), { sandboxName: "e2e-cleanup" }); + + expect(cleanup.calls).toHaveLength(1); + expect(cleanup.calls[0]?.name).toBe("destroy NemoClaw sandbox e2e-cleanup"); + runner.enqueue(shellResult(0, "destroyed\n")); + await cleanup.calls[0]?.run(); + expect(runner.calls[1]).toMatchObject({ + command: "nemoclaw", + args: ["e2e-cleanup", "destroy", "--yes"], + options: { + artifactName: "cleanup-destroy-e2e-cleanup", + timeoutMs: 900_000, + }, + }); + expect(runner.calls[1]?.options?.env).toMatchObject({ + PATH: expect.any(String), + }); + }); + + it("runs the no-Docker negative path with a failing Docker shim", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(7, "Cannot connect to the Docker daemon")); + const secrets = new FakeSecrets({ NVIDIA_API_KEY: "secret-token" }); + const cleanup = new FakeCleanup(); + const onboard = new OnboardingPhaseFixture(new HostCliClient(runner), secrets, cleanup); + + const instance = await onboard.from( + ready({ + runtime: "docker-missing", + onboarding: "cloud-openclaw-no-docker", + docker: { id: "docker-missing", expectation: "missing", available: true }, + }), + { sandboxName: "e2e-no-docker" }, + ); + + expect(instance).toMatchObject({ + onboarding: "cloud-openclaw-no-docker", + sandboxName: "e2e-no-docker", + expectedFailure: { + phase: "preflight", + errorClass: "docker-missing", + }, + }); + expect(runner.calls[0]).toMatchObject({ + command: "nemoclaw", + args: ["onboard", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"], + options: { + artifactName: "onboard-cloud-openclaw-no-docker", + timeoutMs: 900_000, + }, + }); + expect(runner.calls[0]?.options?.inheritEnv).toBeUndefined(); + expect(runner.calls[0]?.options?.redactionValues).toEqual(["secret-token"]); + expect(runner.calls[0]?.options?.env).toMatchObject({ + NEMOCLAW_AGENT: "openclaw", + NEMOCLAW_PROVIDER: "cloud", + NEMOCLAW_SANDBOX_NAME: "e2e-no-docker", + NVIDIA_API_KEY: "secret-token", + }); + expect(runner.calls[0]?.options?.env?.PATH).toContain("e2e-no-docker-"); + expect(secrets.requiredCalls).toEqual(["NVIDIA_API_KEY"]); + expect(cleanup.calls).toHaveLength(1); + expect(cleanup.calls[0]?.name).toBe("destroy NemoClaw sandbox e2e-no-docker"); + }); + + it("publishes redacted legacy preflight evidence for the no-Docker negative path", async () => { + const previousContextDir = process.env.E2E_CONTEXT_DIR; + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-typed-no-docker-")); + process.env.E2E_CONTEXT_DIR = tmp; + try { + const runner = new FakeRunner(); + runner.enqueue(shellResult(7, "Docker is required before onboarding with secret-token")); + const onboard = new OnboardingPhaseFixture( + new HostCliClient(runner), + new FakeSecrets({ NVIDIA_API_KEY: "secret-token" }), + ); + + await onboard.from( + ready({ + runtime: "docker-missing", + onboarding: "cloud-openclaw-no-docker", + docker: { id: "docker-missing", expectation: "missing", available: true }, + }), + ); + + const logBody = fs.readFileSync(path.join(tmp, "negative-preflight.log"), "utf8"); + expect(logBody).toContain("Docker is required before onboarding"); + expect(logBody).toContain("[REDACTED]"); + expect(logBody).not.toContain("secret-token"); + } finally { + if (previousContextDir === undefined) { + delete process.env.E2E_CONTEXT_DIR; + } else { + process.env.E2E_CONTEXT_DIR = previousContextDir; + } + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("accepts current Docker-unreachable wording for the no-Docker negative path", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(7, "Docker is not reachable. Please fix Docker and try again.")); + const onboard = new OnboardingPhaseFixture( + new HostCliClient(runner), + new FakeSecrets({ NVIDIA_API_KEY: "secret" }), + ); + + const instance = await onboard.from( + ready({ + runtime: "docker-missing", + onboarding: "cloud-openclaw-no-docker", + docker: { id: "docker-missing", expectation: "missing", available: false }, + }), + ); + + expect(instance.expectedFailure).toEqual({ + phase: "preflight", + errorClass: "docker-missing", + }); + }); + + it("requires the docker-missing runtime expectation for the no-Docker negative path", async () => { + const runner = new FakeRunner(); + const onboard = new OnboardingPhaseFixture(new HostCliClient(runner), new FakeSecrets({ NVIDIA_API_KEY: "secret" })); + + await expect(onboard.from(ready({ onboarding: "cloud-openclaw-no-docker" }))).rejects.toThrow( + /requires the docker-missing runtime expectation/, + ); + expect(runner.calls).toEqual([]); + }); + + it("does not add an empty PATH segment when the no-Docker base env has no PATH", async () => { + const previousHome = process.env.HOME; + const previousPath = process.env.PATH; + delete process.env.HOME; + delete process.env.PATH; + try { + const runner = new FakeRunner(); + runner.enqueue(shellResult(7, "Docker is required before onboarding")); + const onboard = new OnboardingPhaseFixture( + new HostCliClient(runner), + new FakeSecrets({ NVIDIA_API_KEY: "secret-token" }), + ); + + await onboard.from( + ready({ + runtime: "docker-missing", + onboarding: "cloud-openclaw-no-docker", + docker: { id: "docker-missing", expectation: "missing", available: false }, + }), + ); + + const pathValue = runner.calls[0]?.options?.env?.PATH; + expect(pathValue).toContain("e2e-no-docker-"); + expect(pathValue?.split(":")).not.toContain(""); + } finally { + if (previousHome === undefined) { + delete process.env.HOME; + } else { + process.env.HOME = previousHome; + } + if (previousPath === undefined) { + delete process.env.PATH; + } else { + process.env.PATH = previousPath; + } + } + }); + + it("fails the no-Docker path when onboarding unexpectedly succeeds", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "onboarded\n")); + const cleanup = new FakeCleanup(); + const onboard = new OnboardingPhaseFixture( + new HostCliClient(runner), + new FakeSecrets({ NVIDIA_API_KEY: "secret" }), + cleanup, + ); + + await expect( + onboard.from( + ready({ + runtime: "docker-missing", + onboarding: "cloud-openclaw-no-docker", + docker: { id: "docker-missing", expectation: "missing", available: false }, + }), + { sandboxName: "e2e-no-docker-success" }, + ), + ).rejects.toThrow(/unexpectedly succeeded/); + + expect(cleanup.calls).toHaveLength(1); + expect(cleanup.calls[0]?.name).toBe("destroy NemoClaw sandbox e2e-no-docker-success"); + runner.enqueue(shellResult(0, "destroyed\n")); + await cleanup.calls[0]?.run(); + expect(runner.calls[1]).toMatchObject({ + command: "nemoclaw", + args: ["e2e-no-docker-success", "destroy", "--yes"], + options: { + artifactName: "cleanup-destroy-e2e-no-docker-success", + timeoutMs: 900_000, + }, + }); + }); + + it("rejects unrelated no-Docker onboarding failures", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(9, "provider rejected credential")); + const onboard = new OnboardingPhaseFixture(new HostCliClient(runner), new FakeSecrets({ NVIDIA_API_KEY: "secret" })); + + await expect( + onboard.from( + ready({ + runtime: "docker-missing", + onboarding: "cloud-openclaw-no-docker", + docker: { id: "docker-missing", expectation: "missing", available: false }, + }), + ), + ).rejects.toThrow(/without Docker-missing preflight signature/); + }); + + it("rejects unsupported onboarding profiles", async () => { + const onboard = new OnboardingPhaseFixture(new HostCliClient(new FakeRunner()), new FakeSecrets()); + + await expect(onboard.from(ready({ onboarding: "cloud-hermes" }))).rejects.toThrow( + /Unsupported onboarding profile 'cloud-hermes'/, + ); + }); + + it("exposes the onboarding phase on the Vitest scenario context", () => { + expectTypeOf().toEqualTypeOf(); + }); +}); diff --git a/test/e2e-scenario/framework-tests/e2e-phase-orchestrators.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-orchestrators.test.ts index 52ec95cddba..9fde64b1bea 100644 --- a/test/e2e-scenario/framework-tests/e2e-phase-orchestrators.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-phase-orchestrators.test.ts @@ -773,15 +773,12 @@ describe("required probe and pending steps fail closed", () => { } }); - it("test_expected_failure_no_side_effects_step_in_registry_is_required", async () => { + it("test_expected_failure_no_side_effects_step_is_not_in_active_registry", async () => { const { assertionRegistry } = await import("../scenarios/assertions/registry.ts"); const group = assertionRegistry.groups.find( (g) => g.id === "runtime.expected-failure.no-side-effects", ); - expect(group).toBeDefined(); - for (const step of group?.steps ?? []) { - expect(step.required).toBe(true); - } + expect(group).toBeUndefined(); }); }); diff --git a/test/e2e-scenario/framework/clients/index.ts b/test/e2e-scenario/framework/clients/index.ts index fb4c009d001..e9270890352 100644 --- a/test/e2e-scenario/framework/clients/index.ts +++ b/test/e2e-scenario/framework/clients/index.ts @@ -5,5 +5,5 @@ export { assertExitZero, type CommandRunner } from "./command.ts"; export { GatewayClient } from "./gateway.ts"; export { HostCliClient } from "./host.ts"; export { ProviderClient, trustedProviderEndpoint, type TrustedProviderEndpoint } from "./provider.ts"; -export { SandboxClient } from "./sandbox.ts"; +export { SandboxClient, validateSandboxName } from "./sandbox.ts"; export { StateClient } from "./state.ts"; diff --git a/test/e2e-scenario/framework/clients/sandbox.ts b/test/e2e-scenario/framework/clients/sandbox.ts index af0c0a4154e..a7ee5765bb5 100644 --- a/test/e2e-scenario/framework/clients/sandbox.ts +++ b/test/e2e-scenario/framework/clients/sandbox.ts @@ -56,7 +56,7 @@ export class SandboxClient { } } -function validateSandboxName(name: string): void { +export function validateSandboxName(name: string): void { if (!/^[A-Za-z0-9][A-Za-z0-9_.-]*$/.test(name)) { throw new Error(`sandbox name is invalid for fixture client: ${name}`); } diff --git a/test/e2e-scenario/framework/e2e-test.ts b/test/e2e-scenario/framework/e2e-test.ts index 6d122b16b1a..cc5d63ec7cc 100644 --- a/test/e2e-scenario/framework/e2e-test.ts +++ b/test/e2e-scenario/framework/e2e-test.ts @@ -12,7 +12,7 @@ import { StateClient, } from "./clients/index.ts"; import { assertCleanupPassed, CleanupRegistry } from "./cleanup.ts"; -import { EnvironmentPhaseFixture } from "./phases/index.ts"; +import { EnvironmentPhaseFixture, OnboardingPhaseFixture } from "./phases/index.ts"; import { SecretStore } from "./secrets.ts"; import { ShellProbe } from "./shell-probe.ts"; @@ -27,6 +27,7 @@ export interface E2EScenarioFixtures { provider: ProviderClient; state: StateClient; environment: EnvironmentPhaseFixture; + onboard: OnboardingPhaseFixture; } export const test = base.extend({ @@ -82,6 +83,9 @@ export const test = base.extend({ environment: async ({ host }, use) => { await use(new EnvironmentPhaseFixture(host)); }, + onboard: async ({ cleanup, host, secrets }, use) => { + await use(new OnboardingPhaseFixture(host, secrets, cleanup)); + }, }); export { expect }; diff --git a/test/e2e-scenario/framework/phases/environment.ts b/test/e2e-scenario/framework/phases/environment.ts index 8b4b8d7dd7a..6442badf1bd 100644 --- a/test/e2e-scenario/framework/phases/environment.ts +++ b/test/e2e-scenario/framework/phases/environment.ts @@ -74,9 +74,8 @@ export class EnvironmentPhaseFixture { if (expectation === "required") { assertExitZero(result.result, `docker runtime ${runtime}`); } - if (expectation === "missing" && result.available) { - throw new Error(`docker runtime ${runtime} expected Docker to be unavailable, but 'docker info' succeeded.`); - } + // Missing-runtime scenarios simulate Docker failure at the phase that + // needs it; this probe records host reality without blocking composition. return result; } diff --git a/test/e2e-scenario/framework/phases/index.ts b/test/e2e-scenario/framework/phases/index.ts index b1eba5be54b..25b3dba5147 100644 --- a/test/e2e-scenario/framework/phases/index.ts +++ b/test/e2e-scenario/framework/phases/index.ts @@ -7,3 +7,10 @@ export { type DockerRuntimeReady, type EnvironmentReady, } from "./environment.ts"; +export { + OnboardingPhaseFixture, + type NemoClawInstance, + type OnboardingExpectedFailure, + type OnboardingOptions, + type OnboardingSecrets, +} from "./onboarding.ts"; diff --git a/test/e2e-scenario/framework/phases/onboarding.ts b/test/e2e-scenario/framework/phases/onboarding.ts new file mode 100644 index 00000000000..458813958c0 --- /dev/null +++ b/test/e2e-scenario/framework/phases/onboarding.ts @@ -0,0 +1,242 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { chmod, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; + +import { buildAvailabilityProbeEnv } from "../availability-env.ts"; +import { artifactLabel, assertExitZero } from "../clients/command.ts"; +import type { HostCliClient } from "../clients/host.ts"; +import { validateSandboxName } from "../clients/sandbox.ts"; +import type { ShellProbeResult } from "../shell-probe.ts"; +import { redactString } from "../../scenarios/orchestrators/redaction.ts"; +import type { EnvironmentReady } from "./environment.ts"; + +const ONBOARD_ARGS = ["onboard", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"]; +const DEFAULT_TIMEOUT_MS = 15 * 60_000; +const OPENCLAW_GATEWAY_URL = "http://127.0.0.1:18789"; +const NEGATIVE_PREFLIGHT_LOG = "negative-preflight.log"; +const DOCKER_MISSING_PATTERNS = [ + /Cannot connect to the Docker daemon/i, + /Is the docker daemon running\??/i, + /docker daemon is not running/i, + /docker[- ]missing/i, + /Docker is required before onboarding/i, + /Docker is not reachable/i, + /could not talk to the Docker daemon/i, +]; +const MISSING_SANDBOX_DELETE_PATTERNS = [ + /\bNotFound\b/i, + /\bNot Found\b/i, + /sandbox not found/i, + /sandbox .* not found/i, + /sandbox .* not present/i, + /sandbox does not exist/i, + /no such sandbox/i, +]; + +export interface OnboardingSecrets { + required(name: string): string; + redact?(text: string, extraValues?: string[]): string; +} + +export interface OnboardingCleanup { + add(name: string, run: () => Promise | void): void; +} + +export interface OnboardingOptions { + sandboxName?: string; + timeoutMs?: number; +} + +export interface OnboardingExpectedFailure { + phase: "preflight"; + errorClass: "docker-missing"; +} + +export interface NemoClawInstance { + onboarding: string; + sandboxName: string; + agent: "openclaw"; + provider: "nvidia"; + providerEnv: "cloud"; + gatewayUrl: string; + result: ShellProbeResult; + expectedFailure?: OnboardingExpectedFailure; +} + +function defaultSandboxName(onboarding: string): string { + return `e2e-${artifactLabel(onboarding)}`; +} + +function sandboxNameFromOptions(onboarding: string, options: OnboardingOptions): string { + const sandboxName = options.sandboxName ?? defaultSandboxName(onboarding); + validateSandboxName(sandboxName); + return sandboxName; +} + +function commandEnv(sandboxName: string, extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + ...extra, + NEMOCLAW_AGENT: "openclaw", + NEMOCLAW_PROVIDER: "cloud", + NEMOCLAW_SANDBOX_NAME: sandboxName, + }; +} + +function noDockerShim(): string { + // Migration source of truth for the typed fixture path: simulate the invalid + // state where the Docker client exists but the daemon is unreachable. The + // legacy shell worker keeps a matching shim until live no-Docker onboarding + // dispatch moves fully into Vitest; remove both shims once the scenario can + // inject a Docker client boundary directly instead of shadowing command lookup. + return `#!/usr/bin/env bash +printf 'Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?\\n' >&2 +exit 1 +`; +} + +function prependPath(pathEntry: string, currentPath?: string): string { + return currentPath ? `${pathEntry}:${currentPath}` : pathEntry; +} + +function resultText(result: ShellProbeResult): string { + return [result.stdout, result.stderr].filter(Boolean).join("\n"); +} + +function redactExplicitValues(text: string, values: string[]): string { + return values.reduce((redacted, value) => (value ? redacted.split(value).join("[REDACTED]") : redacted), text); +} + +function legacyNegativePreflightLogPath(): string | undefined { + const contextDir = process.env.E2E_CONTEXT_DIR; + return contextDir ? join(contextDir, NEGATIVE_PREFLIGHT_LOG) : undefined; +} + +function hasDockerMissingSignature(result: ShellProbeResult): boolean { + const text = resultText(result); + return DOCKER_MISSING_PATTERNS.some((pattern) => pattern.test(text)); +} + +function hasMissingSandboxDeleteSignature(result: ShellProbeResult): boolean { + const text = resultText(result); + return MISSING_SANDBOX_DELETE_PATTERNS.some((pattern) => pattern.test(text)); +} + +export class OnboardingPhaseFixture { + constructor( + private readonly host: HostCliClient, + private readonly secrets: OnboardingSecrets, + private readonly cleanup?: OnboardingCleanup, + ) {} + + async from(environment: EnvironmentReady, options: OnboardingOptions = {}): Promise { + switch (environment.onboarding) { + case "cloud-openclaw": + return await this.cloudOpenClaw(environment, options); + case "cloud-openclaw-no-docker": + return await this.cloudOpenClawNoDocker(environment, options); + default: + throw new Error(`Unsupported onboarding profile '${environment.onboarding}'.`); + } + } + + async cloudOpenClaw(environment: EnvironmentReady, options: OnboardingOptions = {}): Promise { + if (!environment.docker.available) { + throw new Error("cloud-openclaw onboarding requires an available Docker runtime."); + } + const sandboxName = sandboxNameFromOptions(environment.onboarding, options); + const apiKey = this.secrets.required("NVIDIA_API_KEY"); + this.registerSandboxCleanup(sandboxName); + const result = await this.host.nemoclaw(ONBOARD_ARGS, { + artifactName: "onboard-cloud-openclaw", + env: commandEnv(sandboxName, { NVIDIA_API_KEY: apiKey }), + redactionValues: [apiKey], + timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, + }); + assertExitZero(result, "cloud-openclaw onboarding"); + return { + onboarding: environment.onboarding, + sandboxName, + agent: "openclaw", + provider: "nvidia", + providerEnv: "cloud", + gatewayUrl: OPENCLAW_GATEWAY_URL, + result, + }; + } + + async cloudOpenClawNoDocker(environment: EnvironmentReady, options: OnboardingOptions = {}): Promise { + if (environment.docker.expectation !== "missing") { + throw new Error("cloud-openclaw-no-docker onboarding requires the docker-missing runtime expectation."); + } + const sandboxName = sandboxNameFromOptions(environment.onboarding, options); + const apiKey = this.secrets.required("NVIDIA_API_KEY"); + this.registerSandboxCleanup(sandboxName); + const shimDir = await mkdtemp(join(tmpdir(), "e2e-no-docker-")); + const shimPath = join(shimDir, "docker"); + try { + await writeFile(shimPath, noDockerShim(), "utf8"); + await chmod(shimPath, 0o700); + const env = commandEnv(sandboxName, { NVIDIA_API_KEY: apiKey }); + env.PATH = prependPath(shimDir, env.PATH); + const result = await this.host.nemoclaw(ONBOARD_ARGS, { + artifactName: "onboard-cloud-openclaw-no-docker", + env, + redactionValues: [apiKey], + timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, + }); + await this.writeNegativePreflightEvidence(result, [apiKey]); + if (result.exitCode === 0) { + throw new Error("cloud-openclaw-no-docker onboarding unexpectedly succeeded."); + } + if (!hasDockerMissingSignature(result)) { + throw new Error( + `cloud-openclaw-no-docker onboarding failed without Docker-missing preflight signature: ${resultText(result)}`, + ); + } + return { + onboarding: environment.onboarding, + sandboxName, + agent: "openclaw", + provider: "nvidia", + providerEnv: "cloud", + gatewayUrl: OPENCLAW_GATEWAY_URL, + result, + expectedFailure: { + phase: "preflight", + errorClass: "docker-missing", + }, + }; + } finally { + await rm(shimDir, { force: true, recursive: true }); + } + } + + private registerSandboxCleanup(sandboxName: string): void { + if (!this.cleanup) return; + this.cleanup.add(`destroy NemoClaw sandbox ${sandboxName}`, async () => { + const result = await this.host.nemoclaw([sandboxName, "destroy", "--yes"], { + artifactName: `cleanup-destroy-${artifactLabel(sandboxName)}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: DEFAULT_TIMEOUT_MS, + }); + if (result.exitCode !== 0 && !hasMissingSandboxDeleteSignature(result)) { + assertExitZero(result, `cleanup destroy sandbox ${sandboxName}`); + } + }); + } + + private redact(text: string, extraValues: string[] = []): string { + return this.secrets.redact?.(text, extraValues) ?? redactString(redactExplicitValues(text, extraValues)); + } + + private async writeNegativePreflightEvidence(result: ShellProbeResult, redactionValues: string[]): Promise { + const logPath = legacyNegativePreflightLogPath(); + if (!logPath) return; + await mkdir(dirname(logPath), { recursive: true }); + await writeFile(logPath, this.redact(resultText(result), redactionValues), "utf8"); + } +} diff --git a/test/e2e-scenario/nemoclaw_scenarios/onboard/cloud-openclaw-no-docker.sh b/test/e2e-scenario/nemoclaw_scenarios/onboard/cloud-openclaw-no-docker.sh index 9c7b9803f15..3cd545e3287 100755 --- a/test/e2e-scenario/nemoclaw_scenarios/onboard/cloud-openclaw-no-docker.sh +++ b/test/e2e-scenario/nemoclaw_scenarios/onboard/cloud-openclaw-no-docker.sh @@ -13,7 +13,8 @@ # see when Docker is installed but the daemon is not running. # # 2. Running `nemoclaw onboard --non-interactive` with stdout+stderr -# captured to `${E2E_CONTEXT_DIR}/negative-preflight.log`. The +# streamed through a redactor into +# `${E2E_CONTEXT_DIR}/negative-preflight.log`. The # `onboarding.preflight.expected-failed` assertion greps that file. # # 3. Asserting that nemoclaw exits non-zero (preflight DID fail). If @@ -29,14 +30,89 @@ # Pattern mirrors test/e2e/e2e-cloud-experimental/test-port8080-conflict.sh, # which sets up a different failure condition (port 8080 occupied) but # follows the same capture-output / check-exit / grep-log shape. +# +# Migration note: the typed OnboardingPhaseFixture owns the future no-Docker +# path. This shell worker remains the live dispatcher target until that phase +# is fully wired into scenario execution. Keep its Docker-daemon-missing +# signature and redacted negative-preflight evidence contract aligned with the +# typed fixture, then remove both PATH-shadow shims once the framework can +# inject a Docker client boundary directly. + +e2e_no_docker_write_redacted_preflight_log() { + local redacted_log="$1" + rm -f "${redacted_log}" + + if command -v python3 >/dev/null 2>&1; then + python3 -c ' +import os +import re +import sys + +target = sys.argv[1] +secret_env_name = re.compile(r"(api[_-]?key|token|secret|password|credential)", re.I) +secret_pattern = re.compile( + r"(sk-[A-Za-z0-9_-]{8,}|nvapi-[A-Za-z0-9_-]{8,}|[A-Za-z0-9._%+-]+:[A-Za-z0-9_/-]{12,}|(api[_-]?key|token|secret|password)[=:][^\s]+)", + re.I, +) + +with open(target, "w", encoding="utf-8") as handle: + for line in sys.stdin: + for name, value in os.environ.items(): + if value and secret_env_name.search(name): + line = line.replace(value, "[REDACTED]") + + line = secret_pattern.sub("[REDACTED]", line) + handle.write(line) + handle.flush() +' "${redacted_log}" + return + fi + + local redacted text name value lower_name pattern + redacted="$(mktemp -t e2e-negative-preflight-redacted-XXXXXX)" + text="$(cat)" + while IFS='=' read -r name value; do + lower_name="${name,,}" + if [[ -n "${value}" && "${lower_name}" =~ (api[_-]?key|token|secret|password|credential) ]]; then + pattern="${value//\\/\\\\}" + pattern="${pattern//\*/\\*}" + pattern="${pattern//\?/\\?}" + pattern="${pattern//\[/\\[}" + text="${text//${pattern}/[REDACTED]}" + fi + done < <(env) + printf "%s" "${text}" | sed -E 's/(sk-[A-Za-z0-9_-]{8,}|nvapi-[A-Za-z0-9_-]{8,}|[A-Za-z0-9._%+-]+:[A-Za-z0-9_\/-]{12,}|(api[_-]?key|token|secret|password)[=:][^[:space:]]+)/[REDACTED]/Ig' >"${redacted}" + mv "${redacted}" "${redacted_log}" +} + +e2e_no_docker_has_missing_signature() { + local log="$1" + [[ -f "${log}" ]] || return 1 + grep -Eiq \ + 'Cannot connect to the Docker daemon|Is the docker daemon running\??|docker daemon is not running|docker[- ]missing|Docker is required before onboarding|Docker is not reachable|could not talk to the Docker daemon' \ + "${log}" +} e2e_onboard_cloud_openclaw_no_docker() { e2e_env_apply_noninteractive - e2e_context_init + # The TS runner already seeded context.env. Resolve the directory and + # keep existing keys (notably E2E_SANDBOX_NAME) for state-validation. + e2e_context_path >/dev/null + mkdir -p "${E2E_CONTEXT_DIR}" - local log shim_dir rc=0 + local log sandbox_name shim_dir rc=0 redactor_rc=0 shim_dir_quoted run_path log="${E2E_CONTEXT_DIR}/negative-preflight.log" + e2e_context_require E2E_SANDBOX_NAME + sandbox_name="$(e2e_context_get E2E_SANDBOX_NAME)" shim_dir="$(mktemp -d -t e2e-no-docker-XXXXXX)" + printf -v shim_dir_quoted "%q" "${shim_dir}" + # shellcheck disable=SC2064 + trap "rm -rf -- ${shim_dir_quoted}" RETURN EXIT + # shellcheck disable=SC2064 + trap "rm -rf -- ${shim_dir_quoted}; exit 130" INT + # shellcheck disable=SC2064 + trap "rm -rf -- ${shim_dir_quoted}; exit 143" TERM + rm -f "${log}" cat >"${shim_dir}/docker" <<'SHIM' #!/usr/bin/env bash @@ -52,12 +128,32 @@ SHIM echo "negative-preflight: log_file=${log}" echo "negative-preflight: invoking nemoclaw onboard --non-interactive (expected to fail at preflight)" - PATH="${shim_dir}:${PATH}" \ - nemoclaw onboard --non-interactive --yes-i-accept-third-party-software \ - >"${log}" 2>&1 || rc=$? + run_path="${shim_dir}" + if [[ -n "${PATH:-}" ]]; then + run_path="${shim_dir}:${PATH}" + fi + local errexit_was_set=0 + if [[ $- == *e* ]]; then + errexit_was_set=1 + set +e + fi + NEMOCLAW_SANDBOX_NAME="${sandbox_name}" NEMOCLAW_AGENT=openclaw NEMOCLAW_PROVIDER=cloud PATH="${run_path}" \ + nemoclaw onboard --non-interactive --yes --yes-i-accept-third-party-software \ + 2>&1 | e2e_no_docker_write_redacted_preflight_log "${log}" + local -a pipeline_status=("${PIPESTATUS[@]}") + if [[ "${errexit_was_set}" -eq 1 ]]; then + set -e + fi + rc="${pipeline_status[0]}" + redactor_rc="${pipeline_status[1]}" rm -rf "${shim_dir}" + if [[ "${redactor_rc}" -ne 0 ]]; then + echo "negative-preflight: ERROR: failed to write redacted preflight log (${log})" >&2 + return "${redactor_rc}" + fi + echo "negative-preflight: nemoclaw onboard exited ${rc}" if [[ -f "${log}" ]]; then echo "--- captured log tail (${log}) ---" @@ -70,5 +166,10 @@ SHIM return 1 fi + if ! e2e_no_docker_has_missing_signature "${log}"; then + echo "negative-preflight: ERROR: nemoclaw onboard failed without Docker-missing preflight signature" >&2 + return "${rc}" + fi + return 0 } diff --git a/test/e2e-scenario/scenarios/assertions/registry.ts b/test/e2e-scenario/scenarios/assertions/registry.ts index c4457cb9edf..e8af749714c 100644 --- a/test/e2e-scenario/scenarios/assertions/registry.ts +++ b/test/e2e-scenario/scenarios/assertions/registry.ts @@ -48,21 +48,6 @@ function probeStep( }; } -function pendingStep( - id: string, - phase: PhaseName, - ref: string, - options: { required?: boolean } = {}, -): AssertionStep { - return { - id, - phase, - implementation: { kind: "pending", ref }, - evidencePath: `.e2e/assertions/${id}.json`, - required: options.required, - }; -} - function group(input: { id: string; phase: PhaseName; @@ -130,6 +115,10 @@ const smokeSteps = [ shellStep({ id: "runtime.smoke.sandbox-shell", phase: "runtime", ref: "test/e2e-scenario/validation_suites/smoke/03-sandbox-shell.sh", reliability: { timeoutSeconds: 30 } }), ]; +const snapshotSteps = [ + shellStep({ id: "runtime.snapshot.sandbox-listed", phase: "runtime", ref: "test/e2e-scenario/validation_suites/smoke/02-sandbox-listed.sh" }), +]; + const cloudInferenceSteps = [ shellStep({ id: "runtime.inference.models-health", @@ -199,30 +188,6 @@ const ollamaProxySteps = [ }), ]; -export const runtimeControlGroups: AssertionGroup[] = [ - { - id: "runtime.expected-failure.no-side-effects", - phase: "runtime", - description: "Negative scenario runtime check ensuring forbidden side effects did not occur.", - migrationStatus: "complete", - steps: [ - pendingStep( - "runtime.expected-failure.no-side-effects", - "runtime", - "expectedFailureNoSideEffectsProbe", - // Negative scenarios assert that a declared failure mode - // produced no forbidden side effects. Until the side-effect - // validator is implemented, this step must fail closed for - // any scenario that opts into runtimeControlGroups[0] - // (i.e. scenario.expectedFailure is set). Skipping it would - // let negative scenarios silently "pass" without verifying - // their core contract. - { required: true }, - ), - ], - }, -]; - export const validationSuiteGroups: AssertionGroup[] = [ suiteGroup("smoke", smokeSteps), suiteGroup("gateway-health", [smokeSteps[1]]), @@ -280,7 +245,7 @@ export const validationSuiteGroups: AssertionGroup[] = [ shellStep({ id: "lifecycle.sandbox.list-and-status", phase: "runtime", ref: "test/e2e-scenario/validation_suites/sandbox/operations/00-list-and-status.sh" }), shellStep({ id: "lifecycle.sandbox.logs-and-exec", phase: "runtime", ref: "test/e2e-scenario/validation_suites/sandbox/operations/01-logs-and-exec.sh" }), ]), - suiteGroup("snapshot", [shellStep({ id: "lifecycle.snapshot.create-list-restore", phase: "runtime", ref: "test/e2e-scenario/validation_suites/sandbox/snapshot/00-create-list-restore.sh" })]), + suiteGroup("snapshot", snapshotSteps), suiteGroup("snapshot-lifecycle", [shellStep({ id: "lifecycle.snapshot.create-list-restore", phase: "runtime", ref: "test/e2e-scenario/validation_suites/sandbox/snapshot/00-create-list-restore.sh" })]), suiteGroup("rebuild", [ shellStep({ id: "lifecycle.rebuild.state-preserved", phase: "runtime", ref: "test/e2e-scenario/validation_suites/rebuild_upgrade/00-state-preserved.sh", reliability: { timeoutSeconds: 120, retry: { attempts: 2, on: ["runner-infra"] } } }), @@ -300,7 +265,7 @@ export const validationSuiteGroups: AssertionGroup[] = [ ]; export const assertionRegistry = { - groups: [...onboardingAssertionGroups, ...runtimeControlGroups, ...validationSuiteGroups], + groups: [...onboardingAssertionGroups, ...validationSuiteGroups], }; export function assertionGroupForSuite(suiteId: string): AssertionGroup | undefined { @@ -410,7 +375,6 @@ export function assertionGroupsForScenario(scenario: ScenarioDefinition): Assert ...onboardingGroups, ...suiteGroups, ...supplementalGroups, - scenario.expectedFailure ? runtimeControlGroups[0] : undefined, ]; return uniqueGroups(groups.filter((entry): entry is AssertionGroup => Boolean(entry))); } diff --git a/test/e2e-scenario/scenarios/orchestrators/negative-matcher.ts b/test/e2e-scenario/scenarios/orchestrators/negative-matcher.ts index dbbe2b0956d..d2dbb85d85d 100644 --- a/test/e2e-scenario/scenarios/orchestrators/negative-matcher.ts +++ b/test/e2e-scenario/scenarios/orchestrators/negative-matcher.ts @@ -20,13 +20,10 @@ import type { // matcher's job. The matcher only inspects what actually happened. // - Forbidden-side-effect verification (did a sandbox actually get // created when the scenario forbids it?) belongs to the -// `expectedFailureNoSideEffectsProbe` implementation registered as -// a probe step. Until that probe lands, the runtime control group -// keeps the negative scenario visibly red via a `required: true` -// pending step. The matcher reports the contract status for -// phase + errorClass independently of the side-effect probe, and -// exposes whether forbiddenSideEffects were declared so callers can -// integrate both signals. +// state-validation phase. The matcher reports the contract status +// for phase + errorClass independently from those post-condition +// probes so callers can combine the signals without confusing the +// originating failure with forbidden side effects. export type NegativeContractMatchOutcome = // Right phase, right errorClass match observed. @@ -45,6 +42,9 @@ export interface NegativeContractObservation { failedActionMessage?: string; failedAssertionId?: string; failedAssertionMessage?: string; + handledActionId?: string; + handledAssertionId?: string; + handledMessage?: string; } export interface NegativeContractResult { @@ -120,7 +120,10 @@ function findFirstObservedFailure(results: readonly PhaseResult[]): NegativeCont }; } const failedAssertion = result.assertions.find( - (assertion) => assertion.status === "failed" && assertion.id !== SIDE_EFFECT_PROBE_STEP_ID, + (assertion) => + assertion.status === "failed" && + assertion.id !== SIDE_EFFECT_PROBE_STEP_ID && + !STATE_VALIDATION_FORBIDDEN_PROBE_IDS.has(assertion.id), ); if (failedAssertion) { return { @@ -133,6 +136,24 @@ function findFirstObservedFailure(results: readonly PhaseResult[]): NegativeCont return undefined; } +function normalizeClass(value: string): string { + return value.toLowerCase().replace(/[\s_-]+/g, "-"); +} + +function errorClassVariants(errorClass: string): string[] { + const normalized = normalizeClass(errorClass); + switch (normalized) { + case "docker-missing": + return [normalized, "no-docker"]; + case "invalid-nvidia-api-key": + return [normalized, "invalid-nvidia-key", "invalid-key"]; + case "gateway-port-conflict": + return [normalized, "port-conflict"]; + default: + return [normalized]; + } +} + function errorClassMatches(message: string | undefined, errorClass: string): boolean { if (!message) { return false; @@ -143,8 +164,49 @@ function errorClassMatches(message: string | undefined, errorClass: string): boo // string or a normalized form where dashes/underscores/spaces are // interchangeable. This stays a pure string check so the matcher // can be fully tested in isolation. - const normalize = (value: string): string => value.toLowerCase().replace(/[\s_-]+/g, "-"); - return normalize(message).includes(normalize(errorClass)); + const normalizedMessage = normalizeClass(message); + return errorClassVariants(errorClass).some((variant) => normalizedMessage.includes(variant)); +} + +function findHandledExpectedFailure( + expected: ExpectedFailureContract, + expectedPhase: PhaseName, + results: readonly PhaseResult[], +): NegativeContractObservation | undefined { + const phaseResult = results.find((result) => result.phase === expectedPhase); + if (!phaseResult || phaseResult.status !== "passed") { + return undefined; + } + + const passedAssertion = phaseResult.assertions.find((assertion) => { + if (assertion.status !== "passed") return false; + const text = [assertion.id, assertion.message].filter(Boolean).join(" "); + return ( + errorClassMatches(text, expected.errorClass) || + (expected.phase === "preflight" && assertion.id === "onboarding.preflight.expected-failed") + ); + }); + if (passedAssertion) { + return { + failedPhase: expectedPhase, + handledAssertionId: passedAssertion.id, + handledMessage: + passedAssertion.message ?? `expected failure assertion passed: ${expected.errorClass}`, + }; + } + + const passedAction = phaseResult.actions.find((action) => { + if (action.status !== "passed") return false; + return errorClassMatches([action.id, action.message].filter(Boolean).join(" "), expected.errorClass); + }); + if (passedAction) { + return { + failedPhase: expectedPhase, + handledActionId: passedAction.id, + handledMessage: passedAction.message ?? passedAction.id, + }; + } + return undefined; } function describeObservation(observation: NegativeContractObservation): string { @@ -158,7 +220,13 @@ function describeObservation(observation: NegativeContractObservation): string { if (observation.failedAssertionId) { parts.push(`assertion=${observation.failedAssertionId}`); } - const message = observation.failedActionMessage ?? observation.failedAssertionMessage; + if (observation.handledActionId) { + parts.push(`handledAction=${observation.handledActionId}`); + } + if (observation.handledAssertionId) { + parts.push(`handledAssertion=${observation.handledAssertionId}`); + } + const message = observation.failedActionMessage ?? observation.failedAssertionMessage ?? observation.handledMessage; if (message) { parts.push(`message="${message.slice(0, 240)}"`); } @@ -173,7 +241,7 @@ export function evaluateNegativeContract(plan: RunPlan, results: readonly PhaseR ); } const expectedPhase = resolveExpectedPhase(expected.phase); - const observation = findFirstObservedFailure(results); + const observation = findFirstObservedFailure(results) ?? findHandledExpectedFailure(expected, expectedPhase, results); if (!observation) { return { @@ -195,7 +263,12 @@ export function evaluateNegativeContract(plan: RunPlan, results: readonly PhaseR }; } - const observedMessage = observation.failedActionMessage ?? observation.failedAssertionMessage; + const observedMessage = + observation.failedActionMessage ?? + observation.failedAssertionMessage ?? + observation.handledMessage ?? + observation.handledActionId ?? + observation.handledAssertionId; if (!errorClassMatches(observedMessage, expected.errorClass)) { return { matched: false, diff --git a/test/e2e-scenario/scenarios/orchestrators/runner.ts b/test/e2e-scenario/scenarios/orchestrators/runner.ts index 02be1b195f5..d91c1b535e2 100644 --- a/test/e2e-scenario/scenarios/orchestrators/runner.ts +++ b/test/e2e-scenario/scenarios/orchestrators/runner.ts @@ -79,8 +79,7 @@ export class ScenarioRunner { // if the plan declared expectedFailure, evaluate the matcher and // append a synthetic phase result. Positive scenarios are // unaffected. Side-effect verification stays the responsibility of - // the runtime control group's required pending step (kept red - // until the probe lands); the matcher only judges phase + errorClass. + // the state-validation phase; the matcher only judges phase + errorClass. if (plan.expectedFailure) { const contractResult = evaluateNegativeContract(plan, results); const synthetic = negativeContractPhaseResult(contractResult); diff --git a/test/e2e-scenario/scenarios/run.ts b/test/e2e-scenario/scenarios/run.ts index ff9fb056c49..e641a79c457 100644 --- a/test/e2e-scenario/scenarios/run.ts +++ b/test/e2e-scenario/scenarios/run.ts @@ -175,16 +175,24 @@ async function main() { // A scenario fails iff: // positive (no expectedFailure): any phase result failed. // negative (expectedFailure declared): the synthetic -// negative-contract phase did not match, OR the runtime -// control group's required side-effect step did not pass. +// negative-contract phase did not match, OR state-validation +// did not prove the declared forbidden side effects stayed absent. // // The matcher decides exit code for negatives so that a scenario // that failed for the right reason in the right phase is no longer -// reported as red just because setup did not complete. Until the -// forbidden-side-effect probe lands, the required pending step in -// runtimeControlGroups keeps negatives visibly red on the side-effect -// axis even when phase + errorClass match. -function planFailed(plan: import("./types.ts").RunPlan, results: PhaseResult[]): boolean { +// reported as red just because setup did not complete. Forbidden +// side effects stay visible through the typed state-validation probes. +function forbiddenSideEffectProbeId(sideEffect: string): string { + if (sideEffect === "gateway-started") { + return "state-validation.gateway-absent"; + } + if (sideEffect === "sandbox-created") { + return "state-validation.sandbox-absent"; + } + return `state-validation.${sideEffect}`; +} + +export function planFailed(plan: import("./types.ts").RunPlan, results: PhaseResult[]): boolean { if (!plan.expectedFailure) { return results.some((result) => result.status === "failed"); } @@ -192,14 +200,18 @@ function planFailed(plan: import("./types.ts").RunPlan, results: PhaseResult[]): if (!contractPhase || contractPhase.status !== "passed") { return true; } - const runtime = results.find((result) => result.phase === "runtime"); - const sideEffectStep = runtime?.assertions.find( - (assertion) => assertion.id === "runtime.expected-failure.no-side-effects", - ); - if (!sideEffectStep || sideEffectStep.status !== "passed") { + const requiredSideEffectProbes = (plan.expectedFailure.forbiddenSideEffects ?? []).map(forbiddenSideEffectProbeId); + if (requiredSideEffectProbes.length === 0) { + return false; + } + const stateValidation = results.find((result) => result.phase === "state-validation"); + if (!stateValidation || stateValidation.status !== "passed") { return true; } - return false; + const passedActionIds = new Set( + stateValidation.actions.filter((action) => action.status === "passed").map((action) => action.id), + ); + return requiredSideEffectProbes.some((id) => !passedActionIds.has(id)); } // Only execute when invoked directly as a script. Importing this module from