test(e2e): add onboarding phase fixture

test/e2e-scenario/framework-tests/e2e-assertion-modules.test.ts

@@ -60,6 +60,19 @@ describe("assertion modules", () => {
     }
   });
 
+  it("test_should_keep_snapshot_suite_distinct_from_snapshot_lifecycle", () => {
+    const snapshot = assertionGroupForSuite("snapshot");
+    const snapshotLifecycle = assertionGroupForSuite("snapshot-lifecycle");
+
+    expect(snapshot?.steps.map((step) => step.id)).toEqual(["runtime.snapshot.sandbox-listed"]);
+    expect(snapshot?.steps.map((step) => step.implementation?.ref)).toEqual([
+      "test/e2e-scenario/validation_suites/smoke/02-sandbox-listed.sh",
+    ]);
+    expect(snapshotLifecycle?.steps.map((step) => step.implementation?.ref)).toEqual([
+      "test/e2e-scenario/validation_suites/sandbox/snapshot/00-create-list-restore.sh",
+    ]);
+  });
+
   it("test_should_require_each_assertion_group_to_have_steps", () => {
     const emptyGroup: AssertionGroup = { id: "empty", phase: "runtime", steps: [] };
 

test/e2e-scenario/framework-tests/e2e-lib-helpers.test.ts

@@ -15,6 +15,7 @@ const ASSERT = path.join(VALIDATION_SUITES, "assert");
 const REBUILD_UPGRADE_LIB = path.join(VALIDATION_SUITES, "lib/rebuild_upgrade.sh");
 const FIXTURES = path.join(REPO_ROOT, "test/e2e-scenario/nemoclaw_scenarios/fixtures");
 const INSTALL_DIR = path.join(REPO_ROOT, "test/e2e-scenario/nemoclaw_scenarios/install");
+const ONBOARD_DIR = path.join(REPO_ROOT, "test/e2e-scenario/nemoclaw_scenarios/onboard");
 
 function runBash(script: string, env: Record<string, string> = {}): SpawnSyncReturns<string> {
   return spawnSync("bash", ["-c", script], {
@@ -60,6 +61,192 @@ describe("E2E shell helpers", () => {
     }
   });
 
+  it("no_docker_onboarding_worker_should_preserve_seeded_context_and_redact_log", () => {
+    const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-no-docker-context-"));
+    const fakeBin = path.join(tmp, "bin");
+    fs.mkdirSync(fakeBin);
+    fs.writeFileSync(
+      path.join(fakeBin, "nemoclaw"),
+      `#!/usr/bin/env bash
+if [[ "\${1:-}" = "onboard" ]]; then
+  expected='onboard --non-interactive --yes --yes-i-accept-third-party-software'
+  if [[ "$*" != "\${expected}" ]]; then
+    echo "unexpected nemoclaw args: $*" >&2
+    exit 2
+  fi
+  if [[ "\${NEMOCLAW_AGENT:-}" != "openclaw" || "\${NEMOCLAW_PROVIDER:-}" != "cloud" || "\${NEMOCLAW_SANDBOX_NAME:-}" != "e2e-preserved" ]]; then
+    echo "unexpected nemoclaw env: agent=\${NEMOCLAW_AGENT:-unset} provider=\${NEMOCLAW_PROVIDER:-unset} sandbox=\${NEMOCLAW_SANDBOX_NAME:-unset}" >&2
+    exit 2
+  fi
+  echo "NVIDIA_API_KEY=\${NVIDIA_API_KEY:-unset}" >&2
+  echo "Docker is required before onboarding" >&2
+  exit 42
+fi
+echo "unexpected nemoclaw invocation: $*" >&2
+exit 2
+`,
+      { mode: 0o755 },
+    );
+    try {
+      fs.writeFileSync(
+        path.join(tmp, "context.env"),
+        "E2E_SCENARIO=ubuntu-no-docker-preflight-negative\nE2E_SANDBOX_NAME=e2e-preserved\n",
+      );
+      const r = runBash(
+        `
+        set -euo pipefail
+        test/e2e-scenario/nemoclaw_scenarios/dispatch-action.sh e2e_onboard cloud-openclaw-no-docker "${ONBOARD_DIR}/dispatch.sh"
+      `,
+        {
+          E2E_ACTION_ID: "onboarding.profile.cloud-openclaw-no-docker",
+          E2E_CONTEXT_DIR: tmp,
+          E2E_PHASE: "onboarding",
+          NVIDIA_API_KEY: "secret-token",
+          PATH: `${fakeBin}:${process.env.PATH ?? ""}`,
+          TMPDIR: tmp,
+        },
+      );
+      expect(r.status, `${r.stdout}\n${r.stderr}`).toBe(0);
+      const contextBody = fs.readFileSync(path.join(tmp, "context.env"), "utf8");
+      expect(contextBody).toMatch(/^E2E_SANDBOX_NAME=e2e-preserved$/m);
+      const logBody = fs.readFileSync(path.join(tmp, "negative-preflight.log"), "utf8");
+      expect(logBody).toContain("Docker is required before onboarding");
+      expect(logBody).toContain("[REDACTED]");
+      expect(logBody).not.toContain("secret-token");
+      const tempEntries = fs.readdirSync(tmp, { recursive: true }).map(String).join("\n");
+      expect(tempEntries).not.toContain("negative-preflight.raw.log");
+    } finally {
+      fs.rmSync(tmp, { recursive: true, force: true });
+    }
+  });
+
+  it("no_docker_onboarding_worker_should_fail_on_unrelated_onboarding_errors", () => {
+    const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-no-docker-unrelated-"));
+    const fakeBin = path.join(tmp, "bin");
+    fs.mkdirSync(fakeBin);
+    fs.writeFileSync(
+      path.join(fakeBin, "nemoclaw"),
+      `#!/usr/bin/env bash
+if [[ "\${1:-}" = "onboard" ]]; then
+  expected='onboard --non-interactive --yes --yes-i-accept-third-party-software'
+  if [[ "$*" != "\${expected}" ]]; then
+    echo "unexpected nemoclaw args: $*" >&2
+    exit 2
+  fi
+  if [[ "\${NEMOCLAW_AGENT:-}" != "openclaw" || "\${NEMOCLAW_PROVIDER:-}" != "cloud" || "\${NEMOCLAW_SANDBOX_NAME:-}" != "e2e-preserved" ]]; then
+    echo "unexpected nemoclaw env: agent=\${NEMOCLAW_AGENT:-unset} provider=\${NEMOCLAW_PROVIDER:-unset} sandbox=\${NEMOCLAW_SANDBOX_NAME:-unset}" >&2
+    exit 2
+  fi
+  echo "provider rejected NVIDIA_API_KEY=\${NVIDIA_API_KEY:-unset}" >&2
+  exit 42
+fi
+echo "unexpected nemoclaw invocation: $*" >&2
+exit 2
+`,
+      { mode: 0o755 },
+    );
+    try {
+      fs.writeFileSync(
+        path.join(tmp, "context.env"),
+        "E2E_SCENARIO=ubuntu-no-docker-preflight-negative\nE2E_SANDBOX_NAME=e2e-preserved\n",
+      );
+      const r = runBash(
+        `
+        set -euo pipefail
+        test/e2e-scenario/nemoclaw_scenarios/dispatch-action.sh e2e_onboard cloud-openclaw-no-docker "${ONBOARD_DIR}/dispatch.sh"
+      `,
+        {
+          E2E_ACTION_ID: "onboarding.profile.cloud-openclaw-no-docker",
+          E2E_CONTEXT_DIR: tmp,
+          E2E_PHASE: "onboarding",
+          NVIDIA_API_KEY: "secret-token",
+          PATH: `${fakeBin}:${process.env.PATH ?? ""}`,
+          TMPDIR: tmp,
+        },
+      );
+      expect(r.status).toBe(42);
+      expect(`${r.stdout}\n${r.stderr}`).toContain("failed without Docker-missing preflight signature");
+      const logBody = fs.readFileSync(path.join(tmp, "negative-preflight.log"), "utf8");
+      expect(logBody).toContain("provider rejected");
+      expect(logBody).toContain("[REDACTED]");
+      expect(logBody).not.toContain("secret-token");
+    } finally {
+      fs.rmSync(tmp, { recursive: true, force: true });
+    }
+  });
+
+  it("no_docker_onboarding_worker_should_accept_current_preflight_wording", () => {
+    const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-no-docker-wording-"));
+    const fakeBin = path.join(tmp, "bin");
+    fs.mkdirSync(fakeBin);
+    fs.writeFileSync(
+      path.join(fakeBin, "nemoclaw"),
+      `#!/usr/bin/env bash
+if [[ "\${1:-}" = "onboard" ]]; then
+  echo "Docker is not reachable. Please fix Docker and try again." >&2
+  exit 1
+fi
+echo "unexpected nemoclaw invocation: $*" >&2
+exit 2
+`,
+      { mode: 0o755 },
+    );
+    try {
+      fs.writeFileSync(
+        path.join(tmp, "context.env"),
+        "E2E_SCENARIO=ubuntu-no-docker-preflight-negative\nE2E_SANDBOX_NAME=e2e-preserved\n",
+      );
+      const r = runBash(
+        `
+        set -euo pipefail
+        test/e2e-scenario/nemoclaw_scenarios/dispatch-action.sh e2e_onboard cloud-openclaw-no-docker "${ONBOARD_DIR}/dispatch.sh"
+      `,
+        {
+          E2E_ACTION_ID: "onboarding.profile.cloud-openclaw-no-docker",
+          E2E_CONTEXT_DIR: tmp,
+          E2E_PHASE: "onboarding",
+          NVIDIA_API_KEY: "secret-token",
+          PATH: `${fakeBin}:${process.env.PATH ?? ""}`,
+          TMPDIR: tmp,
+        },
+      );
+      expect(r.status, `${r.stdout}\n${r.stderr}`).toBe(0);
+      const logBody = fs.readFileSync(path.join(tmp, "negative-preflight.log"), "utf8");
+      expect(logBody).toContain("Docker is not reachable");
+    } finally {
+      fs.rmSync(tmp, { recursive: true, force: true });
+    }
+  });
+
+  it("no_docker_redactor_fallback_should_redact_sensitive_env_values_without_python", () => {
+    const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-no-docker-redactor-"));
+    const noPythonBin = path.join(tmp, "bin");
+    const logPath = path.join(tmp, "negative-preflight.log");
+    try {
+      const r = runBash(
+        `
+        set -euo pipefail
+        mkdir -p "${noPythonBin}"
+        for cmd in rm mktemp sed env cat mv; do
+          ln -s "$(command -v "\${cmd}")" "${noPythonBin}/\${cmd}"
+        done
+        . "${ONBOARD_DIR}/cloud-openclaw-no-docker.sh"
+        export NVIDIA_API_KEY=plain-secret-value
+        PATH="${noPythonBin}"
+        printf 'plain-secret-value\\nDocker is required before onboarding\\n' | e2e_no_docker_write_redacted_preflight_log "${logPath}"
+      `,
+        { TMPDIR: tmp },
+      );
+      expect(r.status, `${r.stdout}\n${r.stderr}`).toBe(0);
+      const logBody = fs.readFileSync(logPath, "utf8");
+      expect(logBody).toContain("[REDACTED]");
+      expect(logBody).toContain("Docker is required before onboarding");
+      expect(logBody).not.toContain("plain-secret-value");
+    } finally {
+      fs.rmSync(tmp, { recursive: true, force: true });
+    }
+  });
+
   it("security_policy_credentials_helper_should_load_with_context_library", () => {
     const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-context-"));
     try {

test/e2e-scenario/framework-tests/e2e-negative-matcher.test.ts

@@ -13,6 +13,7 @@ import {
 } from "../scenarios/orchestrators/negative-matcher.ts";
 import { ScenarioRunner } from "../scenarios/orchestrators/runner.ts";
 import { listScenarios } from "../scenarios/registry.ts";
+import { planFailed } from "../scenarios/run.ts";
 import type {
   ExpectedFailureContract,
   PhaseName,
@@ -75,6 +76,35 @@ function phaseResult(
   };
 }
 
+function passedNegativeContractPhase(): PhaseResult {
+  return {
+    phase: "negative-contract",
+    status: "passed",
+    actions: [],
+    assertions: [
+      {
+        id: "negative-contract.match",
+        status: "passed",
+        attempts: 1,
+        durationMs: 0,
+        message: "matched",
+      },
+    ],
+  };
+}
+
+function stateValidationResult(
+  status: PhaseResult["status"],
+  actionIds: string[] = ["state-validation.gateway-absent", "state-validation.sandbox-absent"],
+): PhaseResult {
+  return {
+    phase: "state-validation",
+    status,
+    actions: actionIds.map((id) => ({ id, status: "passed", durationMs: 1 })),
+    assertions: [],
+  };
+}
+
 describe("evaluateNegativeContract - phase + errorClass matching", () => {
   it("matches when expected phase fails with the declared errorClass", () => {
     const plan = planWithExpectedFailure({
@@ -127,6 +157,70 @@ describe("evaluateNegativeContract - phase + errorClass matching", () => {
     expect(result.message).toMatch(/all phases passed/);
   });
 
+  it("matches when a passed expected-failure assertion handled the failure", () => {
+    const plan = planWithExpectedFailure({
+      phase: "preflight",
+      errorClass: "docker-missing",
+      forbiddenSideEffects: ["gateway-started", "sandbox-created"],
+    });
+    const results: PhaseResult[] = [
+      phaseResult("environment", { status: "passed" }),
+      {
+        phase: "onboarding",
+        status: "passed",
+        actions: [
+          {
+            id: "onboarding.profile.cloud-openclaw-no-docker",
+            status: "passed",
+            durationMs: 1,
+          },
+        ],
+        assertions: [
+          {
+            id: "onboarding.preflight.expected-failed",
+            status: "passed",
+            attempts: 1,
+            durationMs: 1,
+          },
+        ],
+      },
+      phaseResult("state-validation", { status: "passed" }),
+    ];
+
+    const result = evaluateNegativeContract(plan, results);
+    expect(result.matched).toBe(true);
+    expect(result.outcome).toBe("matched");
+    expect(result.observed).toMatchObject({
+      failedPhase: "onboarding",
+      handledAssertionId: "onboarding.preflight.expected-failed",
+    });
+  });
+
+  it("matches handled expected-failure actions using scenario error-class aliases", () => {
+    const plan = planWithExpectedFailure({
+      phase: "onboarding",
+      errorClass: "invalid-nvidia-api-key",
+    });
+    const results: PhaseResult[] = [
+      {
+        phase: "onboarding",
+        status: "passed",
+        actions: [
+          {
+            id: "onboarding.profile.cloud-openclaw-invalid-nvidia-key",
+            status: "passed",
+            durationMs: 1,
+          },
+        ],
+        assertions: [],
+      },
+    ];
+
+    const result = evaluateNegativeContract(plan, results);
+    expect(result.matched).toBe(true);
+    expect(result.observed.handledActionId).toBe("onboarding.profile.cloud-openclaw-invalid-nvidia-key");
+  });
+
   it("fails when the wrong phase failed", () => {
     const plan = planWithExpectedFailure({ phase: "onboarding", errorClass: "docker-missing" });
     const results: PhaseResult[] = [
@@ -228,6 +322,35 @@ describe("evaluateNegativeContract - phase + errorClass matching", () => {
   });
 });
 
+describe("negative plan exit-code contract", () => {
+  const plan = planWithExpectedFailure({
+    phase: "preflight",
+    errorClass: "docker-missing",
+    forbiddenSideEffects: ["gateway-started", "sandbox-created"],
+  });
+
+  it("passes when negative contract and forbidden-side-effect probes pass", () => {
+    expect(planFailed(plan, [passedNegativeContractPhase(), stateValidationResult("passed")])).toBe(false);
+  });
+
+  it("fails when state-validation is missing", () => {
+    expect(planFailed(plan, [passedNegativeContractPhase()])).toBe(true);
+  });
+
+  it("fails when state-validation is skipped", () => {
+    expect(planFailed(plan, [passedNegativeContractPhase(), stateValidationResult("skipped")])).toBe(true);
+  });
+
+  it("fails when a declared forbidden-side-effect probe did not run", () => {
+    expect(
+      planFailed(plan, [
+        passedNegativeContractPhase(),
+        stateValidationResult("passed", ["state-validation.gateway-absent"]),
+      ]),
+    ).toBe(true);
+  });
+});
+
 describe("ScenarioRunner appends negative-contract phase", () => {
   it("invokes matcher and appends a passing synthetic phase when contract matched", async () => {
     const ctx = freshCtx();
@@ -352,21 +475,15 @@ describe("ScenarioRunner appends negative-contract phase", () => {
   });
 });
 
-describe("registry contract: every negative scenario opts into the side-effect probe", () => {
-  it("scenario.expectedFailure implies the runtime no-side-effects required pending step", () => {
+describe("registry contract: negative scenarios use typed state-validation side-effect probes", () => {
+  it("scenario.expectedFailure does not inject the legacy runtime no-side-effects pending step", () => {
     const negatives = listScenarios().filter((scenario) => scenario.expectedFailure);
     expect(negatives.length).toBeGreaterThan(0);
     for (const scenario of negatives) {
-      const runtimeGroups = scenario.assertionGroups.filter((group) => group.phase === "runtime");
-      const hasProbeStep = runtimeGroups.some((group) =>
-        group.steps.some(
-          (step) =>
-            step.id === "runtime.expected-failure.no-side-effects" &&
-            step.implementation?.kind === "pending" &&
-            step.required === true,
-        ),
+      const hasLegacyPendingStep = scenario.assertionGroups.some((group) =>
+        group.steps.some((step) => step.id === "runtime.expected-failure.no-side-effects"),
       );
-      expect(hasProbeStep, `scenario ${scenario.id} must include the required side-effect pending step`).toBe(true);
+      expect(hasLegacyPendingStep, `scenario ${scenario.id} must rely on state-validation, not the legacy pending step`).toBe(false);
     }
   });
 });