diff --git a/test/e2e-scenario/framework-tests/e2e-clients.test.ts b/test/e2e-scenario/framework-tests/e2e-clients.test.ts index b652c6e102e..429ef2c88db 100644 --- a/test/e2e-scenario/framework-tests/e2e-clients.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-clients.test.ts @@ -106,6 +106,30 @@ describe("E2E fixture clients", () => { }); }); + it("gateway client preserves caller-provided probe options", async () => { + const runner = new FakeRunner(); + const host = new HostCliClient(runner, { cliPath: "nemoclaw" }); + const gateway = new GatewayClient(host); + + await gateway.status({ + artifactName: "custom-gateway-status", + env: { NEMOCLAW_TEST_VALUE: "1" }, + inheritEnv: true, + timeoutMs: 123, + }); + + expect(runner.calls[0]).toEqual({ + command: "nemoclaw", + args: ["gateway", "status"], + options: { + artifactName: "custom-gateway-status", + env: { NEMOCLAW_TEST_VALUE: "1" }, + inheritEnv: true, + timeoutMs: 123, + }, + }); + }); + it("sandbox client builds OpenShell sandbox commands", async () => { const runner = new FakeRunner(); const sandbox = new SandboxClient(runner, { openshellPath: "openshell" }); @@ -121,6 +145,29 @@ describe("E2E fixture clients", () => { }); }); + it("sandbox client preserves caller-provided probe options", async () => { + const runner = new FakeRunner(); + const sandbox = new SandboxClient(runner, { openshellPath: "openshell" }); + + await sandbox.status("assistant", { + artifactName: "custom-sandbox-status", + env: { NEMOCLAW_TEST_VALUE: "1" }, + inheritEnv: true, + timeoutMs: 123, + }); + + expect(runner.calls[0]).toEqual({ + command: "openshell", + args: ["sandbox", "status", "assistant"], + options: { + artifactName: "custom-sandbox-status", + env: { NEMOCLAW_TEST_VALUE: "1" }, + inheritEnv: true, + timeoutMs: 123, + }, + }); + }); + it("sandbox client rejects flag-shaped sandbox names before command construction", async () => { const runner = new FakeRunner(); const sandbox = new SandboxClient(runner, { openshellPath: "openshell" }); diff --git a/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts new file mode 100644 index 00000000000..ecf66f18d12 --- /dev/null +++ b/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts @@ -0,0 +1,448 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, expectTypeOf, it } from "vitest"; + +import { + GatewayClient, + HostCliClient, + SandboxClient, + type CommandRunner, +} from "../framework/clients/index.ts"; +import type { E2EScenarioFixtures } from "../framework/e2e-test.ts"; +import { StateValidationPhaseFixture, type NemoClawInstance } from "../framework/phases/index.ts"; +import type { ShellProbeResult, ShellProbeRunOptions, TrustedShellCommand } from "../framework/shell-probe.ts"; + +interface RunnerCall { + command: string; + args: string[]; + options?: ShellProbeRunOptions; +} + +const GATEWAY_HEALTH_CURL_ARGS = [ + "-fsS", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "--max-time", + "5", + "http://127.0.0.1:18789/health", +]; +const GATEWAY_BASE_CURL_ARGS = [ + "-fsS", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "--max-time", + "5", + "http://127.0.0.1:18789/", +]; +const GATEWAY_ABSENT_HEALTH_CURL_ARGS = [ + "-fsS", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "--max-time", + "3", + "http://127.0.0.1:18789/health", +]; + +function shellResult(exitCode: number, output = ""): ShellProbeResult { + return { + command: [], + exitCode, + signal: null, + timedOut: false, + stdout: exitCode === 0 ? output : "", + stderr: exitCode === 0 ? "" : output, + artifacts: { + stdout: "/tmp/stdout.txt", + stderr: "/tmp/stderr.txt", + result: "/tmp/result.json", + }, + }; +} + +class FakeRunner implements CommandRunner { + readonly calls: RunnerCall[] = []; + private readonly responses: Array = []; + + enqueue(response: ShellProbeResult): void { + this.responses.push(response); + } + + enqueueError(error: Error): void { + this.responses.push(error); + } + + async run(command: TrustedShellCommand, options?: ShellProbeRunOptions): Promise { + this.calls.push({ command: command.command, args: [...command.args], options }); + const response = this.responses.shift(); + if (response instanceof Error) { + throw response; + } + return response ?? shellResult(0); + } +} + +function instance(overrides: Partial = {}): NemoClawInstance { + return { + onboarding: "cloud-openclaw", + sandboxName: "e2e-ubuntu-repo-cloud-openclaw", + agent: "openclaw", + provider: "nvidia", + providerEnv: "cloud", + gatewayUrl: "http://127.0.0.1:18789", + result: shellResult(0), + ...overrides, + }; +} + +function fixture(runner: FakeRunner): StateValidationPhaseFixture { + const host = new HostCliClient(runner); + return new StateValidationPhaseFixture(host, new GatewayClient(host), new SandboxClient(runner)); +} + +describe("state-validation phase fixture", () => { + it("validates a ready expected state through CLI, gateway health, and sandbox registry probes", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(0, "200")); + runner.enqueue(shellResult(0, "NAME\ne2e-ubuntu-repo-cloud-openclaw\n")); + + const result = await fixture(runner).from("cloud-openclaw-ready", instance()); + + expect(result.state.id).toBe("cloud-openclaw-ready"); + expect(result.probes.map((probe) => probe.id)).toEqual([ + "cli-installed", + "gateway-healthy", + "sandbox-running", + ]); + expect(runner.calls).toEqual([ + { + command: "nemoclaw", + args: ["--version"], + options: { + artifactName: "nemoclaw-version", + env: expect.objectContaining({ + PATH: expect.any(String), + }), + }, + }, + { + command: "curl", + args: GATEWAY_HEALTH_CURL_ARGS, + options: { + artifactName: "gateway-health", + env: expect.objectContaining({ + PATH: expect.any(String), + }), + redactionValues: ["http://127.0.0.1:18789/health"], + }, + }, + { + command: "nemoclaw", + args: ["list"], + options: { + artifactName: "sandbox-running-nemoclaw-list", + env: expect.objectContaining({ + PATH: expect.any(String), + }), + }, + }, + ]); + }); + + it("accepts a healthy gateway base URL fallback when the health endpoint is unavailable", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(7, "connection refused")); + runner.enqueue(shellResult(0, "204")); + runner.enqueue(shellResult(0, "NAME\ne2e-ubuntu-repo-cloud-openclaw\n")); + + const result = await fixture(runner).from("cloud-openclaw-ready", instance()); + + expect(result.probes.find((probe) => probe.id === "gateway-healthy")?.results).toHaveLength(2); + expect(runner.calls.map((call) => call.args)).toEqual([ + ["--version"], + GATEWAY_HEALTH_CURL_ARGS, + GATEWAY_BASE_CURL_ARGS, + ["list"], + ]); + }); + + it("accepts the sandbox-local Ollama gateway fallback", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(7, "connection refused")); + runner.enqueue(shellResult(7, "connection refused")); + runner.enqueue(shellResult(0, "401")); + runner.enqueue(shellResult(0, "NAME\ne2e-ubuntu-repo-cloud-openclaw\n")); + + const result = await fixture(runner).from( + "local-ollama-openclaw-ready", + instance({ + provider: "ollama", + providerEnv: "local", + }), + ); + + expect(result.probes.find((probe) => probe.id === "gateway-healthy")?.results).toHaveLength(3); + expect(runner.calls[3]).toMatchObject({ + command: "openshell", + args: [ + "sandbox", + "exec", + "e2e-ubuntu-repo-cloud-openclaw", + "--", + "curl", + "-sS", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "--max-time", + "5", + "http://localhost:18789/health", + ], + }); + }); + + it("fails a gateway-healthy probe if the gateway HTTP probes are unhealthy", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(7, "connection refused")); + runner.enqueue(shellResult(7, "connection refused")); + + await expect(fixture(runner).from("cloud-openclaw-ready", instance())).rejects.toThrow( + /expected gateway .* to be healthy/, + ); + expect(runner.calls.map((call) => call.args)).toEqual([ + ["--version"], + GATEWAY_HEALTH_CURL_ARGS, + GATEWAY_BASE_CURL_ARGS, + ]); + }); + + it("fails a sandbox-running probe if NemoClaw does not list the sandbox", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(0, "200")); + runner.enqueue(shellResult(0, "NAME\nother-sandbox\n")); + + await expect(fixture(runner).from("cloud-openclaw-ready", instance())).rejects.toThrow( + /nemoclaw did not list it/, + ); + }); + + it("validates an expected preflight failure with absent gateway and sandbox probes", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(1, "gateway stopped")); + runner.enqueue(shellResult(7, "connection refused")); + runner.enqueue(shellResult(0, "NAME\nother-sandbox\n")); + runner.enqueue(shellResult(0, "other-sandbox\n")); + + const result = await fixture(runner).from( + "preflight-failure-no-sandbox", + instance({ + onboarding: "cloud-openclaw-no-docker", + sandboxName: "e2e-no-docker", + expectedFailure: { + phase: "preflight", + errorClass: "docker-missing", + }, + }), + ); + + expect(result.probes.map((probe) => probe.id)).toEqual([ + "cli-installed", + "gateway-absent", + "sandbox-absent", + ]); + expect(runner.calls.map((call) => call.args)).toEqual([ + ["--version"], + ["gateway", "status"], + GATEWAY_ABSENT_HEALTH_CURL_ARGS, + ["list"], + ["sandbox", "list"], + ]); + expect(result.probes.find((probe) => probe.id === "gateway-absent")?.results).toHaveLength(2); + }); + + it("fails a gateway-absent probe if the gateway is running", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(0, "gateway healthy\n")); + + await expect(fixture(runner).from("preflight-failure-no-sandbox", instance())).rejects.toThrow( + /expected gateway to be absent/, + ); + }); + + it("fails a gateway-absent probe if the gateway health endpoint responds", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(1, "gateway status unavailable\n")); + runner.enqueue(shellResult(0, "ok\n")); + + await expect(fixture(runner).from("preflight-failure-no-sandbox", instance())).rejects.toThrow( + /health responded healthy/, + ); + expect(runner.calls.map((call) => call.args)).toEqual([ + ["--version"], + ["gateway", "status"], + GATEWAY_ABSENT_HEALTH_CURL_ARGS, + ]); + }); + + it("requires a loopback gateway URL for gateway-absent health probes", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(1, "gateway stopped")); + + await expect( + fixture(runner).from( + "preflight-failure-no-sandbox", + instance({ + gatewayUrl: "http://10.0.0.1:18789", + }), + ), + ).rejects.toThrow(/private or link-local/); + }); + + it("fails a sandbox-absent probe if NemoClaw lists the sandbox", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(1, "gateway stopped")); + runner.enqueue(shellResult(7, "connection refused")); + runner.enqueue(shellResult(0, "NAME\ne2e-ubuntu-repo-cloud-openclaw\n")); + + await expect(fixture(runner).from("preflight-failure-no-sandbox", instance())).rejects.toThrow( + /nemoclaw listed it/, + ); + }); + + it("fails a sandbox-absent probe if OpenShell lists the sandbox", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(1, "gateway stopped")); + runner.enqueue(shellResult(7, "connection refused")); + runner.enqueue(shellResult(0, "NAME\nother-sandbox\n")); + runner.enqueue(shellResult(0, "NAME\ne2e-ubuntu-repo-cloud-openclaw\n")); + + await expect(fixture(runner).from("preflight-failure-no-sandbox", instance())).rejects.toThrow( + /OpenShell listed it/, + ); + }); + + it("tolerates an unavailable OpenShell list after NemoClaw list reports no sandbox", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(1, "gateway stopped")); + runner.enqueue(shellResult(7, "connection refused")); + runner.enqueue(shellResult(0, "NAME\nother-sandbox\n")); + runner.enqueueError(new Error("spawn openshell ENOENT")); + + const result = await fixture(runner).from("preflight-failure-no-sandbox", instance()); + + const sandboxAbsent = result.probes.find((probe) => probe.id === "sandbox-absent"); + expect(sandboxAbsent?.results).toHaveLength(1); + expect(runner.calls.map((call) => call.args)).toEqual([ + ["--version"], + ["gateway", "status"], + GATEWAY_ABSENT_HEALTH_CURL_ARGS, + ["list"], + ["sandbox", "list"], + ]); + }); + + it("fails a sandbox-absent probe if OpenShell list errors unexpectedly", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(1, "gateway stopped")); + runner.enqueue(shellResult(7, "connection refused")); + runner.enqueue(shellResult(0, "NAME\nother-sandbox\n")); + runner.enqueueError(new Error("openshell permission denied")); + + await expect(fixture(runner).from("preflight-failure-no-sandbox", instance())).rejects.toThrow( + /could not verify OpenShell sandbox absence/, + ); + }); + + it("does not treat sandbox name substrings as present", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(1, "gateway stopped")); + runner.enqueue(shellResult(7, "connection refused")); + runner.enqueue(shellResult(0, "NAME\ne2e-ubuntu-repo-cloud-openclaw-old\n")); + runner.enqueue(shellResult(0, "NAME\nold-e2e-ubuntu-repo-cloud-openclaw\n")); + + const result = await fixture(runner).from("preflight-failure-no-sandbox", instance()); + + expect(result.probes.map((probe) => probe.id)).toEqual([ + "cli-installed", + "gateway-absent", + "sandbox-absent", + ]); + }); + + it("does not pass unrelated secret environment values to status probes", async () => { + const original = process.env.NVIDIA_API_KEY; + process.env.NVIDIA_API_KEY = "nvapi-test-secret-value"; + try { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(1, "gateway stopped")); + runner.enqueue(shellResult(7, "connection refused")); + runner.enqueue(shellResult(0, "NAME\nother-sandbox\n")); + runner.enqueue(shellResult(0, "other-sandbox\n")); + + await fixture(runner).from("preflight-failure-no-sandbox", instance()); + + for (const call of runner.calls.slice(1)) { + expect(call.options).not.toHaveProperty("inheritEnv"); + expect(call.options?.env).toEqual(expect.objectContaining({ PATH: expect.any(String) })); + expect(call.options?.env).not.toHaveProperty("NVIDIA_API_KEY"); + } + } finally { + if (original === undefined) { + delete process.env.NVIDIA_API_KEY; + } else { + process.env.NVIDIA_API_KEY = original; + } + } + }); + + it("requires an instance for probes that use instance context", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + + await expect(fixture(runner).from("cloud-openclaw-ready")).rejects.toThrow( + /probe 'gateway-healthy' requires a NemoClaw instance/, + ); + }); + + it("runs only the CLI probe for optional platform state", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + + const result = await fixture(runner).from("macos-cli-ready-docker-optional"); + + expect(result.probes.map((probe) => probe.id)).toEqual(["cli-installed"]); + expect(runner.calls.map((call) => call.args)).toEqual([["--version"]]); + }); + + it("rejects unknown expected-state IDs", async () => { + const runner = new FakeRunner(); + + await expect(fixture(runner).from("missing-state", instance())).rejects.toThrow(/Unknown expected_state/); + }); + + it("exposes the state-validation phase on the Vitest scenario context", () => { + expectTypeOf().toEqualTypeOf(); + }); +}); diff --git a/test/e2e-scenario/framework/clients/gateway.ts b/test/e2e-scenario/framework/clients/gateway.ts index 196d57c48a0..aca6e37a7b3 100644 --- a/test/e2e-scenario/framework/clients/gateway.ts +++ b/test/e2e-scenario/framework/clients/gateway.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import type { ShellProbeResult } from "../shell-probe.ts"; +import type { ShellProbeResult, ShellProbeRunOptions } from "../shell-probe.ts"; import { assertExitZero } from "./command.ts"; import type { HostCliClient } from "./host.ts"; @@ -12,12 +12,12 @@ export class GatewayClient { this.host = host; } - status(): Promise { - return this.host.nemoclaw(["gateway", "status"], { artifactName: "gateway-status" }); + status(options: ShellProbeRunOptions = {}): Promise { + return this.host.nemoclaw(["gateway", "status"], { artifactName: "gateway-status", ...options }); } - async expectHealthy(): Promise { - const result = await this.status(); + async expectHealthy(options: ShellProbeRunOptions = {}): Promise { + const result = await this.status(options); assertExitZero(result, "nemoclaw gateway status"); return result; } diff --git a/test/e2e-scenario/framework/clients/sandbox.ts b/test/e2e-scenario/framework/clients/sandbox.ts index a7ee5765bb5..792bd02b941 100644 --- a/test/e2e-scenario/framework/clients/sandbox.ts +++ b/test/e2e-scenario/framework/clients/sandbox.ts @@ -32,13 +32,13 @@ export class SandboxClient { ); } - list(): Promise { - return this.openshell(["sandbox", "list"], { artifactName: "sandbox-list" }); + list(options: ShellProbeRunOptions = {}): Promise { + return this.openshell(["sandbox", "list"], { artifactName: "sandbox-list", ...options }); } - status(name: string): Promise { + status(name: string, options: ShellProbeRunOptions = {}): Promise { validateSandboxName(name); - return this.openshell(["sandbox", "status", name], { artifactName: `sandbox-status-${name}` }); + return this.openshell(["sandbox", "status", name], { artifactName: `sandbox-status-${name}`, ...options }); } exec(name: string, command: string[], options: ShellProbeRunOptions = {}): Promise { @@ -49,8 +49,8 @@ export class SandboxClient { }); } - async expectRunning(name: string): Promise { - const result = await this.status(name); + async expectRunning(name: string, options: ShellProbeRunOptions = {}): Promise { + const result = await this.status(name, options); assertExitZero(result, `openshell sandbox status ${name}`); return result; } diff --git a/test/e2e-scenario/framework/e2e-test.ts b/test/e2e-scenario/framework/e2e-test.ts index cc5d63ec7cc..4acd429c1cc 100644 --- a/test/e2e-scenario/framework/e2e-test.ts +++ b/test/e2e-scenario/framework/e2e-test.ts @@ -12,7 +12,7 @@ import { StateClient, } from "./clients/index.ts"; import { assertCleanupPassed, CleanupRegistry } from "./cleanup.ts"; -import { EnvironmentPhaseFixture, OnboardingPhaseFixture } from "./phases/index.ts"; +import { EnvironmentPhaseFixture, OnboardingPhaseFixture, StateValidationPhaseFixture } from "./phases/index.ts"; import { SecretStore } from "./secrets.ts"; import { ShellProbe } from "./shell-probe.ts"; @@ -28,6 +28,7 @@ export interface E2EScenarioFixtures { state: StateClient; environment: EnvironmentPhaseFixture; onboard: OnboardingPhaseFixture; + stateValidation: StateValidationPhaseFixture; } export const test = base.extend({ @@ -86,6 +87,9 @@ export const test = base.extend({ onboard: async ({ cleanup, host, secrets }, use) => { await use(new OnboardingPhaseFixture(host, secrets, cleanup)); }, + stateValidation: async ({ host, gateway, sandbox }, use) => { + await use(new StateValidationPhaseFixture(host, gateway, sandbox)); + }, }); export { expect }; diff --git a/test/e2e-scenario/framework/phases/index.ts b/test/e2e-scenario/framework/phases/index.ts index 25b3dba5147..f6345ba476f 100644 --- a/test/e2e-scenario/framework/phases/index.ts +++ b/test/e2e-scenario/framework/phases/index.ts @@ -14,3 +14,8 @@ export { type OnboardingOptions, type OnboardingSecrets, } from "./onboarding.ts"; +export { + StateValidationPhaseFixture, + type StateValidationProbeResult, + type StateValidationResult, +} from "./state-validation.ts"; diff --git a/test/e2e-scenario/framework/phases/onboarding.ts b/test/e2e-scenario/framework/phases/onboarding.ts index 458813958c0..717aad6ed19 100644 --- a/test/e2e-scenario/framework/phases/onboarding.ts +++ b/test/e2e-scenario/framework/phases/onboarding.ts @@ -58,9 +58,10 @@ export interface OnboardingExpectedFailure { export interface NemoClawInstance { onboarding: string; sandboxName: string; - agent: "openclaw"; - provider: "nvidia"; - providerEnv: "cloud"; + agent: "openclaw" | "hermes"; + provider: "nvidia" | "ollama"; + providerEnv: "cloud" | "local"; + platformOs?: "ubuntu" | "macos" | "windows"; gatewayUrl: string; result: ShellProbeResult; expectedFailure?: OnboardingExpectedFailure; diff --git a/test/e2e-scenario/framework/phases/state-validation.ts b/test/e2e-scenario/framework/phases/state-validation.ts new file mode 100644 index 00000000000..b665e4ee233 --- /dev/null +++ b/test/e2e-scenario/framework/phases/state-validation.ts @@ -0,0 +1,238 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { buildAvailabilityProbeEnv } from "../availability-env.ts"; +import { + trustedProviderEndpoint, + type GatewayClient, + type HostCliClient, + type SandboxClient, +} from "../clients/index.ts"; +import type { ShellProbeResult } from "../shell-probe.ts"; +import { probesForState, requireExpectedState } from "../../scenarios/expected-states.ts"; +import type { ExpectedState, StateProbeId } from "../../scenarios/types.ts"; +import type { NemoClawInstance } from "./onboarding.ts"; + +export interface StateValidationProbeResult { + id: StateProbeId; + status: "passed"; + results: ShellProbeResult[]; +} + +export interface StateValidationResult { + state: ExpectedState; + probes: StateValidationProbeResult[]; +} + +function requireInstance(probe: StateProbeId, instance: NemoClawInstance | undefined): NemoClawInstance { + if (!instance) { + throw new Error(`state-validation probe '${probe}' requires a NemoClaw instance.`); + } + return instance; +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function outputContainsSandbox(result: ShellProbeResult, sandboxName: string): boolean { + const output = `${result.stdout}\n${result.stderr}`; + return new RegExp(`(^|\\s)${escapeRegExp(sandboxName)}(\\s|$)`, "m").test(output); +} + +function statusProbeEnv(): NodeJS.ProcessEnv { + return buildAvailabilityProbeEnv(); +} + +function gatewayHealthEndpoint(gatewayUrl: string): string { + return trustedProviderEndpoint(`${gatewayUrl.replace(/\/+$/, "")}/health`).url; +} + +function gatewayBaseEndpoint(gatewayUrl: string): string { + return trustedProviderEndpoint(gatewayUrl).url; +} + +function resultHttpCode(result: ShellProbeResult): string { + return result.stdout.trim(); +} + +function resultHasHttpCode(result: ShellProbeResult, allowedCodes: readonly string[]): boolean { + return result.exitCode === 0 && allowedCodes.includes(resultHttpCode(result)); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function isMissingOpenShellError(error: unknown): boolean { + const code = typeof error === "object" && error !== null && "code" in error ? error.code : undefined; + return code === "ENOENT" || /\bENOENT\b/.test(errorMessage(error)); +} + +export class StateValidationPhaseFixture { + constructor( + private readonly host: HostCliClient, + private readonly gateway: GatewayClient, + private readonly sandbox: SandboxClient, + ) {} + + async from(expectedStateId: string, instance?: NemoClawInstance): Promise { + const state = requireExpectedState(expectedStateId); + const probes: StateValidationProbeResult[] = []; + for (const probe of probesForState(state)) { + probes.push(await this.runProbe(probe, instance)); + } + return { state, probes }; + } + + private async runProbe(probe: StateProbeId, instance: NemoClawInstance | undefined): Promise { + switch (probe) { + case "cli-installed": + return await this.expectCliInstalled(); + case "gateway-healthy": + return await this.expectGatewayHealthy(requireInstance(probe, instance)); + case "gateway-absent": + return await this.expectGatewayAbsent(requireInstance(probe, instance)); + case "sandbox-running": + return await this.expectSandboxRunning(requireInstance(probe, instance)); + case "sandbox-absent": + return await this.expectSandboxAbsent(requireInstance(probe, instance)); + default: { + const _exhaustive: never = probe; + throw new Error(`Unsupported state-validation probe '${_exhaustive}'.`); + } + } + } + + private async expectCliInstalled(): Promise { + const result = await this.host.expectNemoclawAvailable(); + return { id: "cli-installed", status: "passed", results: [result] }; + } + + private curlHttpStatus(url: string, artifactName: string, maxTimeSeconds: string): Promise { + return this.host.command( + "curl", + ["-fsS", "-o", "/dev/null", "-w", "%{http_code}", "--max-time", maxTimeSeconds, url], + { + artifactName, + env: statusProbeEnv(), + redactionValues: [url], + }, + ); + } + + private async expectGatewayHealthy(instance: NemoClawInstance): Promise { + const results: ShellProbeResult[] = []; + const health = await this.curlHttpStatus(gatewayHealthEndpoint(instance.gatewayUrl), "gateway-health", "5"); + results.push(health); + if (resultHasHttpCode(health, ["200"])) { + return { id: "gateway-healthy", status: "passed", results }; + } + + const base = await this.curlHttpStatus(gatewayBaseEndpoint(instance.gatewayUrl), "gateway-base", "5"); + results.push(base); + if (resultHasHttpCode(base, ["200", "204"])) { + return { id: "gateway-healthy", status: "passed", results }; + } + + if ((instance.platformOs ?? "ubuntu") === "ubuntu" && instance.provider === "ollama") { + const sandboxLocal = await this.sandbox.exec( + instance.sandboxName, + [ + "curl", + "-sS", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "--max-time", + "5", + "http://localhost:18789/health", + ], + { + artifactName: "gateway-sandbox-local-health", + env: statusProbeEnv(), + timeoutMs: 15_000, + }, + ); + results.push(sandboxLocal); + if (resultHasHttpCode(sandboxLocal, ["200", "401"])) { + return { id: "gateway-healthy", status: "passed", results }; + } + } + + const last = results.at(-1) ?? base; + throw new Error( + `state-validation expected gateway '${instance.gatewayUrl}' to be healthy, ` + + `but HTTP probes failed (last http_code=${resultHttpCode(last) || "000"}).`, + ); + } + + private async expectGatewayAbsent(instance: NemoClawInstance): Promise { + const result = await this.gateway.status({ + artifactName: "gateway-absent-status", + env: statusProbeEnv(), + }); + if (result.exitCode === 0) { + throw new Error("state-validation expected gateway to be absent, but 'nemoclaw gateway status' succeeded."); + } + const healthUrl = gatewayHealthEndpoint(instance.gatewayUrl); + const health = await this.curlHttpStatus(healthUrl, "gateway-absent-health", "3"); + if (health.exitCode === 0) { + throw new Error(`state-validation expected gateway to be absent, but ${healthUrl} responded healthy.`); + } + return { id: "gateway-absent", status: "passed", results: [result, health] }; + } + + private async expectSandboxRunning(instance: NemoClawInstance): Promise { + const result = await this.host.nemoclaw(["list"], { + artifactName: "sandbox-running-nemoclaw-list", + env: statusProbeEnv(), + }); + if (result.exitCode !== 0) { + throw new Error("state-validation expected sandbox to be running, but 'nemoclaw list' failed."); + } + if (!outputContainsSandbox(result, instance.sandboxName)) { + throw new Error( + `state-validation expected sandbox '${instance.sandboxName}' to be running, but nemoclaw did not list it.`, + ); + } + return { id: "sandbox-running", status: "passed", results: [result] }; + } + + private async expectSandboxAbsent(instance: NemoClawInstance): Promise { + const results: ShellProbeResult[] = []; + const nemoclawList = await this.host.nemoclaw(["list"], { + artifactName: "sandbox-absent-nemoclaw-list", + env: statusProbeEnv(), + }); + results.push(nemoclawList); + if (nemoclawList.exitCode === 0 && outputContainsSandbox(nemoclawList, instance.sandboxName)) { + throw new Error(`state-validation expected sandbox '${instance.sandboxName}' to be absent, but nemoclaw listed it.`); + } + + let openshellList: ShellProbeResult | undefined; + try { + openshellList = await this.sandbox.list({ + artifactName: "sandbox-absent-openshell-list", + env: statusProbeEnv(), + }); + } catch (error) { + if (!isMissingOpenShellError(error)) { + throw new Error(`state-validation could not verify OpenShell sandbox absence: ${errorMessage(error)}`); + } + // Bridge tolerance for negative preflight states: `nemoclaw list` is the + // user-facing registry authority, while OpenShell may be absent before any + // sandbox setup happens. Once the fixture has a typed OpenShell + // availability probe, make this path fail closed. + } + if (openshellList) { + results.push(openshellList); + if (openshellList.exitCode === 0 && outputContainsSandbox(openshellList, instance.sandboxName)) { + throw new Error(`state-validation expected sandbox '${instance.sandboxName}' to be absent, but OpenShell listed it.`); + } + } + + return { id: "sandbox-absent", status: "passed", results }; + } +}