diff --git a/src/commands/sandbox/oclif-command-adapters.test.ts b/src/commands/sandbox/oclif-command-adapters.test.ts index 8bab85dff9f..1d90c43c86e 100644 --- a/src/commands/sandbox/oclif-command-adapters.test.ts +++ b/src/commands/sandbox/oclif-command-adapters.test.ts @@ -142,6 +142,28 @@ describe("sandbox oclif command adapters", () => { } }); + it("rejects the removed connect permission bypass before dispatch", async () => { + const previousExitCode = process.exitCode; + const lines: string[] = []; + const errorSpy = vi.spyOn(console, "error").mockImplementation((line = "") => { + lines.push(String(line)); + }); + process.exitCode = undefined; + + try { + await ConnectCliCommand.run(["alpha", "--dangerously-skip-permissions"], rootDir); + + expect(lines.join("\n")).toContain( + "--dangerously-skip-permissions was removed; use shields commands instead.", + ); + expect(process.exitCode).toBe(1); + expect(mocks.connectSandbox).not.toHaveBeenCalled(); + } finally { + errorSpy.mockRestore(); + process.exitCode = previousExitCode; + } + }); + it("threads --cleanup-gateway / --no-cleanup-gateway through destroy (#2166)", async () => { const originalCleanupGatewayEnv = process.env.NEMOCLAW_CLEANUP_GATEWAY; delete process.env.NEMOCLAW_CLEANUP_GATEWAY; diff --git a/src/lib/actions/sandbox/connect-flow.test.ts b/src/lib/actions/sandbox/connect-flow.test.ts index 13af67e6462..2efe22a6717 100644 --- a/src/lib/actions/sandbox/connect-flow.test.ts +++ b/src/lib/actions/sandbox/connect-flow.test.ts @@ -27,6 +27,7 @@ describe("connectSandbox flow", () => { afterEach(() => { vi.restoreAllMocks(); + vi.unstubAllEnvs(); if (originalStdoutIsTty === undefined) { Object.defineProperty(process.stdout, "isTTY", { configurable: true, value: undefined }); } else { diff --git a/src/lib/actions/sandbox/connect-route-lifecycle.test.ts b/src/lib/actions/sandbox/connect-route-lifecycle.test.ts new file mode 100644 index 00000000000..e04d2ca93ae --- /dev/null +++ b/src/lib/actions/sandbox/connect-route-lifecycle.test.ts @@ -0,0 +1,190 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, beforeEach, describe, expect, it, type MockInstance, vi } from "vitest"; + +import { + connectModulePath, + createConnectHarness, + requireDist, +} from "../../../../test/support/connect-flow-test-harness"; + +describe("connectSandbox route lifecycle", () => { + let exitSpy: MockInstance; + const originalStdoutIsTty = process.stdout.isTTY; + + beforeEach(() => { + process.env.NEMOCLAW_TEST_NO_SLEEP = "1"; + Object.defineProperty(process.stdout, "isTTY", { configurable: true, value: true }); + exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number | string | null) => { + throw new Error(`process.exit(${code ?? 0})`); + }) as never); + }); + + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + Object.defineProperty(process.stdout, "isTTY", { + configurable: true, + value: originalStdoutIsTty, + }); + delete process.env.NEMOCLAW_TEST_NO_SLEEP; + delete require.cache[requireDist.resolve(connectModulePath)]; + }); + + it("skips the vLLM model preflight only for probe-only connects (#4585)", async () => { + const harness = createConnectHarness(); + + await expect(harness.connectSandbox("alpha", { probeOnly: true })).resolves.toBeUndefined(); + await expect(harness.connectSandbox("alpha", { probeOnly: true })).resolves.toBeUndefined(); + expect(harness.preflightVllmSpy).not.toHaveBeenCalled(); + + await expect(harness.connectSandbox("alpha")).rejects.toThrow("process.exit(0)"); + expect(harness.preflightVllmSpy).toHaveBeenCalledOnce(); + }); + + it("warns and aligns a diverged route during a quiet probe-only connect (#3726)", async () => { + const harness = createConnectHarness({ + inferenceGetOutput: + "Gateway inference:\n Provider: nvidia-prod\n Model: nvidia/nemotron-3-super-120b-a12b\n", + registryEntry: { + model: "claude-sonnet-4-20250514", + provider: "anthropic-prod", + }, + }); + + await expect(harness.connectSandbox("alpha", { probeOnly: true })).resolves.toBeUndefined(); + + const errorOutput = harness.errorSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); + expect(errorOutput).toContain("differs from the recorded route"); + expect(errorOutput).toContain( + "Aligning the gateway to anthropic-prod/claude-sonnet-4-20250514", + ); + expect(harness.runOpenshellSpy).toHaveBeenCalledWith( + [ + "inference", + "set", + "--provider", + "anthropic-prod", + "--model", + "claude-sonnet-4-20250514", + "--no-verify", + ], + expect.objectContaining({ ignoreError: true }), + ); + expect(harness.spawnSyncSpy).not.toHaveBeenCalledWith( + "openshell", + ["sandbox", "connect", "alpha"], + expect.any(Object), + ); + }); + + it("wires the forced VM DNS monkeypatch into connect route repair", async () => { + vi.stubEnv("NEMOCLAW_FORCE_VM_DNS_MONKEYPATCH", "1"); + try { + const harness = createConnectHarness({ + inferenceGetOutput: + "Gateway inference:\n Provider: nvidia-prod\n Model: nvidia/nemotron-3-super-120b-a12b\n", + inferenceProbeResponses: ['BROKEN 503 {"error":"inference service unavailable"}', "OK 200"], + registryEntry: { + model: "nvidia/nemotron-3-super-120b-a12b", + openshellDriver: "vm", + provider: "nvidia-prod", + }, + }); + + await expect(harness.connectSandbox("alpha", { probeOnly: true })).resolves.toBeUndefined(); + + expect(harness.applyVmDnsMonkeypatchSpy).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ openshellDriver: "vm" }), + ); + expect(harness.runSetupDnsProxySpy).not.toHaveBeenCalled(); + expect(harness.runOpenshellSpy).not.toHaveBeenCalled(); + const routeProbeCalls = harness.captureOpenshellSpy.mock.calls.filter((call) => + JSON.stringify(call[0]).includes("inference.local/v1/models"), + ); + expect(routeProbeCalls).toHaveLength(2); + } finally { + vi.unstubAllEnvs(); + } + }); + + it.each([ + ["null", null, null], + ["provider-only", "nvidia-prod", null], + ["model-only", null, "nvidia/test"], + ["blank-provider", " ", "nvidia/test"], + ["blank-model", "nvidia-prod", " "], + ] as const)("skips inference reconciliation for %s registry entries (#5937)", async (_description, provider, model) => { + const harness = createConnectHarness({ registryEntry: { model, provider } }); + + await expect(harness.connectSandbox("alpha", { probeOnly: true })).resolves.toBeUndefined(); + + expect(harness.captureOpenshellSpy).not.toHaveBeenCalledWith( + ["inference", "get"], + expect.any(Object), + ); + expect(harness.runOpenshellSpy).not.toHaveBeenCalled(); + }); + + it("does not reset an inference route that already matches the sandbox", async () => { + const harness = createConnectHarness({ + inferenceGetOutput: + "Gateway inference:\n Provider: nvidia-prod\n Model: nvidia/nemotron-3-super-120b-a12b\n", + registryEntry: { + model: "nvidia/nemotron-3-super-120b-a12b", + provider: "nvidia-prod", + }, + }); + + await expect(harness.connectSandbox("alpha", { probeOnly: true })).resolves.toBeUndefined(); + + expect(harness.captureOpenshellSpy).toHaveBeenCalledWith( + ["inference", "get"], + expect.objectContaining({ ignoreError: true }), + ); + expect(harness.runOpenshellSpy).not.toHaveBeenCalled(); + }); + + it("stops before opening SSH when route repair and reset both fail", async () => { + const harness = createConnectHarness({ + inferenceGetOutput: + "Gateway inference:\n Provider: nvidia-prod\n Model: nvidia/nemotron-3-super-120b-a12b\n", + inferenceProbeResponses: Array(7).fill('BROKEN 503 {"error":"upstream unavailable"}'), + registryEntry: { + model: "nvidia/nemotron-3-super-120b-a12b", + openshellDriver: "kubernetes", + provider: "nvidia-prod", + }, + }); + + await expect(harness.connectSandbox("alpha")).rejects.toThrow("process.exit(1)"); + + expect(harness.runSetupDnsProxySpy).toHaveBeenCalledOnce(); + expect(harness.runOpenshellSpy).toHaveBeenCalledOnce(); + expect(harness.runOpenshellSpy).toHaveBeenCalledWith( + [ + "inference", + "set", + "--provider", + "nvidia-prod", + "--model", + "nvidia/nemotron-3-super-120b-a12b", + "--no-verify", + ], + expect.objectContaining({ ignoreError: true }), + ); + expect(harness.spawnSyncSpy).not.toHaveBeenCalledWith( + "openshell", + ["sandbox", "connect", "alpha"], + expect.any(Object), + ); + const errorOutput = harness.errorSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); + expect(errorOutput).toContain("inference.local is still unavailable"); + expect(errorOutput).toContain( + "Connect is stopping because the sandbox inference route is known to be broken", + ); + expect(exitSpy).toHaveBeenCalledWith(1); + }); +}); diff --git a/src/lib/actions/sandbox/connect-route-repair.test.ts b/src/lib/actions/sandbox/connect-route-repair.test.ts index f558513d00e..c06d83a5f30 100644 --- a/src/lib/actions/sandbox/connect-route-repair.test.ts +++ b/src/lib/actions/sandbox/connect-route-repair.test.ts @@ -161,6 +161,10 @@ describe("sandbox connect route repair unit flow", () => { }); expect(calls.legacyRepairs).toEqual([{ sandboxName: "legacy-box", quiet: false }]); expect(calls.reapplications).toEqual([]); + expect(calls.probeOptions).toEqual([undefined, { attempts: 3, delayMs: 2000 }]); + expect(calls.logs).toContain( + " inference.local is unavailable inside 'legacy-box'. Repairing sandbox DNS proxy...", + ); expect(calls.logs).toContain(" inference.local route repaired."); }); @@ -225,6 +229,13 @@ describe("sandbox connect route repair unit flow", () => { expect(calls.monkeypatches).toEqual(["vm-box"]); expect(calls.reapplications).toEqual([]); expect(calls.legacyRepairs).toEqual([]); + expect(calls.probeOptions).toEqual([undefined, { attempts: 3, delayMs: 2000 }]); + expect(calls.logs).toContain( + " inference.local is unavailable inside 'vm-box'. Applying OpenShell VM DNS monkeypatch...", + ); + expect(calls.logs).not.toContain( + " inference.local is unavailable inside 'vm-box'. Reapplying OpenShell inference route...", + ); }); it("falls back to inference reapply when the VM monkeypatch leaves the route broken", () => { diff --git a/src/lib/actions/sandbox/gateway-state-drift.test.ts b/src/lib/actions/sandbox/gateway-state-drift.test.ts index 560db134936..b1cb161b1b4 100644 --- a/src/lib/actions/sandbox/gateway-state-drift.test.ts +++ b/src/lib/actions/sandbox/gateway-state-drift.test.ts @@ -10,6 +10,12 @@ import type { OpenShellStateRpcIssue } from "../../adapters/openshell/gateway-dr type GatewayStateModule = typeof import("./gateway-state"); const requireDist = createRequire(import.meta.url); +const gatewayDrift = requireDist("../../adapters/openshell/gateway-drift.js"); +const openshellRuntime = requireDist("../../adapters/openshell/runtime.js"); +const gatewayRuntime = requireDist("../../gateway-runtime-action.js"); +const dockerDriverRecovery = requireDist("../../onboard/docker-driver-sandbox-recovery.js"); +const registry = requireDist("../../state/registry.js"); +const gatewayState: GatewayStateModule = requireDist("./gateway-state.js"); const driftIssue: OpenShellStateRpcIssue = { kind: "image_drift", @@ -28,7 +34,6 @@ function mockExit() { } describe("sandbox gateway state drift guard", () => { - let gatewayState: GatewayStateModule; let exitSpy: ReturnType; let errorSpy: MockInstance; let spies: MockInstance[]; @@ -41,16 +46,11 @@ describe("sandbox gateway state drift guard", () => { let runOpenshellSpy: MockInstance; let removeSandboxSpy: MockInstance; - beforeEach(async () => { + beforeEach(() => { spies = []; exitSpy = mockExit(); errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); - const gatewayDrift = requireDist("../../adapters/openshell/gateway-drift.js"); - const openshellRuntime = requireDist("../../adapters/openshell/runtime.js"); - const gatewayRuntime = requireDist("../../gateway-runtime-action.js"); - const registry = requireDist("../../state/registry.js"); - getSandboxSpy = vi.spyOn(registry, "getSandbox").mockReturnValue(null); captureOpenshellSpy = vi @@ -96,10 +96,11 @@ describe("sandbox gateway state drift guard", () => { getNamedGatewayLifecycleStateSpy, getSandboxSpy, recoverNamedGatewayRuntimeSpy, + vi + .spyOn(dockerDriverRecovery, "recoverDockerDriverSandbox") + .mockReturnValue({ recovered: false, via: null }), removeSandboxSpy, ); - - gatewayState = requireDist("./gateway-state.js"); }); afterEach(() => { @@ -146,6 +147,96 @@ describe("sandbox gateway state drift guard", () => { expect(captureOpenshellSpy).not.toHaveBeenCalled(); }); + it("preserves a local registry entry when a healthy named gateway still lacks the sandbox", async () => { + detectPreflightIssueSpy.mockReturnValue(null); + getSandboxSpy.mockReturnValue({ + name: "alpha", + gatewayName: "nemoclaw", + gatewayPort: 8080, + }); + captureOpenshellSpy.mockReturnValue({ + status: 1, + output: 'Error: status: NotFound, message: "sandbox not found"', + }); + getNamedGatewayLifecycleStateSpy.mockReturnValue({ + state: "healthy_named", + status: "Gateway: nemoclaw\nStatus: Connected", + }); + + await expect(gatewayState.ensureLiveSandboxOrExit("alpha")).rejects.toThrow("process.exit(1)"); + + const output = errorSpy.mock.calls.flat().join("\n"); + expect(output).toContain("Your local registry entry has been preserved — nothing was removed."); + expect(output).toContain("nemoclaw alpha rebuild --yes"); + expect(output).toContain("nemoclaw alpha destroy"); + expect(removeSandboxSpy).not.toHaveBeenCalled(); + }); + + it("preserves registry state and prints deterministic guidance when gateway selection cannot expose the sandbox (#2276)", async () => { + vi.stubEnv("NEMOCLAW_NON_INTERACTIVE", "1"); + detectPreflightIssueSpy.mockReturnValue(null); + getSandboxSpy.mockReturnValue({ + name: "alpha", + gatewayName: "nemoclaw", + gatewayPort: 8080, + }); + captureOpenshellSpy.mockReturnValue({ + status: 1, + output: 'Error: status: NotFound, message: "sandbox not found"', + }); + getNamedGatewayLifecycleStateSpy.mockReturnValue({ + state: "connected_other", + activeGateway: "openshell", + status: "Gateway: openshell\nStatus: Connected", + }); + + await expect(gatewayState.ensureLiveSandboxOrExit("alpha")).rejects.toThrow("process.exit(1)"); + + const output = errorSpy.mock.calls.flat().join("\n"); + expect(output).toContain("Your sandbox has NOT been removed"); + expect(output).toContain("openshell gateway select nemoclaw"); + expect(output).not.toMatch(/Press (?:enter|any key)|\?\s+\[/i); + expect(runOpenshellSpy).toHaveBeenCalledWith( + ["gateway", "select", "nemoclaw"], + expect.objectContaining({ ignoreError: true }), + ); + expect(removeSandboxSpy).not.toHaveBeenCalled(); + }); + + it.each([ + { + lifecycle: { state: "missing_named", status: "No gateway configured" }, + expected: "gateway is no longer configured after restart/rebuild", + }, + { + lifecycle: { + state: "named_unreachable", + status: "Gateway: nemoclaw\nConnection refused", + }, + expected: "gateway is still refusing connections after restart", + }, + ])("preserves registry state when the named gateway reports $lifecycle.state", async ({ + lifecycle, + expected, + }) => { + detectPreflightIssueSpy.mockReturnValue(null); + getSandboxSpy.mockReturnValue({ + name: "alpha", + gatewayName: "nemoclaw", + gatewayPort: 8080, + }); + captureOpenshellSpy.mockReturnValue({ + status: 1, + output: 'Error: status: NotFound, message: "sandbox not found"', + }); + getNamedGatewayLifecycleStateSpy.mockReturnValue(lifecycle); + + await expect(gatewayState.ensureLiveSandboxOrExit("alpha")).rejects.toThrow("process.exit(1)"); + + expect(errorSpy.mock.calls.flat().join("\n")).toContain(expected); + expect(removeSandboxSpy).not.toHaveBeenCalled(); + }); + it("propagates schema mismatch after selecting the named gateway", () => { getNamedGatewayLifecycleStateSpy.mockReturnValue({ state: "connected_other", @@ -256,5 +347,6 @@ describe("sandbox gateway state drift guard", () => { ["gateway", "select", "nemoclaw"], expect.objectContaining({ ignoreError: true }), ); + expect(removeSandboxSpy).not.toHaveBeenCalled(); }); }); diff --git a/src/lib/actions/sandbox/gateway-state-hints.test.ts b/src/lib/actions/sandbox/gateway-state-hints.test.ts index 82d17e1eb3f..bcf71ce0f8f 100644 --- a/src/lib/actions/sandbox/gateway-state-hints.test.ts +++ b/src/lib/actions/sandbox/gateway-state-hints.test.ts @@ -11,13 +11,33 @@ const requireDist = createRequire(import.meta.url); describe("printGatewayLifecycleHint multi-instance hints", () => { let gatewayState: GatewayStateModule; + let captureOpenshellSpy: MockInstance; + let getNamedGatewayLifecycleStateSpy: MockInstance; let getSandboxSpy: MockInstance; + let recoverNamedGatewayRuntimeSpy: MockInstance; beforeEach(async () => { + const gatewayStatePath = requireDist.resolve("./gateway-state.js"); + delete require.cache[gatewayStatePath]; + const gatewayDrift = requireDist("../../adapters/openshell/gateway-drift.js"); + const openshellRuntime = requireDist("../../adapters/openshell/runtime.js"); + const gatewayRuntime = requireDist("../../gateway-runtime-action.js"); const registry = requireDist("../../state/registry.js"); + vi.spyOn(gatewayDrift, "detectOpenShellStateRpcPreflightIssue").mockReturnValue(null); + vi.spyOn(gatewayDrift, "detectOpenShellStateRpcResultIssue").mockReturnValue(null); + captureOpenshellSpy = vi.spyOn(openshellRuntime, "captureOpenshell").mockReturnValue({ + status: 0, + output: "Sandbox:\n Name: instance-a\n Phase: Ready", + }); + getNamedGatewayLifecycleStateSpy = vi + .spyOn(gatewayRuntime, "getNamedGatewayLifecycleState") + .mockReturnValue({ state: "healthy_named", status: "Gateway: nemoclaw" }); + recoverNamedGatewayRuntimeSpy = vi + .spyOn(gatewayRuntime, "recoverNamedGatewayRuntime") + .mockResolvedValue({ recovered: false }); getSandboxSpy = vi.spyOn(registry, "getSandbox").mockReturnValue({ name: "instance-a", - gatewayName: "nemoclaw-8080", + gatewayName: "nemoclaw", gatewayPort: 8080, }); gatewayState = requireDist("./gateway-state.js"); @@ -25,6 +45,7 @@ describe("printGatewayLifecycleHint multi-instance hints", () => { afterEach(() => { vi.restoreAllMocks(); + delete require.cache[requireDist.resolve("./gateway-state.js")]; }); it("surfaces a switch-gateway hint when the underlying gRPC error is `sandbox has no spec`", () => { @@ -68,4 +89,139 @@ describe("printGatewayLifecycleHint multi-instance hints", () => { expect(combined).not.toContain("sandbox has no spec"); expect(combined).toContain("openshell gateway start"); }); + + it.each([ + { + label: "transport", + output: "\u001b[31mError: trans\u001b[0mport error: Connec\u001b[33mtion refused\u001b[0m", + expected: "current gateway/runtime is not reachable", + }, + { + label: "authentication", + output: "\u001b[31mMissing gateway auth\u001b[0m token", + expected: "Verify the active gateway and retry after re-establishing the runtime.", + }, + ])("matches ANSI-decorated $label lifecycle errors", ({ output, expected }) => { + const lines: string[] = []; + + gatewayState.printGatewayLifecycleHint(output, "instance-a", (line: string) => + lines.push(line), + ); + + expect(lines.join("\n")).toContain(expected); + }); + + it("classifies a failed post-recovery handshake as identity drift", async () => { + recoverNamedGatewayRuntimeSpy.mockResolvedValue({ recovered: true, via: "start" }); + const getState = vi + .fn() + .mockResolvedValueOnce({ state: "gateway_error", output: "transport error" }) + .mockResolvedValueOnce({ + state: "gateway_error", + output: "transport error: handshake verification failed", + }); + + const lookup = await gatewayState.getReconciledSandboxGatewayState("instance-a", { getState }); + + expect(lookup).toEqual( + expect.objectContaining({ + state: "identity_drift", + recoveredGateway: true, + recoveryVia: "start", + }), + ); + }); + + it.each([ + { + lifecycle: { + state: "named_unreachable", + status: "Gateway: nemoclaw\nConnection refused", + }, + expectedState: "gateway_unreachable_after_restart", + expectedGatewayRecoveryFailed: undefined, + }, + { + lifecycle: { state: "missing_named", status: "No gateway configured" }, + expectedState: "gateway_missing_after_restart", + expectedGatewayRecoveryFailed: undefined, + }, + { + lifecycle: { + state: "connected_other", + activeGateway: "openshell", + status: "Gateway: openshell\nStatus: Connected", + }, + expectedState: "gateway_error", + expectedGatewayRecoveryFailed: true, + }, + ])("maps failed gateway recovery to $expectedState", async ({ + lifecycle, + expectedState, + expectedGatewayRecoveryFailed, + }) => { + getNamedGatewayLifecycleStateSpy.mockReturnValue(lifecycle); + + const lookup = await gatewayState.getReconciledSandboxGatewayState("instance-a", { + getState: async () => ({ state: "gateway_error", output: "transport error" }), + }); + + expect(lookup.state).toBe(expectedState); + expect(lookup.gatewayRecoveryFailed).toBe(expectedGatewayRecoveryFailed); + }); + + it("prints reconnect and recreate guidance when identity drift persists", async () => { + captureOpenshellSpy.mockReturnValue({ + status: 1, + output: "Error: transport error: handshake verification failed", + }); + recoverNamedGatewayRuntimeSpy.mockResolvedValue({ recovered: true, via: "start" }); + const lines: string[] = []; + const errorSpy = vi.spyOn(console, "error").mockImplementation((line = "") => { + lines.push(String(line)); + }); + const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new Error(`process.exit(${code ?? 0})`); + }) as never); + + await expect(gatewayState.ensureLiveSandboxOrExit("instance-a")).rejects.toThrow( + "process.exit(1)", + ); + + const output = lines.join("\n"); + expect(output).toContain("Could not reconnect to sandbox 'instance-a'"); + expect(output).toContain("Recreate this sandbox"); + expect(exitSpy).toHaveBeenCalledWith(1); + errorSpy.mockRestore(); + exitSpy.mockRestore(); + }); + + it("prints restart guidance when the named gateway remains unreachable", async () => { + captureOpenshellSpy.mockReturnValue({ + status: 1, + output: "Error: transport error: Connection refused", + }); + getNamedGatewayLifecycleStateSpy.mockReturnValue({ + state: "named_unreachable", + status: "Gateway: nemoclaw\nConnection refused", + }); + const lines: string[] = []; + const errorSpy = vi.spyOn(console, "error").mockImplementation((line = "") => { + lines.push(String(line)); + }); + const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new Error(`process.exit(${code ?? 0})`); + }) as never); + + await expect(gatewayState.ensureLiveSandboxOrExit("instance-a")).rejects.toThrow( + "process.exit(1)", + ); + + const output = lines.join("\n"); + expect(output).toContain("gateway is still refusing connections after restart"); + expect(output).toContain("If the gateway never becomes healthy"); + expect(exitSpy).toHaveBeenCalledWith(1); + errorSpy.mockRestore(); + exitSpy.mockRestore(); + }); }); diff --git a/src/lib/actions/sandbox/skill-install.test.ts b/src/lib/actions/sandbox/skill-install.test.ts index d3c83a862f5..9f1dba812de 100644 --- a/src/lib/actions/sandbox/skill-install.test.ts +++ b/src/lib/actions/sandbox/skill-install.test.ts @@ -193,6 +193,24 @@ describe("sandbox skill action orchestration", () => { expect(process.exitCode).toBeUndefined(); }); + it("stops skill installation at the shared gateway liveness guard (#2276)", async () => { + const skillDir = makeSkillDir(); + ensureLiveSandboxOrExit.mockRejectedValueOnce(new Error("wrong gateway active")); + vi.spyOn(console, "log").mockImplementation(() => undefined); + + try { + await expect( + installSandboxSkill("alpha", { command: "install", path: skillDir }), + ).rejects.toThrow("wrong gateway active"); + } finally { + fs.rmSync(skillDir, { recursive: true, force: true }); + } + + expect(ensureLiveSandboxOrExit).toHaveBeenCalledWith("alpha"); + expect(captureSandboxSshConfig).not.toHaveBeenCalled(); + expect(skillInstall.uploadDirectory).not.toHaveBeenCalled(); + }); + it("continues skill install when the existence probe is unknown because upload plus verify are authoritative", async () => { const skillDir = makeSkillDir(); let tempConfig = ""; diff --git a/src/lib/actions/sandbox/status-flow.test.ts b/src/lib/actions/sandbox/status-flow.test.ts index f6e515ef880..5d0b663d2d9 100644 --- a/src/lib/actions/sandbox/status-flow.test.ts +++ b/src/lib/actions/sandbox/status-flow.test.ts @@ -1,176 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { createRequire } from "node:module"; - import { afterEach, beforeEach, describe, expect, it, type MockInstance, vi } from "vitest"; -type ShowSandboxStatus = typeof import("./status")["showSandboxStatus"]; - -const requireDist = createRequire(import.meta.url); -const statusModulePath = "./status.js"; - -// Warm the CommonJS source graph outside the first test's timeout. Each harness -// still reloads the entry module after installing its dependency spies. -requireDist(statusModulePath); -delete require.cache[requireDist.resolve(statusModulePath)]; - -type StatusFlowHarness = { - checkAgentVersionSpy: MockInstance; - getActiveSandboxSessionsSpy: MockInstance; - getSandboxDockerRuntimeSpy: MockInstance; - logSpy: MockInstance; - showSandboxStatus: ShowSandboxStatus; -}; - -const baseSandboxEntry = { - name: "alpha", - model: "nvidia/nemotron", - provider: "ollama-local", - policies: ["npm", "telegram"], - hostGpuDetected: true, - gpuEnabled: true, - sandboxGpuEnabled: true, - sandboxGpuMode: "auto", - sandboxGpuDevice: "all", - sandboxGpuProof: { - status: "failed", - label: "cuInit", - detail: "CUDA initialization failed", - }, - openshellDriver: "docker", - openshellVersion: "0.1.2", - dashboardPort: 18789, - agentVersion: "0.1.0", -}; - -function createStatusFlowHarness( - options: { - lookupState?: "present" | "missing"; - sandboxEntry?: Partial> & { - agent?: string | null; - agentVersion?: string | null; - }; - } = {}, -) { - delete require.cache[requireDist.resolve(statusModulePath)]; - - const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); - vi.spyOn(console, "error").mockImplementation(() => undefined); - - const statusPreflight = requireDist("./status-preflight.js"); - const statusSnapshot = requireDist("./status-snapshot.js"); - const dockerHealth = requireDist("./docker-health.js"); - const processRecovery = requireDist("./process-recovery.js"); - const resolve = requireDist("../../adapters/openshell/resolve.js"); - const agentRuntime = requireDist("../../agent/runtime.js"); - const nim = requireDist("../../inference/nim.js"); - const sandboxVersion = requireDist("../../sandbox/version.js"); - const shields = requireDist("../../shields/index.js"); - const registry = requireDist("../../state/registry.js"); - const sandboxSession = requireDist("../../state/sandbox-session.js"); - - const lookup = - options.lookupState === "missing" - ? { - state: "missing", - output: "sandbox alpha not found", - recoveredGateway: true, - recoveryVia: "gateway reattach", - } - : { - state: "present", - output: "Name: alpha\nPhase: Ready\nEndpoint: http://127.0.0.1:18789\n", - recoveredGateway: true, - recoveryVia: "gateway reattach", - recoveredSandbox: true, - recoverySandboxVia: "docker unpause", - }; - - const sandboxEntry = { ...baseSandboxEntry, ...options.sandboxEntry }; - - vi.spyOn(registry, "getSandbox").mockReturnValue(sandboxEntry); - vi.spyOn(statusPreflight, "getSandboxStatusPreflight").mockResolvedValue({ - failure: null, - failureLayer: null, - suppressInferenceProbe: false, - exitCode: 0, - }); - vi.spyOn(statusSnapshot, "collectSandboxStatusSnapshot").mockResolvedValue({ - sb: sandboxEntry, - lookup, - rpcIssue: null, - currentModel: "nvidia/nemotron-live", - currentProvider: "ollama-local", - inferenceHealth: { - ok: true, - probed: true, - providerLabel: "Ollama", - endpoint: "http://127.0.0.1:11434/v1/chat/completions", - detail: "chat completions probe passed", - subprobes: [ - { - ok: false, - probed: true, - providerLabel: "Inference gateway chain", - endpoint: "http://127.0.0.1:19000/v1/chat/completions", - detail: "gateway refused connection", - probeLabel: "gateway", - failureLabel: "unreachable", - }, - ], - }, - }); - const getSandboxDockerRuntimeSpy = vi - .spyOn(dockerHealth, "getSandboxDockerRuntime") - .mockReturnValue({ - containerName: "openshell-alpha", - health: "unhealthy", - paused: false, - }); - vi.spyOn(processRecovery, "isSandboxGatewayRunningForStatus").mockResolvedValue(false); - vi.spyOn(resolve, "resolveOpenshell").mockReturnValue("/usr/bin/openshell"); - vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ name: "openclaw" }); - vi.spyOn(agentRuntime, "getAgentDisplayName").mockReturnValue("OpenClaw"); - vi.spyOn(agentRuntime, "getGatewayCommand").mockReturnValue("openclaw daemon"); - vi.spyOn(nim, "nimStatus").mockReturnValue({ - running: true, - healthy: false, - container: "alpha-nim", - }); - vi.spyOn(nim, "nimStatusByName").mockReturnValue({ - running: false, - healthy: false, - container: null, - }); - vi.spyOn(nim, "shouldShowNimLine").mockReturnValue(true); - const checkAgentVersionSpy = vi.spyOn(sandboxVersion, "checkAgentVersion").mockReturnValue({ - sandboxVersion: "0.1.0", - expectedVersion: "0.2.0", - isStale: true, - detectionMethod: "runtime", - }); - vi.spyOn(shields, "getShieldsPosture").mockReturnValue({ - mode: "mutable_default", - detail: "mutable default", - }); - const getActiveSandboxSessionsSpy = vi - .spyOn(sandboxSession, "getActiveSandboxSessions") - .mockReturnValue({ - detected: true, - sessions: [{ pid: 1 }, { pid: 2 }], - }); - - logSpy.mockClear(); - - return { - checkAgentVersionSpy, - getActiveSandboxSessionsSpy, - getSandboxDockerRuntimeSpy, - logSpy, - showSandboxStatus: requireDist(statusModulePath).showSandboxStatus, - } satisfies StatusFlowHarness; -} +import { + createStatusFlowHarness, + resetStatusFlowModuleCache, +} from "../../../../test/support/status-flow-test-harness"; describe("showSandboxStatus flow", () => { let exitSpy: MockInstance; @@ -185,7 +21,7 @@ describe("showSandboxStatus flow", () => { afterEach(() => { vi.restoreAllMocks(); process.exitCode = undefined; - delete require.cache[requireDist.resolve(statusModulePath)]; + resetStatusFlowModuleCache(); }); it("prints the live sandbox, inference, runtime, session, version, and recovery signals", async () => { @@ -252,6 +88,207 @@ describe("showSandboxStatus flow", () => { expect(output).toContain("No local registry entry was removed by this status check"); expect(output).toContain("nemoclaw alpha status"); expect(exitSpy).toHaveBeenCalledWith(1); + expect(harness.removeSandboxSpy).not.toHaveBeenCalled(); expect(harness.getSandboxDockerRuntimeSpy).not.toHaveBeenCalled(); }); + + it("prints switch guidance without removing registry state for a wrong active gateway (#2276)", async () => { + const harness = createStatusFlowHarness({ + inferenceHealth: null, + lookup: { + state: "wrong_gateway_active", + activeGateway: "openshell", + output: "Gateway: openshell\nStatus: Connected", + }, + }); + + await expect(harness.showSandboxStatus("alpha")).rejects.toThrow("process.exit(1)"); + + const output = harness.logSpy.mock.calls.flat().join("\n"); + expect(output).toContain("Your sandbox has NOT been removed"); + expect(output).toContain("openshell gateway select nemoclaw"); + expect(harness.removeSandboxSpy).not.toHaveBeenCalled(); + }); + + it("renders a local Ollama outage with the backend endpoint and recovery hint", async () => { + const harness = createStatusFlowHarness({ + currentModel: "llama3.2:1b", + currentProvider: "ollama-local", + inferenceHealth: { + ok: false, + probed: true, + providerLabel: "Ollama", + endpoint: "http://127.0.0.1:11434/api/tags", + detail: "Start Ollama and retry", + probeLabel: "ollama backend", + failureLabel: "unreachable", + }, + sandboxEntry: { + model: "llama3.2:1b", + provider: "ollama-local", + }, + }); + + await expect(harness.showSandboxStatus("alpha")).resolves.toBeUndefined(); + + const output = harness.logSpy.mock.calls.flat().join("\n"); + expect(output).toContain("Inference (ollama backend):"); + expect(output).toContain("unreachable"); + expect(output).toContain("Start Ollama and retry"); + expect(output).toContain("http://127.0.0.1:11434/api/tags"); + }); + + it("renders fresh shields posture as not configured rather than down", async () => { + const harness = createStatusFlowHarness({ + shieldsPosture: { + mode: "mutable_default", + detail: "not configured (default mutable state)", + }, + }); + + await expect(harness.showSandboxStatus("alpha")).resolves.toBeUndefined(); + + const output = harness.logSpy.mock.calls.flat().join("\n"); + expect(output).toContain("Permissions: not configured (default mutable state)"); + expect(output).not.toContain("Permissions: shields down"); + }); + + it("renders the live agent version instead of stale registry metadata", async () => { + const harness = createStatusFlowHarness({ + sandboxEntry: { agentVersion: "2026.5.18" }, + versionCheck: { + sandboxVersion: "2026.3.11", + expectedVersion: "2026.6.1", + isStale: true, + detectionMethod: "runtime", + }, + }); + + await expect(harness.showSandboxStatus("alpha")).resolves.toBeUndefined(); + + const output = harness.logSpy.mock.calls.flat().join("\n"); + expect(output).toContain("Agent: OpenClaw v2026.3.11"); + expect(output).toContain("Update:"); + expect(output).toContain("v2026.6.1 available"); + expect(output).toContain("Run `nemoclaw alpha rebuild` to upgrade"); + expect(output).not.toContain("Agent: OpenClaw v2026.5.18"); + expect(harness.checkAgentVersionSpy).toHaveBeenCalledWith("alpha", { + forceProbe: true, + skipProbe: false, + }); + }); + + it("does not report inference healthy when gateway verification fails", async () => { + const harness = createStatusFlowHarness({ + inferenceHealth: null, + lookup: { + state: "gateway_unreachable_after_restart", + output: "Gateway: nemoclaw\nclient error (Connect): Connection refused (os error 111)", + }, + preflight: { + failure: null, + failureLayer: "docker_unreachable", + suppressInferenceProbe: true, + exitCode: 1, + }, + }); + + await expect(harness.showSandboxStatus("alpha")).rejects.toThrow("process.exit(1)"); + + const output = harness.logSpy.mock.calls.flat().join("\n"); + expect(output).not.toContain("Inference: healthy"); + expect(output).toContain("Inference: not verified (gateway/sandbox state not verified)"); + expect(output).toContain("gateway is still refusing connections after restart"); + expect(output).toContain("Retry `openshell gateway start --name nemoclaw`"); + expect(output).toContain("If the gateway never becomes healthy"); + expect(harness.collectSandboxStatusSnapshotSpy).toHaveBeenCalledWith("alpha", { + suppressInferenceProbe: true, + }); + }); + + it("renders missing gateway metadata after restart without claiming recovery", async () => { + const harness = createStatusFlowHarness({ + inferenceHealth: null, + lookup: { + state: "gateway_missing_after_restart", + output: "Status: No gateway configured.", + }, + preflight: { + failure: null, + failureLayer: "docker_unreachable", + suppressInferenceProbe: true, + exitCode: 1, + }, + }); + + await expect(harness.showSandboxStatus("alpha")).rejects.toThrow("process.exit(1)"); + + const output = harness.logSpy.mock.calls.flat().join("\n"); + expect(output).toContain("gateway is no longer configured after restart/rebuild"); + expect(output).toContain("Start the gateway again"); + expect(output).not.toContain("Recovered NemoClaw gateway runtime"); + }); + + it("renders gateway identity drift as an unsafe reattachment", async () => { + const harness = createStatusFlowHarness({ + inferenceHealth: null, + lookup: { + state: "identity_drift", + output: "Error: transport error: handshake verification failed", + }, + }); + + await expect(harness.showSandboxStatus("alpha")).rejects.toThrow("process.exit(1)"); + + const output = harness.logSpy.mock.calls.flat().join("\n"); + expect(output).toContain("gateway trust material rotated after restart"); + expect(output).toContain("cannot be reattached safely"); + expect(output).not.toContain("Inference: healthy"); + }); + + it("keeps a failed foreign-gateway lookup distinct from recovered status", async () => { + const harness = createStatusFlowHarness({ + inferenceHealth: null, + lookup: { + state: "gateway_error", + output: "Error: transport error: Connection refused", + }, + preflight: { + failure: null, + failureLayer: "docker_unreachable", + suppressInferenceProbe: true, + exitCode: 1, + }, + }); + + await expect(harness.showSandboxStatus("alpha")).rejects.toThrow("process.exit(1)"); + + const output = harness.logSpy.mock.calls.flat().join("\n"); + expect(output).toContain("Could not verify sandbox 'alpha'"); + expect(output).toContain("verify the active gateway"); + expect(output).not.toContain("Recovered NemoClaw gateway runtime"); + }); + + it("renders gateway-level handshake failures without removing registry state", async () => { + const harness = createStatusFlowHarness({ + inferenceHealth: null, + lookup: { + state: "gateway_error", + output: "Error: transport error: handshake verification failed", + }, + preflight: { + failure: null, + failureLayer: "docker_unreachable", + suppressInferenceProbe: true, + exitCode: 1, + }, + }); + + await expect(harness.showSandboxStatus("alpha")).rejects.toThrow("process.exit(1)"); + + const output = harness.logSpy.mock.calls.flat().join("\n"); + expect(output).toContain("Could not verify sandbox 'alpha'"); + expect(output).toContain("gateway identity drift after restart"); + expect(harness.removeSandboxSpy).not.toHaveBeenCalled(); + }); }); diff --git a/src/lib/cli/argv-normalizer.test.ts b/src/lib/cli/argv-normalizer.test.ts index 4ab9330ce66..07bc742be3d 100644 --- a/src/lib/cli/argv-normalizer.test.ts +++ b/src/lib/cli/argv-normalizer.test.ts @@ -81,6 +81,18 @@ describe("normalizeArgv", () => { actionArgs: ["--help"], connectHelpRequested: true, }); + expect( + normalizeArgv(["alpha", "--help"], { + globalCommands, + isSandboxConnectFlag: isConnectFlag, + }), + ).toMatchObject({ + kind: "sandbox", + sandboxName: "alpha", + action: "connect", + actionArgs: ["--help"], + connectHelpRequested: true, + }); }); }); diff --git a/src/lib/gateway-runtime-action.test.ts b/src/lib/gateway-runtime-action.test.ts index feac6a492f8..db740620282 100644 --- a/src/lib/gateway-runtime-action.test.ts +++ b/src/lib/gateway-runtime-action.test.ts @@ -78,6 +78,43 @@ describe("gateway-runtime-action per-sandbox gateway routing", () => { expect(result.activeGateway).toBe("nemoclaw"); }); + it.each([ + { + label: "failed gateway metadata under a connected foreign gateway", + status: "Gateway: openshell\nStatus: Connected\n", + gatewayInfo: "No gateway metadata found", + gatewayInfoStatus: 1, + expected: "connected_other", + }, + { + label: "empty lifecycle output", + status: "", + gatewayInfo: "", + gatewayInfoStatus: 0, + expected: "missing_named", + }, + { + label: "malformed lifecycle output", + status: "??? garbage output ???", + gatewayInfo: "garbage gateway info", + gatewayInfoStatus: 0, + expected: "missing_named", + }, + ])("classifies $label conservatively as $expected", ({ + status, + gatewayInfo, + gatewayInfoStatus, + expected, + }) => { + captureSpy + .mockReturnValueOnce({ status: 0, output: status }) + .mockReturnValueOnce({ status: gatewayInfoStatus, output: gatewayInfo }); + + const result = gatewayRuntime.getNamedGatewayLifecycleState("nemoclaw"); + + expect(result.state).toBe(expected); + }); + it("keeps probes fatal by default, but still captures stderr (ignoreError falsy)", () => { captureSpy.mockReturnValue({ status: 0, output: "Status: Connected\nGateway: nemoclaw\n" }); diff --git a/src/lib/registry-recovery-action.test.ts b/src/lib/registry-recovery-action.test.ts index e23e6fab92d..fbd69dfc089 100644 --- a/src/lib/registry-recovery-action.test.ts +++ b/src/lib/registry-recovery-action.test.ts @@ -73,10 +73,26 @@ import { recoverRegistryEntries } from "./registry-recovery-action.js"; import { parseLiveSandboxEntries } from "./runtime-recovery.js"; import { loadSession } from "./state/onboard-session.js"; +function resetRegistryRecoveryDependencyMocks(): void { + vi.mocked(loadSession).mockReset().mockReturnValue(null); + vi.mocked(resolveOpenshell).mockReset().mockReturnValue(null); + vi.mocked(recoverNamedGatewayRuntime) + .mockReset() + .mockResolvedValue({ recovered: false } as never); + vi.mocked(getNamedGatewayLifecycleState) + .mockReset() + .mockReturnValue({ state: "missing_named" } as never); + vi.mocked(captureOpenshell) + .mockReset() + .mockReturnValue({ output: "", status: 0 } as never); + vi.mocked(parseLiveSandboxEntries).mockReset().mockReturnValue([]); +} + describe("recoverRegistryEntries seed-time guard (#2753)", () => { beforeEach(() => { mockRegistryState.sandboxes = {}; mockRegistryState.defaultSandbox = null; + resetRegistryRecoveryDependencyMocks(); }); afterEach(() => { @@ -228,14 +244,10 @@ describe("recoverRegistryEntries seed-time guard (#2753)", () => { describe("recoverRegistryEntries empty-registry live gateway recovery (#5714)", () => { beforeEach(() => { - vi.clearAllMocks(); mockRegistryState.sandboxes = {}; mockRegistryState.defaultSandbox = null; - vi.mocked(loadSession).mockReturnValue(null); + resetRegistryRecoveryDependencyMocks(); vi.mocked(resolveOpenshell).mockReturnValue("/usr/bin/openshell"); - vi.mocked(captureOpenshell).mockReturnValue({ output: "", status: 0 } as never); - vi.mocked(parseLiveSandboxEntries).mockReturnValue([]); - vi.mocked(getNamedGatewayLifecycleState).mockReturnValue({ state: "missing_named" } as never); }); afterEach(() => { @@ -274,41 +286,6 @@ describe("recoverRegistryEntries empty-registry live gateway recovery (#5714)", expect(recovered?.livePhase).toBe("Ready"); }); - it("treats an incomplete (phantom) session as unseeded — stays in read-only/display-only path", async () => { - // PRA-2: a session that recorded sandboxName but whose sandbox step never - // completed is a phantom (#2753). It must NOT count as a recovery seed, - // otherwise an empty registry + phantom session would take the mutating, - // persisting seeded path. Recovery must stay read-only/display-only. - vi.mocked(loadSession).mockReturnValue({ - sandboxName: "phantom", - provider: "nvidia", - model: "nemotron", - policyPresets: [], - nimContainer: null, - steps: { - sandbox: { status: "pending", startedAt: null, completedAt: null, error: null }, - }, - } as never); - vi.mocked(getNamedGatewayLifecycleState).mockReturnValue({ state: "healthy_named" } as never); - vi.mocked(parseLiveSandboxEntries).mockReturnValue([{ name: "dcode-station", phase: "Ready" }]); - - const result = await recoverRegistryEntries(); - - // Read-only path: never invokes the mutating gateway recovery, inspects - // lifecycle directly, and surfaces the live sandbox display-only. - expect(recoverNamedGatewayRuntime).not.toHaveBeenCalled(); - expect(getNamedGatewayLifecycleState).toHaveBeenCalledWith(undefined, { - ignoreProbeErrors: true, - }); - const recovered = result.sandboxes.find((s) => s.name === "dcode-station") as - | { recoveredFromGateway?: boolean } - | undefined; - expect(recovered?.recoveredFromGateway).toBe(true); - // Nothing persisted — neither the phantom session sandbox nor the recovered one. - expect(mockRegistryState.sandboxes["dcode-station"]).toBeUndefined(); - expect(mockRegistryState.sandboxes["phantom"]).toBeUndefined(); - }); - it("incomplete session with existing registry entries does not trigger mutating gateway recovery solely because the phantom session name is missing", async () => { // PRA-5: with an existing registry entry plus an incomplete (phantom) // session naming a DIFFERENT, missing sandbox, recovery must not flip on and diff --git a/src/lib/registry-recovery-seeded-paths.test.ts b/src/lib/registry-recovery-seeded-paths.test.ts new file mode 100644 index 00000000000..cfef57cdb4f --- /dev/null +++ b/src/lib/registry-recovery-seeded-paths.test.ts @@ -0,0 +1,209 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import type { SandboxEntry } from "./state/registry.js"; + +interface MockRegistryState { + sandboxes: Record; + defaultSandbox: string | null; +} + +const mockRegistryState: MockRegistryState = { sandboxes: {}, defaultSandbox: null }; + +vi.mock("./state/registry.js", () => ({ + listSandboxes: () => ({ + sandboxes: Object.values(mockRegistryState.sandboxes), + defaultSandbox: mockRegistryState.defaultSandbox, + }), + getSandbox: (name: string) => mockRegistryState.sandboxes[name] ?? null, + registerSandbox: (entry: SandboxEntry) => { + mockRegistryState.sandboxes[entry.name] = entry; + }, + updateSandbox: (name: string, partial: Partial) => { + mockRegistryState.sandboxes[name] = { + ...mockRegistryState.sandboxes[name], + ...partial, + } as SandboxEntry; + }, + setDefault: (name: string) => { + mockRegistryState.defaultSandbox = name; + }, +})); + +vi.mock("./adapters/openshell/resolve.js", () => ({ + resolveOpenshell: vi.fn(), +})); + +vi.mock("./gateway-runtime-action.js", () => ({ + recoverNamedGatewayRuntime: vi.fn(), + getNamedGatewayLifecycleState: vi.fn(), +})); + +vi.mock("./adapters/openshell/runtime.js", () => ({ + captureOpenshell: vi.fn(), +})); + +vi.mock("./state/onboard-session.js", () => ({ + loadSession: vi.fn(), +})); + +vi.mock("./runtime-recovery.js", () => ({ + parseLiveSandboxEntries: vi.fn(), +})); + +vi.mock("./runner.js", async () => { + const actual = await vi.importActual("./runner.js"); + return { validateName: actual.validateName }; +}); + +import { resolveOpenshell } from "./adapters/openshell/resolve.js"; +import { captureOpenshell } from "./adapters/openshell/runtime.js"; +import { + getNamedGatewayLifecycleState, + recoverNamedGatewayRuntime, +} from "./gateway-runtime-action.js"; +import { recoverRegistryEntries } from "./registry-recovery-action.js"; +import { parseLiveSandboxEntries } from "./runtime-recovery.js"; +import { loadSession } from "./state/onboard-session.js"; + +const gammaEntry = (policies: string[]): SandboxEntry => ({ + name: "gamma", + provider: "existing-provider", + model: "existing-model", + gpuEnabled: false, + policies, +}); + +const completedSession = (sandboxName: string, policyPresets: string[]) => + ({ + sandboxName, + provider: "nvidia-prod", + model: "nvidia/nemotron-3-super-120b-a12b", + policyPresets, + nimContainer: null, + steps: { + sandbox: { status: "complete", startedAt: null, completedAt: null, error: null }, + }, + }) as never; + +function resetSeededRecoveryMocks(): void { + mockRegistryState.sandboxes = {}; + mockRegistryState.defaultSandbox = null; + vi.mocked(loadSession).mockReset().mockReturnValue(null); + vi.mocked(resolveOpenshell).mockReset().mockReturnValue("/usr/bin/openshell"); + vi.mocked(recoverNamedGatewayRuntime) + .mockReset() + .mockResolvedValue({ recovered: true } as never); + vi.mocked(getNamedGatewayLifecycleState) + .mockReset() + .mockReturnValue({ state: "missing_named" } as never); + vi.mocked(captureOpenshell) + .mockReset() + .mockReturnValue({ output: "live sandboxes", status: 0 } as never); + vi.mocked(parseLiveSandboxEntries).mockReset().mockReturnValue([]); +} + +describe("recoverRegistryEntries seeded recovery paths", () => { + beforeEach(resetSeededRecoveryMocks); + + it("merges a confirmed session and additional live sandboxes without replacing the default", async () => { + mockRegistryState.sandboxes.gamma = gammaEntry(["npm"]); + mockRegistryState.defaultSandbox = "gamma"; + vi.mocked(loadSession).mockReturnValue(completedSession("alpha", ["pypi"])); + vi.mocked(parseLiveSandboxEntries).mockReturnValue([ + { name: "alpha", phase: "Ready" }, + { name: "beta", phase: "Ready" }, + ]); + + const result = await recoverRegistryEntries(); + + expect(result.recoveredFromSession).toBe(true); + expect(result.recoveredFromGateway).toBe(1); + expect(result.sandboxes.map((sandbox) => sandbox.name).sort()).toEqual([ + "alpha", + "beta", + "gamma", + ]); + expect(mockRegistryState.sandboxes.alpha?.policies).toEqual(["pypi"]); + expect(mockRegistryState.defaultSandbox).toBe("gamma"); + }); + + it("skips invalid session and live sandbox names during seeded recovery", async () => { + mockRegistryState.sandboxes.gamma = gammaEntry([]); + mockRegistryState.defaultSandbox = "gamma"; + vi.mocked(loadSession).mockReturnValue(completedSession("Alpha", [])); + vi.mocked(parseLiveSandboxEntries).mockReturnValue([ + { name: "alpha", phase: "Ready" }, + { name: "Bad_Name", phase: "Ready" }, + ]); + + const result = await recoverRegistryEntries(); + + expect(result.sandboxes.map((sandbox) => sandbox.name).sort()).toEqual(["alpha", "gamma"]); + expect(mockRegistryState.sandboxes.Alpha).toBeUndefined(); + expect(mockRegistryState.sandboxes.Bad_Name).toBeUndefined(); + expect(mockRegistryState.defaultSandbox).toBe("gamma"); + }); + + it("treats an incomplete (phantom) session as unseeded — stays in read-only/display-only path", async () => { + // PRA-2: a session that recorded sandboxName but whose sandbox step never + // completed is a phantom (#2753). It must NOT count as a recovery seed, + // otherwise an empty registry + phantom session would take the mutating, + // persisting seeded path. Recovery must stay read-only/display-only. + vi.mocked(loadSession).mockReturnValue({ + sandboxName: "phantom", + provider: "nvidia", + model: "nemotron", + policyPresets: [], + nimContainer: null, + steps: { + sandbox: { status: "pending", startedAt: null, completedAt: null, error: null }, + }, + } as never); + vi.mocked(getNamedGatewayLifecycleState).mockReturnValue({ state: "healthy_named" } as never); + vi.mocked(parseLiveSandboxEntries).mockReturnValue([{ name: "dcode-station", phase: "Ready" }]); + + const result = await recoverRegistryEntries(); + + // Read-only path: never invokes the mutating gateway recovery, inspects + // lifecycle directly, and surfaces the live sandbox display-only. + expect(recoverNamedGatewayRuntime).not.toHaveBeenCalled(); + expect(getNamedGatewayLifecycleState).toHaveBeenCalledWith(undefined, { + ignoreProbeErrors: true, + }); + const recovered = result.sandboxes.find((s) => s.name === "dcode-station") as + | { recoveredFromGateway?: boolean } + | undefined; + expect(recovered?.recoveredFromGateway).toBe(true); + // Nothing persisted — neither the phantom session sandbox nor the recovered one. + expect(mockRegistryState.sandboxes["dcode-station"]).toBeUndefined(); + expect(mockRegistryState.sandboxes["phantom"]).toBeUndefined(); + }); + + it("persists a requested live sandbox and makes it the default", async () => { + vi.mocked(parseLiveSandboxEntries).mockReturnValue([{ name: "alpha", phase: "Ready" }]); + + const result = await recoverRegistryEntries({ requestedSandboxName: "alpha" }); + + expect(recoverNamedGatewayRuntime).toHaveBeenCalledOnce(); + expect(getNamedGatewayLifecycleState).not.toHaveBeenCalled(); + expect(result.recoveredFromGateway).toBe(1); + expect(result.sandboxes.map((sandbox) => sandbox.name)).toEqual(["alpha"]); + expect(mockRegistryState.sandboxes.alpha).toBeDefined(); + expect(mockRegistryState.defaultSandbox).toBe("alpha"); + }); + + it("keeps a missing requested sandbox absent while recovering other live entries", async () => { + vi.mocked(parseLiveSandboxEntries).mockReturnValue([{ name: "alpha", phase: "Ready" }]); + + const result = await recoverRegistryEntries({ requestedSandboxName: "beta" }); + + expect(result.recoveredFromGateway).toBe(1); + expect(result.sandboxes.map((sandbox) => sandbox.name)).toEqual(["alpha"]); + expect(mockRegistryState.sandboxes.alpha).toBeDefined(); + expect(mockRegistryState.sandboxes.beta).toBeUndefined(); + expect(mockRegistryState.defaultSandbox).toBeNull(); + }); +}); diff --git a/test/cli-oclif-compatibility.test.ts b/test/cli-oclif-compatibility.test.ts index 912a7b025be..5b65f8d0c16 100644 --- a/test/cli-oclif-compatibility.test.ts +++ b/test/cli-oclif-compatibility.test.ts @@ -8,6 +8,9 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import SandboxStatusCommand from "../src/commands/sandbox/status"; +import StatusCommand from "../src/commands/status"; + const require = createRequire(import.meta.url); const requireCache: Record = require.cache as any; @@ -16,6 +19,163 @@ function restoreCache(path: string, prior: unknown): void { else delete requireCache[path]; } +type DirectStatusDispatchHarness = { + dispatchCli: (argv: string[]) => Promise; + exitSpy: ReturnType; + runOclifArgv: ReturnType; + runOclifCommandById: ReturnType; + stderr: string[]; +}; + +async function withDirectStatusDispatch( + run: (harness: DirectStatusDispatchHarness) => Promise, +): Promise { + const publicDispatchPath = require.resolve("../src/lib/cli/public-dispatch.js"); + const oclifRunnerPath = require.resolve("../src/lib/cli/oclif-runner.js"); + const sandboxConnectPath = require.resolve("../src/lib/actions/sandbox/connect.js"); + const priorPublicDispatch = require.cache[publicDispatchPath]; + const priorOclifRunner = require.cache[oclifRunnerPath]; + const priorSandboxConnect = require.cache[sandboxConnectPath]; + const runOclifArgv = vi.fn(async () => undefined); + const runOclifCommandById = vi.fn(async () => undefined); + const stderr: string[] = []; + const errorSpy = vi.spyOn(console, "error").mockImplementation((message = "") => { + stderr.push(String(message)); + }); + const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number | string | null) => { + throw new Error(`process.exit:${String(code)}`); + }) as never); + + requireCache[oclifRunnerPath] = { + id: oclifRunnerPath, + filename: oclifRunnerPath, + loaded: true, + exports: { runOclifArgv, runOclifCommandById }, + } as any; + requireCache[sandboxConnectPath] = { + id: sandboxConnectPath, + filename: sandboxConnectPath, + loaded: true, + exports: { + isSandboxConnectFlag: vi.fn(() => false), + parseSandboxConnectArgs: vi.fn(), + printSandboxConnectHelp: vi.fn(), + }, + } as any; + + try { + delete require.cache[publicDispatchPath]; + const { dispatchCli } = require(publicDispatchPath); + await run({ dispatchCli, exitSpy, runOclifArgv, runOclifCommandById, stderr }); + } finally { + errorSpy.mockRestore(); + exitSpy.mockRestore(); + restoreCache(publicDispatchPath, priorPublicDispatch); + restoreCache(oclifRunnerPath, priorOclifRunner); + restoreCache(sandboxConnectPath, priorSandboxConnect); + } +} + +type DirectSandboxRecoveryDispatchHarness = { + dispatchCli: (argv: string[]) => Promise; + exitSpy: ReturnType; + getSandbox: ReturnType; + listSandboxes: ReturnType; + recoverRegistryEntries: ReturnType; + runOclifArgv: ReturnType; + runOclifCommandById: ReturnType; + sandboxes: Map; + stderr: string[]; +}; + +async function withDirectSandboxRecoveryDispatch( + run: (harness: DirectSandboxRecoveryDispatchHarness) => Promise, +): Promise { + const publicDispatchPath = require.resolve("../src/lib/cli/public-dispatch.js"); + const oclifRunnerPath = require.resolve("../src/lib/cli/oclif-runner.js"); + const sandboxConnectPath = require.resolve("../src/lib/actions/sandbox/connect.js"); + const registryPath = require.resolve("../src/lib/state/registry.js"); + const registryRecoveryPath = require.resolve("../src/lib/registry-recovery-action.js"); + const priorPublicDispatch = require.cache[publicDispatchPath]; + const priorOclifRunner = require.cache[oclifRunnerPath]; + const priorSandboxConnect = require.cache[sandboxConnectPath]; + const priorRegistry = require.cache[registryPath]; + const priorRegistryRecovery = require.cache[registryRecoveryPath]; + const sandboxes = new Map(); + const getSandbox = vi.fn((name: string) => sandboxes.get(name) ?? null); + const listSandboxes = vi.fn(() => ({ + sandboxes: [...sandboxes.values()], + defaultSandbox: null, + })); + const recoverRegistryEntries = vi.fn(async () => ({ + ...listSandboxes(), + recoveredFromSession: false, + recoveredFromGateway: 0, + })); + const runOclifArgv = vi.fn(async () => undefined); + const runOclifCommandById = vi.fn(async () => undefined); + const stderr: string[] = []; + const errorSpy = vi.spyOn(console, "error").mockImplementation((message = "") => { + stderr.push(String(message)); + }); + const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number | string | null) => { + throw new Error(`process.exit:${String(code)}`); + }) as never); + + requireCache[registryPath] = { + id: registryPath, + filename: registryPath, + loaded: true, + exports: { getSandbox, listSandboxes }, + } as any; + requireCache[registryRecoveryPath] = { + id: registryRecoveryPath, + filename: registryRecoveryPath, + loaded: true, + exports: { recoverRegistryEntries }, + } as any; + requireCache[oclifRunnerPath] = { + id: oclifRunnerPath, + filename: oclifRunnerPath, + loaded: true, + exports: { runOclifArgv, runOclifCommandById }, + } as any; + requireCache[sandboxConnectPath] = { + id: sandboxConnectPath, + filename: sandboxConnectPath, + loaded: true, + exports: { + isSandboxConnectFlag: vi.fn(() => false), + parseSandboxConnectArgs: vi.fn(), + printSandboxConnectHelp: vi.fn(), + }, + } as any; + + try { + delete require.cache[publicDispatchPath]; + const { dispatchCli } = require(publicDispatchPath); + await run({ + dispatchCli, + exitSpy, + getSandbox, + listSandboxes, + recoverRegistryEntries, + runOclifArgv, + runOclifCommandById, + sandboxes, + stderr, + }); + } finally { + errorSpy.mockRestore(); + exitSpy.mockRestore(); + restoreCache(publicDispatchPath, priorPublicDispatch); + restoreCache(oclifRunnerPath, priorOclifRunner); + restoreCache(sandboxConnectPath, priorSandboxConnect); + restoreCache(registryPath, priorRegistry); + restoreCache(registryRecoveryPath, priorRegistryRecovery); + } +} + describe("oclif compatibility dispatch", () => { afterEach(() => { vi.restoreAllMocks(); @@ -221,6 +381,87 @@ describe("oclif compatibility dispatch", () => { } }); + it("recovers a requested sandbox, rereads the registry, and dispatches connect", async () => { + await withDirectSandboxRecoveryDispatch( + async ({ + dispatchCli, + getSandbox, + recoverRegistryEntries, + runOclifArgv, + runOclifCommandById, + sandboxes, + stderr, + }) => { + recoverRegistryEntries.mockImplementationOnce( + async ({ requestedSandboxName }: { requestedSandboxName: string }) => { + expect(requestedSandboxName).toBe("alpha"); + sandboxes.set("alpha", { name: "alpha" }); + return { + sandboxes: [...sandboxes.values()], + defaultSandbox: "alpha", + recoveredFromSession: true, + recoveredFromGateway: 0, + }; + }, + ); + + await dispatchCli(["alpha", "connect"]); + + expect(recoverRegistryEntries).toHaveBeenCalledWith({ requestedSandboxName: "alpha" }); + expect(getSandbox.mock.results[0]?.value).toBeNull(); + expect( + getSandbox.mock.results.slice(1).some((result) => result.value?.name === "alpha"), + ).toBe(true); + expect(runOclifCommandById).toHaveBeenCalledWith( + "sandbox:connect", + ["alpha"], + expect.objectContaining({ rootDir: process.cwd() }), + ); + expect(runOclifArgv).not.toHaveBeenCalled(); + expect(stderr).toEqual([]); + }, + ); + }); + + it("guides a missing requested sandbox after recovery finds a different live sandbox", async () => { + await withDirectSandboxRecoveryDispatch( + async ({ + dispatchCli, + exitSpy, + listSandboxes, + recoverRegistryEntries, + runOclifArgv, + runOclifCommandById, + sandboxes, + stderr, + }) => { + recoverRegistryEntries.mockImplementationOnce( + async ({ requestedSandboxName }: { requestedSandboxName: string }) => { + expect(requestedSandboxName).toBe("beta"); + sandboxes.set("alpha", { name: "alpha" }); + return { + sandboxes: [...sandboxes.values()], + defaultSandbox: "alpha", + recoveredFromSession: true, + recoveredFromGateway: 0, + }; + }, + ); + + await expect(dispatchCli(["beta", "connect"])).rejects.toThrow("process.exit:1"); + + expect(recoverRegistryEntries).toHaveBeenCalledWith({ requestedSandboxName: "beta" }); + expect(listSandboxes).toHaveBeenCalled(); + expect(stderr.join("\n")).toContain("Sandbox 'beta' does not exist."); + expect(stderr.join("\n")).toContain("Registered sandboxes: alpha"); + expect(stderr.join("\n")).toContain("Run 'nemoclaw list' to see all sandboxes."); + expect(exitSpy).toHaveBeenCalledWith(1); + expect(runOclifArgv).not.toHaveBeenCalled(); + expect(runOclifCommandById).not.toHaveBeenCalled(); + }, + ); + }); + it("forwards exec command help flags after -- instead of rendering NemoClaw help", async () => { const cliPath = require.resolve("../src/nemoclaw.js"); const registryPath = require.resolve("../src/lib/state/registry.js"); @@ -395,6 +636,124 @@ describe("oclif compatibility dispatch", () => { } }); + it("corrects a single sandbox-like global status argument without a CLI subprocess", async () => { + await withDirectStatusDispatch( + async ({ dispatchCli, exitSpy, runOclifArgv, runOclifCommandById, stderr }) => { + const cases = [ + { argv: ["status", "alpha"], command: "nemoclaw alpha status" }, + { argv: ["status", "--json", "alpha"], command: "nemoclaw alpha status --json" }, + { argv: ["status", "alpha", "--json"], command: "nemoclaw alpha status --json" }, + { argv: ["status", "alpha", "--help"], command: "nemoclaw alpha status --help" }, + { + argv: ["status", "alpha", "--json", "--help"], + command: "nemoclaw alpha status --help", + }, + { + argv: ["status", "alpha", "--help", "--json"], + command: "nemoclaw alpha status --help", + }, + ]; + + for (const { argv, command } of cases) { + stderr.length = 0; + exitSpy.mockClear(); + runOclifArgv.mockClear(); + runOclifCommandById.mockClear(); + + await expect(dispatchCli(argv)).rejects.toThrow("process.exit:2"); + + const output = stderr.join("\n"); + expect(output).toContain("'nemoclaw status' shows the global sandbox/service overview"); + expect(output).toContain(`Run: ${command}`); + expect(output).not.toContain("nemoclaw alpha status --json --help"); + expect(exitSpy).toHaveBeenCalledWith(2); + expect(runOclifArgv).not.toHaveBeenCalled(); + expect(runOclifCommandById).not.toHaveBeenCalled(); + } + }, + ); + }); + + it("leaves ambiguous or unsafe global status arguments to the strict parser", async () => { + await withDirectStatusDispatch( + async ({ dispatchCli, exitSpy, runOclifArgv, runOclifCommandById, stderr }) => { + const cases = [ + ["status", "--bogus"], + ["status", "--bogus", "alpha"], + ["status", "alpha", "--bogus"], + ["status", "alpha", "beta"], + ["status", "status"], + ["status", "help"], + ["status", "sandbox"], + ["status", "internal"], + ["status", "alpha;echo pwned"], + ]; + + for (const argv of cases) { + stderr.length = 0; + exitSpy.mockClear(); + runOclifArgv.mockClear(); + runOclifCommandById.mockClear(); + + await dispatchCli(argv); + + expect(runOclifCommandById).toHaveBeenCalledWith( + "status", + argv.slice(1), + expect.objectContaining({ rootDir: process.cwd() }), + ); + expect(runOclifArgv).not.toHaveBeenCalled(); + expect(exitSpy).not.toHaveBeenCalled(); + expect(stderr.join("\n")).not.toContain("does not take a sandbox name"); + expect(stderr.join("\n")).not.toContain("Run:"); + } + }, + ); + }); + + it("keeps strict status parser errors in process", async () => { + for (const args of [["--bogus"], ["--bogus", "alpha"], ["alpha", "--bogus"]]) { + await expect(StatusCommand.run(args, process.cwd())).rejects.toThrow( + "Nonexistent flag: --bogus", + ); + } + + await expect(StatusCommand.run(["alpha", "beta"], process.cwd())).rejects.toThrow( + "Unexpected arguments: alpha, beta", + ); + for (const token of ["status", "help", "sandbox", "internal", "alpha;echo pwned"]) { + await expect(StatusCommand.run([token], process.cwd())).rejects.toThrow( + `Unexpected argument: ${token}`, + ); + } + }); + + it("routes sandbox status help directly and keeps its JSON help metadata", async () => { + await withDirectStatusDispatch(async ({ dispatchCli, runOclifArgv, runOclifCommandById }) => { + await dispatchCli(["alpha", "status", "--help"]); + expect(runOclifCommandById).toHaveBeenCalledWith( + "sandbox:status", + ["alpha", "--help"], + expect.objectContaining({ rootDir: process.cwd() }), + ); + + await dispatchCli(["sandbox", "status", "alpha", "--help"]); + expect(runOclifArgv).toHaveBeenCalledWith( + ["sandbox", "status", "alpha", "--help"], + expect.objectContaining({ rootDir: process.cwd() }), + ); + }); + + expect(SandboxStatusCommand.enableJsonFlag).toBe(true); + expect(SandboxStatusCommand.usage.join(" ")).toContain(" [--json]"); + expect(SandboxStatusCommand.examples).toEqual( + expect.arrayContaining([ + "<%= config.bin %> alpha status", + "<%= config.bin %> sandbox status alpha --json", + ]), + ); + }); + it("uses the alias binary name in native oclif help", () => { const result = spawnSync( process.execPath, diff --git a/test/cli/connect-recovery.test.ts b/test/cli/connect-recovery.test.ts index 3b946de8cd2..51c74b08dbf 100644 --- a/test/cli/connect-recovery.test.ts +++ b/test/cli/connect-recovery.test.ts @@ -8,11 +8,8 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import { - execTimeout, runWithEnv, - testTimeout, testTimeoutOptions, - writeHealthyDockerStub, writeRecordingCommand, writeSandboxRegistry, } from "./helpers"; @@ -156,138 +153,7 @@ async function startForwardListeners(ports: number[]): Promise<() => Promise { - it("connect does not pre-start a duplicate port forward", testTimeoutOptions(15_000), () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-connect-forward-")); - const localBin = path.join(home, "bin"); - const registryDir = path.join(home, ".nemoclaw"); - const markerFile = path.join(home, "openshell-calls"); - fs.mkdirSync(localBin, { recursive: true }); - fs.mkdirSync(registryDir, { recursive: true }); - fs.writeFileSync( - path.join(registryDir, "sandboxes.json"), - JSON.stringify({ - sandboxes: { - alpha: { - name: "alpha", - model: "test-model", - provider: "nvidia-prod", - gpuEnabled: false, - policies: [], - }, - }, - defaultSandbox: "alpha", - }), - { mode: 0o600 }, - ); - fs.writeFileSync( - path.join(localBin, "openshell"), - [ - "#!/usr/bin/env bash", - `marker_file=${JSON.stringify(markerFile)}`, - 'printf \'%s\\n\' "$*" >> "$marker_file"', - 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', - " echo 'Sandbox:'", - " echo", - " echo ' Id: abc'", - " echo ' Name: alpha'", - " echo ' Namespace: openshell'", - " echo ' Phase: Ready'", - " exit 0", - "fi", - 'if [ "$1" = "sandbox" ] && [ "$2" = "list" ]; then', - " echo 'alpha Ready 2m ago'", - " exit 0", - "fi", - 'if [ "$1" = "sandbox" ] && [ "$2" = "connect" ] && [ "$3" = "alpha" ]; then', - " exit 0", - "fi", - 'if [ "$1" = "forward" ] && [ "$2" = "list" ]; then echo "alpha 127.0.0.1 18789 12345 running"; exit 0; fi', - 'if [ "$1" = "forward" ]; then exit 99; fi', - "exit 0", - ].join("\n"), - { mode: 0o755 }, - ); - fs.writeFileSync(path.join(localBin, "sleep"), ["#!/usr/bin/env bash", "exit 0"].join("\n"), { - mode: 0o755, - }); - - const r = runWithEnv("alpha connect", { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }); - - expect(r.code).toBe(0); - const calls = fs.readFileSync(markerFile, "utf8").trim().split("\n").filter(Boolean); - expect(calls).toContain("sandbox get alpha"); - expect(calls).toContain("sandbox connect alpha"); - expect(calls.some((call) => call.startsWith("forward start --background 18789"))).toBe(false); - }); - - it("shows connect help without opening an interactive session", () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-connect-help-")); - const localBin = path.join(home, "bin"); - const markerFile = path.join(home, "openshell-calls"); - const sshMarkerFile = path.join(home, "ssh-calls"); - fs.mkdirSync(localBin, { recursive: true }); - fs.writeFileSync( - path.join(localBin, "openshell"), - [ - "#!/usr/bin/env bash", - `printf '%s\\n' "$*" >> ${JSON.stringify(markerFile)}`, - "exit 99", - ].join("\n"), - { mode: 0o755 }, - ); - writeRecordingCommand(localBin, "ssh", sshMarkerFile, 98); - - const r = runWithEnv("alpha connect --help", { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }); - const implicit = runWithEnv("alpha --help", { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }); - - expect(r.code).toBe(0); - expect(r.out).toContain("Usage: nemoclaw alpha connect"); - expect(r.out).toContain("--probe-only"); - expect(implicit.code).toBe(0); - expect(implicit.out).toContain("Usage: nemoclaw alpha connect"); - expect(fs.existsSync(markerFile)).toBe(false); - expect(fs.existsSync(sshMarkerFile)).toBe(false); - }); - - it("rejects the removed skip-permissions connect flag", () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-connect-probe-flags-")); - const localBin = path.join(home, "bin"); - const markerFile = path.join(home, "openshell-calls"); - const sshMarkerFile = path.join(home, "ssh-calls"); - fs.mkdirSync(localBin, { recursive: true }); - fs.writeFileSync( - path.join(localBin, "openshell"), - [ - "#!/usr/bin/env bash", - `printf '%s\\n' "$*" >> ${JSON.stringify(markerFile)}`, - "exit 99", - ].join("\n"), - { mode: 0o755 }, - ); - writeRecordingCommand(localBin, "ssh", sshMarkerFile, 98); - writeSandboxRegistry(home); - - const r = runWithEnv("alpha connect --dangerously-skip-permissions", { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }); - - expect(r.code).toBe(1); - expect(r.out).toContain("--dangerously-skip-permissions was removed"); - expect(fs.existsSync(markerFile)).toBe(false); - expect(fs.existsSync(sshMarkerFile)).toBe(false); - }); - +describe("CLI connect recovery process contracts", () => { it( "connect --probe-only recovers the gateway without opening SSH", testTimeoutOptions(15_000), @@ -338,13 +204,13 @@ describe("CLI dispatch", () => { const stopForwardListeners = await startForwardListeners([18789]); try { - const r = runWithEnv("alpha connect --probe-only", { + const result = runWithEnv("alpha connect --probe-only", { HOME: home, PATH: `${localBin}:${process.env.PATH || ""}`, }); - expect(r.code).toBe(0); - expect(r.out).toContain("Probe complete: recovered OpenClaw gateway"); + expect(result.code).toBe(0); + expect(result.out).toContain("Probe complete: recovered OpenClaw gateway"); const calls = fs.readFileSync(markerFile, "utf8").trim().split("\n").filter(Boolean); expect(calls).toContain("sandbox get alpha"); expect(calls.some((call) => call.startsWith("sandbox exec --name alpha -- sh -c"))).toBe( @@ -360,24 +226,94 @@ describe("CLI dispatch", () => { }, ); - it("uses the authenticated recovery marker as the initial managed health proof", async () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-connect-probe-wait-")); + it( + "fails closed when privileged gateway recovery exits non-zero", + testTimeoutOptions(15_000), + async () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-connect-probe-failure-")); + const localBin = path.join(home, "bin"); + const openshellCalls = path.join(home, "openshell-calls"); + const dockerCalls = path.join(home, "docker-calls"); + const sshCalls = path.join(home, "ssh-calls"); + const stateFile = path.join(home, "probe-state"); + fs.mkdirSync(localBin, { recursive: true }); + writeSandboxRegistry(home); + fs.writeFileSync(stateFile, "stopped"); + fs.writeFileSync( + path.join(localBin, "openshell"), + [ + "#!/usr/bin/env bash", + `calls=${JSON.stringify(openshellCalls)}`, + `state_file=${JSON.stringify(stateFile)}`, + 'printf \'%s\\n\' "$*" >> "$calls"', + 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', + " echo 'Sandbox:'", + " echo", + " echo ' Id: abc'", + " echo ' Name: alpha'", + " echo ' Namespace: openshell'", + " echo ' Phase: Ready'", + " exit 0", + "fi", + 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ] && [ "$3" = "--name" ] && [ "$4" = "alpha" ]; then', + ' cmd="$8"', + ' if [[ "$cmd" == *"curl -so"* ]]; then', + " echo '__NEMOCLAW_SANDBOX_EXEC_STARTED__'", + ' if [ "$(cat "$state_file")" = recovered ]; then echo RUNNING; else echo STOPPED; fi', + " exit 0", + " fi", + "fi", + 'if [ "$1" = "forward" ] && [ "$2" = "list" ]; then echo "alpha 127.0.0.1 18789 12345 running"; exit 0; fi', + 'if [ "$1" = "forward" ]; then exit 99; fi', + "exit 0", + ].join("\n"), + { mode: 0o755 }, + ); + writeGatewayControlDockerStub(localBin, { + callsFile: dockerCalls, + stateFile, + recoveryStatus: 42, + }); + writeRecordingCommand(localBin, "ssh", sshCalls, 98); + const stopForwardListeners = await startForwardListeners([18789]); + + try { + const result = runWithEnv("alpha connect --probe-only", { + HOME: home, + PATH: `${localBin}:${process.env.PATH || ""}`, + }); + + expect(result.code).toBe(1); + expect(fs.readFileSync(stateFile, "utf8")).toBe("stopped"); + const openshellLog = fs.readFileSync(openshellCalls, "utf8"); + expect(openshellLog).toContain("sandbox exec --name alpha -- sh -c"); + expect(openshellLog).not.toContain("sandbox ssh-config alpha"); + expect(openshellLog).not.toContain("sandbox connect alpha"); + expect(fs.existsSync(sshCalls)).toBe(false); + expectGatewayControlRecovery(dockerCalls); + } finally { + await stopForwardListeners(); + } + }, + ); + + it("recovers stopped Hermes agents through privileged Docker control instead of SSH", async () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-connect-probe-agent-")); const localBin = path.join(home, "bin"); - const markerFile = path.join(home, "openshell-calls"); + const openshellCalls = path.join(home, "openshell-calls"); const dockerCalls = path.join(home, "docker-calls"); + const sshCalls = path.join(home, "ssh-calls"); const stateFile = path.join(home, "probe-state"); - const readyCountFile = path.join(home, "ready-count"); fs.mkdirSync(localBin, { recursive: true }); - writeSandboxRegistry(home); + writeSandboxRegistry(home, { agent: "hermes" }); fs.writeFileSync(stateFile, "stopped"); fs.writeFileSync( path.join(localBin, "openshell"), [ "#!/usr/bin/env bash", - `marker_file=${JSON.stringify(markerFile)}`, + `calls=${JSON.stringify(openshellCalls)}`, `state_file=${JSON.stringify(stateFile)}`, - `ready_count_file=${JSON.stringify(readyCountFile)}`, - 'printf \'%s\\n\' "$*" >> "$marker_file"', + 'printf \'%s\\n\' "$*" >> "$calls"', 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', " echo 'Sandbox:'", " echo", @@ -389,57 +325,119 @@ describe("CLI dispatch", () => { "fi", 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ] && [ "$3" = "--name" ] && [ "$4" = "alpha" ]; then', ' cmd="$8"', - ' case "$cmd" in', - " *'curl -so'*)", - " echo '__NEMOCLAW_SANDBOX_EXEC_STARTED__'", - ' if [ "$(cat "$state_file")" != recovered ]; then echo STOPPED; exit 0; fi', - ' count=$(cat "$ready_count_file" 2>/dev/null || echo 0)', - " count=$((count + 1))", - ' echo "$count" > "$ready_count_file"', - ' if [ "$count" -ge 3 ]; then echo RUNNING; else echo STOPPED; fi', - " exit 0", - " ;;", - " esac", + ' if [[ "$cmd" == *"curl -so"* ]]; then', + " echo '__NEMOCLAW_SANDBOX_EXEC_STARTED__'", + ' if [ "$(cat "$state_file")" = recovered ]; then echo RUNNING; else echo STOPPED; fi', + " exit 0", + " fi", + "fi", + 'if [ "$1" = "sandbox" ] && [ "$2" = "ssh-config" ] && [ "$3" = "alpha" ]; then', + ' echo UNEXPECTED_SSH_CONFIG >> "$calls"', + " exit 1", "fi", - 'if [ "$1" = "forward" ] && [ "$2" = "list" ]; then echo "alpha 127.0.0.1 18789 12345 running"; exit 0; fi', + 'if [ "$1" = "forward" ] && [ "$2" = "list" ]; then { echo "alpha 127.0.0.1 18789 12345 running"; echo "alpha 127.0.0.1 8642 12346 running"; }; exit 0; fi', 'if [ "$1" = "forward" ]; then exit 99; fi', "exit 0", ].join("\n"), { mode: 0o755 }, ); writeGatewayControlDockerStub(localBin, { callsFile: dockerCalls, stateFile }); - const stopForwardListeners = await startForwardListeners([18789]); + writeRecordingCommand(localBin, "ssh", sshCalls, 98); + const stopForwardListeners = await startForwardListeners([18789, 8642]); try { - const r = runWithEnv("alpha connect --probe-only", { + const result = runWithEnv("alpha connect --probe-only", { HOME: home, PATH: `${localBin}:${process.env.PATH || ""}`, - NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS: "3", - NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS: "0", }); - expect(r.code).toBe(0); - expect(r.out).toContain("Probe complete: recovered OpenClaw gateway"); - expect(fs.existsSync(readyCountFile)).toBe(false); + expect(result.code).toBe(0); + expect(result.out).toContain("Probe complete: recovered Hermes Agent gateway"); + const openshellLog = fs.readFileSync(openshellCalls, "utf8"); + expect(openshellLog).toContain("sandbox exec --name alpha -- sh -c"); + expect(openshellLog).not.toContain("sandbox ssh-config alpha"); + expect(openshellLog).not.toContain("sandbox connect"); + expect(fs.existsSync(sshCalls)).toBe(false); expectGatewayControlRecovery(dockerCalls); } finally { await stopForwardListeners(); } }); - it("treats leading --probe-only as an implicit connect probe", async () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-connect-probe-leading-")); + it("connect recovers a named sandbox from the last onboard session when the registry is empty", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-connect-recover-session-")); const localBin = path.join(home, "bin"); - const markerFile = path.join(home, "openshell-calls"); - const sshMarkerFile = path.join(home, "ssh-calls"); + const nemoclawDir = path.join(home, ".nemoclaw"); + const markerFile = path.join(home, "connect-args"); fs.mkdirSync(localBin, { recursive: true }); - writeSandboxRegistry(home); + fs.mkdirSync(nemoclawDir, { recursive: true }); + fs.writeFileSync( + path.join(nemoclawDir, "onboard-session.json"), + JSON.stringify( + { + version: 1, + sessionId: "session-1", + resumable: true, + status: "complete", + mode: "interactive", + startedAt: "2026-03-31T00:00:00.000Z", + updatedAt: "2026-03-31T00:00:00.000Z", + lastStepStarted: "policies", + lastCompletedStep: "policies", + failure: null, + sandboxName: "alpha", + provider: "nvidia-prod", + model: "nvidia/nemotron-3-super-120b-a12b", + endpointUrl: null, + credentialEnv: null, + preferredInferenceApi: null, + nimContainer: null, + policyPresets: null, + metadata: { gatewayName: "nemoclaw" }, + steps: { + preflight: { status: "complete", startedAt: null, completedAt: null, error: null }, + gateway: { status: "complete", startedAt: null, completedAt: null, error: null }, + sandbox: { status: "complete", startedAt: null, completedAt: null, error: null }, + provider_selection: { + status: "complete", + startedAt: null, + completedAt: null, + error: null, + }, + inference: { status: "complete", startedAt: null, completedAt: null, error: null }, + openclaw: { status: "complete", startedAt: null, completedAt: null, error: null }, + policies: { status: "complete", startedAt: null, completedAt: null, error: null }, + }, + }, + null, + 2, + ), + { mode: 0o600 }, + ); fs.writeFileSync( path.join(localBin, "openshell"), [ "#!/usr/bin/env bash", `marker_file=${JSON.stringify(markerFile)}`, 'printf \'%s\\n\' "$*" >> "$marker_file"', + 'if [ "$1" = "status" ]; then', + " echo 'Server Status'", + " echo", + " echo ' Gateway: nemoclaw'", + " echo ' Status: Connected'", + " exit 0", + "fi", + 'if [ "$1" = "gateway" ] && [ "$2" = "info" ]; then', + " echo 'Gateway Info'", + " echo", + " echo ' Gateway: nemoclaw'", + " exit 0", + "fi", + 'if [ "$1" = "sandbox" ] && [ "$2" = "list" ]; then', + " echo 'NAME STATUS AGE'", + " echo 'alpha Ready 2m ago'", + " exit 0", + "fi", 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', " echo 'Sandbox:'", " echo", @@ -449,902 +447,36 @@ describe("CLI dispatch", () => { " echo ' Phase: Ready'", " exit 0", "fi", - 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ] && [ "$3" = "--name" ] && [ "$4" = "alpha" ]; then', - ' cmd="$8"', - ' if [[ "$cmd" == *"curl -so"* ]]; then echo "__NEMOCLAW_SANDBOX_EXEC_STARTED__"; echo RUNNING; exit 0; fi', + 'if [ "$1" = "sandbox" ] && [ "$2" = "connect" ] && [ "$3" = "alpha" ]; then', + " exit 0", + "fi", + 'if [ "$1" = "--version" ]; then', + " echo 'openshell 0.0.16'", + " exit 0", "fi", - 'if [ "$1" = "forward" ] && [ "$2" = "list" ]; then echo "alpha 127.0.0.1 18789 12345 running"; exit 0; fi', - 'if [ "$1" = "forward" ]; then exit 99; fi', "exit 0", ].join("\n"), { mode: 0o755 }, ); - writeRecordingCommand(localBin, "ssh", sshMarkerFile, 98); - const stopForwardListeners = await startForwardListeners([18789]); - - try { - const r = runWithEnv("alpha --probe-only", { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }); - expect(r.code).toBe(0); - expect(r.out).toContain("Probe complete: OpenClaw gateway is running"); - const calls = fs.readFileSync(markerFile, "utf8").trim().split("\n").filter(Boolean); - expect(calls).toContain("sandbox get alpha"); - expect(calls.some((call) => call.startsWith("sandbox exec --name alpha -- sh -c"))).toBe( - true, - ); - expect(calls).not.toContain("sandbox ssh-config alpha"); - expect(calls).not.toContain("sandbox connect alpha"); - expect(fs.existsSync(sshMarkerFile)).toBe(false); - } finally { - await stopForwardListeners(); - } - }); + const result = runWithEnv("alpha connect", { + HOME: home, + PATH: `${localBin}:${process.env.PATH || ""}`, + }); - it("connect --probe-only does not retry failed privileged recovery over SSH", () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-connect-probe-no-ssh-")); - const localBin = path.join(home, "bin"); - const markerFile = path.join(home, "openshell-calls"); - const dockerCalls = path.join(home, "docker-calls"); - const sshMarkerFile = path.join(home, "ssh-calls"); - const stateFile = path.join(home, "probe-state"); - fs.mkdirSync(localBin, { recursive: true }); - writeSandboxRegistry(home); - fs.writeFileSync(stateFile, "stopped"); - fs.writeFileSync( - path.join(localBin, "openshell"), - [ - "#!/usr/bin/env bash", - `marker_file=${JSON.stringify(markerFile)}`, - 'printf \'%s\\n\' "$*" >> "$marker_file"', - 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', - " echo 'Sandbox:'", - " echo", - " echo ' Id: abc'", - " echo ' Name: alpha'", - " echo ' Namespace: openshell'", - " echo ' Phase: Ready'", - " exit 0", - "fi", - 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ] && [ "$3" = "--name" ] && [ "$4" = "alpha" ]; then', - ' cmd="$8"', - ' if [[ "$cmd" == *"curl -so"* ]]; then echo "__NEMOCLAW_SANDBOX_EXEC_STARTED__"; echo STOPPED; exit 0; fi', - "fi", - 'if [ "$1" = "sandbox" ] && [ "$2" = "ssh-config" ]; then', - " echo 'Host openshell-alpha'", - " exit 0", - "fi", - "exit 0", - ].join("\n"), - { mode: 0o755 }, - ); - writeGatewayControlDockerStub(localBin, { - callsFile: dockerCalls, - stateFile, - recoveryStatus: 42, - }); - writeRecordingCommand(localBin, "ssh", sshMarkerFile, 98); - - const r = runWithEnv("alpha connect --probe-only", { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }); - - expect(r.code).toBe(1); - const calls = fs.readFileSync(markerFile, "utf8").trim().split("\n").filter(Boolean); + expect(result.code).toBe(0); + const calls = fs.readFileSync(markerFile, "utf8"); + expect(calls).toContain("sandbox list"); expect(calls).toContain("sandbox get alpha"); - expect(calls.some((call) => call.startsWith("sandbox exec --name alpha -- sh -c"))).toBe(true); - expect(calls).not.toContain("sandbox ssh-config alpha"); - expect(fs.existsSync(sshMarkerFile)).toBe(false); - expectGatewayControlRecovery(dockerCalls); - }); - - it( - "connect --probe-only does not fall back to SSH when sandbox exec never starts", - testTimeoutOptions(15_000), - () => { - const home = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-cli-connect-probe-exec-fallback-"), - ); - const localBin = path.join(home, "bin"); - const openshellCalls = path.join(home, "openshell-calls"); - const dockerCalls = path.join(home, "docker-calls"); - const sshCalls = path.join(home, "ssh-calls"); - const stateFile = path.join(home, "probe-state"); - fs.mkdirSync(localBin, { recursive: true }); - writeSandboxRegistry(home); - fs.writeFileSync(stateFile, "stopped"); - fs.writeFileSync( - path.join(localBin, "openshell"), - [ - "#!/usr/bin/env bash", - `calls=${JSON.stringify(openshellCalls)}`, - 'printf \'%s\\n\' "$*" >> "$calls"', - 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', - " echo 'Sandbox:'", - " echo", - " echo ' Id: abc'", - " echo ' Name: alpha'", - " echo ' Namespace: openshell'", - " echo ' Phase: Ready'", - " exit 0", - "fi", - 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ]; then', - " echo 'error: sandbox exec transport failed before command start' >&2", - " exit 2", - "fi", - 'if [ "$1" = "forward" ] && [ "$2" = "list" ]; then echo "alpha 127.0.0.1 18789 12345 running"; exit 0; fi', - 'if [ "$1" = "forward" ]; then exit 99; fi', - "exit 0", - ].join("\n"), - { mode: 0o755 }, - ); - writeGatewayControlDockerStub(localBin, { - callsFile: dockerCalls, - stateFile, - recoveryStatus: 42, - }); - writeRecordingCommand(localBin, "ssh", sshCalls, 98); - - const r = runWithEnv("alpha connect --probe-only", { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }); - - expect(r.code).toBe(1); - const openshellLog = fs.readFileSync(openshellCalls, "utf8"); - expect(openshellLog).toContain("sandbox exec --name alpha -- sh -c"); - expect(openshellLog).not.toContain("sandbox ssh-config alpha"); - expect(openshellLog).not.toContain("sandbox connect"); - expect(fs.existsSync(sshCalls)).toBe(false); - expectGatewayControlRecovery(dockerCalls); - }, - ); - - it( - "connect --probe-only does not fall back to SSH when sandbox exec times out after starting", - testTimeoutOptions(15_000), - () => { - const home = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-cli-connect-probe-exec-timeout-"), - ); - const localBin = path.join(home, "bin"); - const openshellCalls = path.join(home, "openshell-calls"); - const dockerCalls = path.join(home, "docker-calls"); - const sshCalls = path.join(home, "ssh-calls"); - const stateFile = path.join(home, "probe-state"); - fs.mkdirSync(localBin, { recursive: true }); - writeSandboxRegistry(home); - fs.writeFileSync(stateFile, "stopped"); - fs.writeFileSync( - path.join(localBin, "openshell"), - [ - "#!/usr/bin/env bash", - `calls=${JSON.stringify(openshellCalls)}`, - 'printf \'%s\\n\' "$*" >> "$calls"', - 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', - " echo 'Sandbox:'", - " echo", - " echo ' Id: abc'", - " echo ' Name: alpha'", - " echo ' Namespace: openshell'", - " echo ' Phase: Ready'", - " exit 0", - "fi", - 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ]; then', - " echo '__NEMOCLAW_SANDBOX_EXEC_STARTED__'", - " sleep 1", - " exit 0", - "fi", - 'if [ "$1" = "forward" ] && [ "$2" = "list" ]; then echo "alpha 127.0.0.1 18789 12345 running"; exit 0; fi', - 'if [ "$1" = "forward" ]; then exit 99; fi', - "exit 0", - ].join("\n"), - { mode: 0o755 }, - ); - writeGatewayControlDockerStub(localBin, { - callsFile: dockerCalls, - stateFile, - recoveryStatus: 42, - }); - writeRecordingCommand(localBin, "ssh", sshCalls, 98); - - const r = runWithEnv("alpha connect --probe-only", { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - NEMOCLAW_SANDBOX_EXEC_TIMEOUT_MS: "50", - }); - - expect(r.code).toBe(1); - const openshellLog = fs.readFileSync(openshellCalls, "utf8"); - expect(openshellLog).toContain("sandbox exec --name alpha -- sh -c"); - expect(openshellLog).not.toContain("sandbox ssh-config alpha"); - expect(fs.existsSync(sshCalls)).toBe(false); - expectGatewayControlRecovery(dockerCalls); - }, - ); - - it("recovers stopped Hermes agents through privileged Docker control instead of SSH", async () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-connect-probe-agent-")); - const localBin = path.join(home, "bin"); - const openshellCalls = path.join(home, "openshell-calls"); - const dockerCalls = path.join(home, "docker-calls"); - const sshCalls = path.join(home, "ssh-calls"); - const stateFile = path.join(home, "probe-state"); - fs.mkdirSync(localBin, { recursive: true }); - writeSandboxRegistry(home, { agent: "hermes" }); - fs.writeFileSync(stateFile, "stopped"); - fs.writeFileSync( - path.join(localBin, "openshell"), - [ - "#!/usr/bin/env bash", - `calls=${JSON.stringify(openshellCalls)}`, - `state_file=${JSON.stringify(stateFile)}`, - 'printf \'%s\\n\' "$*" >> "$calls"', - 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', - " echo 'Sandbox:'", - " echo", - " echo ' Id: abc'", - " echo ' Name: alpha'", - " echo ' Namespace: openshell'", - " echo ' Phase: Ready'", - " exit 0", - "fi", - 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ] && [ "$3" = "--name" ] && [ "$4" = "alpha" ]; then', - ' cmd="$8"', - ' if [[ "$cmd" == *"curl -so"* ]]; then', - " echo '__NEMOCLAW_SANDBOX_EXEC_STARTED__'", - ' if [ "$(cat "$state_file")" = recovered ]; then echo RUNNING; else echo STOPPED; fi', - " exit 0", - " fi", - "fi", - 'if [ "$1" = "sandbox" ] && [ "$2" = "ssh-config" ] && [ "$3" = "alpha" ]; then', - ' echo UNEXPECTED_SSH_CONFIG >> "$calls"', - " exit 1", - "fi", - 'if [ "$1" = "forward" ] && [ "$2" = "list" ]; then { echo "alpha 127.0.0.1 18789 12345 running"; echo "alpha 127.0.0.1 8642 12346 running"; }; exit 0; fi', - 'if [ "$1" = "forward" ]; then exit 99; fi', - "exit 0", - ].join("\n"), - { mode: 0o755 }, - ); - writeGatewayControlDockerStub(localBin, { callsFile: dockerCalls, stateFile }); - writeRecordingCommand(localBin, "ssh", sshCalls, 98); - const stopForwardListeners = await startForwardListeners([18789, 8642]); - - try { - const r = runWithEnv("alpha connect --probe-only", { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }); - - expect(r.code).toBe(0); - expect(r.out).toContain("Probe complete: recovered Hermes Agent gateway"); - const openshellLog = fs.readFileSync(openshellCalls, "utf8"); - expect(openshellLog).toContain("sandbox exec --name alpha -- sh -c"); - expect(openshellLog).not.toContain("sandbox ssh-config alpha"); - expect(openshellLog).not.toContain("sandbox connect"); - expect(fs.existsSync(sshCalls)).toBe(false); - expectGatewayControlRecovery(dockerCalls); - } finally { - await stopForwardListeners(); - } - }); - - it("preserves the registry entry when connect targets a missing live sandbox (#4497)", () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-stale-connect-")); - const localBin = path.join(home, "bin"); - const registryDir = path.join(home, ".nemoclaw"); - fs.mkdirSync(localBin, { recursive: true }); - fs.mkdirSync(registryDir, { recursive: true }); - fs.writeFileSync( - path.join(registryDir, "sandboxes.json"), - JSON.stringify({ - sandboxes: { - alpha: { - name: "alpha", - model: "test-model", - provider: "nvidia-prod", - gpuEnabled: false, - policies: [], - }, - }, - defaultSandbox: "alpha", - }), - { mode: 0o600 }, - ); - fs.writeFileSync( - path.join(localBin, "openshell"), - [ - "#!/usr/bin/env bash", - 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', - " echo 'Error: status: NotFound, message: \"sandbox not found\"' >&2", - " exit 1", - "fi", - // Simulate a healthy, active `nemoclaw` named gateway so the - // lifecycle guard confirms healthy_named. Even on this path connect - // must now preserve the entry so a follow-up rebuild can recover it - // (#4497); it previously removed it here (#2276). - 'if [ "$1" = "status" ]; then', - " printf 'Server Status\\n\\n Gateway: nemoclaw\\n Status: Connected\\n'", - " exit 0", - "fi", - 'if [ "$1" = "gateway" ] && [ "$2" = "info" ]; then', - " printf 'Gateway: nemoclaw\\n'", - " exit 0", - "fi", - "exit 0", - ].join("\n"), - { mode: 0o755 }, + expect(calls).toContain("sandbox connect alpha"); + const recoveredRegistry = JSON.parse( + fs.readFileSync(path.join(nemoclawDir, "sandboxes.json"), "utf8"), ); - - const r = runWithEnv("alpha connect", { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }); - - expect(r.code).toBe(1); - expect(r.out.includes("Removed stale local registry entry")).toBe(false); - expect(r.out.includes("registered locally, but is not present")).toBeTruthy(); - expect(r.out.includes("preserved")).toBeTruthy(); - const saved = JSON.parse(fs.readFileSync(path.join(registryDir, "sandboxes.json"), "utf8")); - expect(saved.sandboxes.alpha).toBeDefined(); - }); - - it("recovers a missing registry entry from the last onboard session during list", () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-list-session-recover-")); - const localBin = path.join(home, "bin"); - const nemoclawDir = path.join(home, ".nemoclaw"); - fs.mkdirSync(localBin, { recursive: true }); - fs.mkdirSync(nemoclawDir, { recursive: true }); - fs.writeFileSync( - path.join(nemoclawDir, "sandboxes.json"), - JSON.stringify({ - sandboxes: { - gamma: { - name: "gamma", - model: "existing-model", - provider: "existing-provider", - gpuEnabled: false, - policies: ["npm"], - }, - }, - defaultSandbox: "gamma", + expect(recoveredRegistry.sandboxes.alpha).toEqual( + expect.objectContaining({ + model: "nvidia/nemotron-3-super-120b-a12b", + provider: "nvidia-prod", }), - { mode: 0o600 }, ); - fs.writeFileSync( - path.join(nemoclawDir, "onboard-session.json"), - JSON.stringify( - { - version: 1, - sessionId: "session-1", - resumable: true, - status: "complete", - mode: "interactive", - startedAt: "2026-03-31T00:00:00.000Z", - updatedAt: "2026-03-31T00:00:00.000Z", - lastStepStarted: "policies", - lastCompletedStep: "policies", - failure: null, - sandboxName: "alpha", - provider: "nvidia-prod", - model: "nvidia/nemotron-3-super-120b-a12b", - endpointUrl: null, - credentialEnv: null, - preferredInferenceApi: null, - nimContainer: null, - policyPresets: ["pypi"], - metadata: { gatewayName: "nemoclaw" }, - steps: { - preflight: { status: "complete", startedAt: null, completedAt: null, error: null }, - gateway: { status: "complete", startedAt: null, completedAt: null, error: null }, - sandbox: { status: "complete", startedAt: null, completedAt: null, error: null }, - provider_selection: { - status: "complete", - startedAt: null, - completedAt: null, - error: null, - }, - inference: { status: "complete", startedAt: null, completedAt: null, error: null }, - openclaw: { status: "complete", startedAt: null, completedAt: null, error: null }, - policies: { status: "complete", startedAt: null, completedAt: null, error: null }, - }, - }, - null, - 2, - ), - { mode: 0o600 }, - ); - fs.writeFileSync( - path.join(localBin, "openshell"), - [ - "#!/usr/bin/env bash", - 'if [ "$1" = "status" ]; then', - " echo 'Server Status'", - " echo", - " echo ' Gateway: nemoclaw'", - " echo ' Status: Connected'", - " exit 0", - "fi", - 'if [ "$1" = "gateway" ] && [ "$2" = "info" ]; then', - " echo 'Gateway Info'", - " echo", - " echo ' Gateway: nemoclaw'", - " exit 0", - "fi", - 'if [ "$1" = "sandbox" ] && [ "$2" = "list" ]; then', - " echo 'NAME STATUS AGE'", - " echo 'alpha Ready 2m ago'", - " exit 0", - "fi", - 'if [ "$1" = "inference" ] && [ "$2" = "get" ]; then', - " exit 0", - "fi", - 'if [ "$1" = "--version" ]; then', - " echo 'openshell 0.0.16'", - " exit 0", - "fi", - "exit 0", - ].join("\n"), - { mode: 0o755 }, - ); - - const r = runWithEnv("list", { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }); - - expect(r.code).toBe(0); - expect( - r.out.includes("Recovered sandbox inventory from the last onboard session."), - ).toBeTruthy(); - expect(r.out.includes("alpha")).toBeTruthy(); - expect(r.out.includes("gamma")).toBeTruthy(); - const saved = JSON.parse(fs.readFileSync(path.join(nemoclawDir, "sandboxes.json"), "utf8")); - expect(saved.sandboxes.alpha).toBeTruthy(); - expect(saved.sandboxes.alpha.policies).toEqual(["pypi"]); - expect(saved.sandboxes.gamma).toBeTruthy(); - expect(saved.defaultSandbox).toBe("gamma"); - }); - - it("imports additional live sandboxes into the registry during list recovery", () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-list-live-recover-")); - const localBin = path.join(home, "bin"); - const nemoclawDir = path.join(home, ".nemoclaw"); - fs.mkdirSync(localBin, { recursive: true }); - fs.mkdirSync(nemoclawDir, { recursive: true }); - fs.writeFileSync( - path.join(nemoclawDir, "sandboxes.json"), - JSON.stringify({ - sandboxes: { - gamma: { - name: "gamma", - model: "existing-model", - provider: "existing-provider", - gpuEnabled: false, - policies: ["npm"], - }, - }, - defaultSandbox: "gamma", - }), - { mode: 0o600 }, - ); - fs.writeFileSync( - path.join(nemoclawDir, "onboard-session.json"), - JSON.stringify( - { - version: 1, - sessionId: "session-1", - resumable: true, - status: "complete", - mode: "interactive", - startedAt: "2026-03-31T00:00:00.000Z", - updatedAt: "2026-03-31T00:00:00.000Z", - lastStepStarted: "policies", - lastCompletedStep: "policies", - failure: null, - sandboxName: "alpha", - provider: "nvidia-prod", - model: "nvidia/nemotron-3-super-120b-a12b", - endpointUrl: null, - credentialEnv: null, - preferredInferenceApi: null, - nimContainer: null, - policyPresets: ["pypi"], - metadata: { gatewayName: "nemoclaw" }, - steps: { - preflight: { status: "complete", startedAt: null, completedAt: null, error: null }, - gateway: { status: "complete", startedAt: null, completedAt: null, error: null }, - sandbox: { status: "complete", startedAt: null, completedAt: null, error: null }, - provider_selection: { - status: "complete", - startedAt: null, - completedAt: null, - error: null, - }, - inference: { status: "complete", startedAt: null, completedAt: null, error: null }, - openclaw: { status: "complete", startedAt: null, completedAt: null, error: null }, - policies: { status: "complete", startedAt: null, completedAt: null, error: null }, - }, - }, - null, - 2, - ), - { mode: 0o600 }, - ); - fs.writeFileSync( - path.join(localBin, "openshell"), - [ - "#!/usr/bin/env bash", - 'if [ "$1" = "status" ]; then', - " echo 'Server Status'", - " echo", - " echo ' Gateway: nemoclaw'", - " echo ' Status: Connected'", - " exit 0", - "fi", - 'if [ "$1" = "gateway" ] && [ "$2" = "info" ]; then', - " echo 'Gateway Info'", - " echo", - " echo ' Gateway: nemoclaw'", - " exit 0", - "fi", - 'if [ "$1" = "sandbox" ] && [ "$2" = "list" ]; then', - " echo 'NAME PHASE'", - " echo 'alpha Ready'", - " echo 'beta Ready'", - " exit 0", - "fi", - 'if [ "$1" = "inference" ] && [ "$2" = "get" ]; then', - " exit 0", - "fi", - 'if [ "$1" = "--version" ]; then', - " echo 'openshell 0.0.16'", - " exit 0", - "fi", - "exit 0", - ].join("\n"), - { mode: 0o755 }, - ); - - const r = runWithEnv("list", { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }); - - expect(r.code).toBe(0); - expect( - r.out.includes("Recovered sandbox inventory from the last onboard session."), - ).toBeTruthy(); - expect( - r.out.includes("Recovered 1 sandbox entry from the live OpenShell gateway."), - ).toBeTruthy(); - expect(r.out.includes("alpha")).toBeTruthy(); - expect(r.out.includes("beta")).toBeTruthy(); - expect(r.out.includes("gamma")).toBeTruthy(); - const saved = JSON.parse(fs.readFileSync(path.join(nemoclawDir, "sandboxes.json"), "utf8")); - expect(saved.sandboxes.alpha).toBeTruthy(); - expect(saved.sandboxes.alpha.policies).toEqual(["pypi"]); - expect(saved.sandboxes.beta).toBeTruthy(); - expect(saved.sandboxes.gamma).toBeTruthy(); - expect(saved.defaultSandbox).toBe("gamma"); - }); - - it("skips invalid recovered sandbox names during list recovery", () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-list-invalid-recover-")); - const localBin = path.join(home, "bin"); - const nemoclawDir = path.join(home, ".nemoclaw"); - fs.mkdirSync(localBin, { recursive: true }); - fs.mkdirSync(nemoclawDir, { recursive: true }); - fs.writeFileSync( - path.join(nemoclawDir, "sandboxes.json"), - JSON.stringify({ - sandboxes: { - gamma: { - name: "gamma", - model: "existing-model", - provider: "existing-provider", - gpuEnabled: false, - policies: ["npm"], - }, - }, - defaultSandbox: "gamma", - }), - { mode: 0o600 }, - ); - fs.writeFileSync( - path.join(nemoclawDir, "onboard-session.json"), - JSON.stringify( - { - version: 1, - sessionId: "session-1", - resumable: true, - status: "complete", - mode: "interactive", - startedAt: "2026-03-31T00:00:00.000Z", - updatedAt: "2026-03-31T00:00:00.000Z", - lastStepStarted: "policies", - lastCompletedStep: "policies", - failure: null, - sandboxName: "Alpha", - provider: "nvidia-prod", - model: "nvidia/nemotron-3-super-120b-a12b", - endpointUrl: null, - credentialEnv: null, - preferredInferenceApi: null, - nimContainer: null, - policyPresets: ["pypi"], - metadata: { gatewayName: "nemoclaw" }, - steps: { - preflight: { status: "complete", startedAt: null, completedAt: null, error: null }, - gateway: { status: "complete", startedAt: null, completedAt: null, error: null }, - sandbox: { status: "complete", startedAt: null, completedAt: null, error: null }, - provider_selection: { - status: "complete", - startedAt: null, - completedAt: null, - error: null, - }, - inference: { status: "complete", startedAt: null, completedAt: null, error: null }, - openclaw: { status: "complete", startedAt: null, completedAt: null, error: null }, - policies: { status: "complete", startedAt: null, completedAt: null, error: null }, - }, - }, - null, - 2, - ), - { mode: 0o600 }, - ); - fs.writeFileSync( - path.join(localBin, "openshell"), - [ - "#!/usr/bin/env bash", - 'if [ "$1" = "status" ]; then', - " echo 'Server Status'", - " echo", - " echo ' Gateway: nemoclaw'", - " echo ' Status: Connected'", - " exit 0", - "fi", - 'if [ "$1" = "gateway" ] && [ "$2" = "info" ]; then', - " echo 'Gateway Info'", - " echo", - " echo ' Gateway: nemoclaw'", - " exit 0", - "fi", - 'if [ "$1" = "sandbox" ] && [ "$2" = "list" ]; then', - " echo 'NAME PHASE'", - " echo 'alpha Ready'", - " echo 'Bad_Name Ready'", - " exit 0", - "fi", - 'if [ "$1" = "inference" ] && [ "$2" = "get" ]; then', - " exit 0", - "fi", - 'if [ "$1" = "--version" ]; then', - " echo 'openshell 0.0.16'", - " exit 0", - "fi", - "exit 0", - ].join("\n"), - { mode: 0o755 }, - ); - - const r = runWithEnv("list", { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }); - - expect(r.code).toBe(0); - expect(r.out.includes("alpha")).toBeTruthy(); - expect(r.out.includes("Bad_Name")).toBeFalsy(); - const saved = JSON.parse(fs.readFileSync(path.join(nemoclawDir, "sandboxes.json"), "utf8")); - expect(saved.sandboxes.alpha).toBeTruthy(); - expect(saved.sandboxes.Bad_Name).toBeUndefined(); - expect(saved.sandboxes.Alpha).toBeUndefined(); - expect(saved.sandboxes.gamma).toBeTruthy(); - }); - - it("connect recovers a named sandbox from the last onboard session when the registry is empty", () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-connect-recover-session-")); - const localBin = path.join(home, "bin"); - const nemoclawDir = path.join(home, ".nemoclaw"); - const markerFile = path.join(home, "connect-args"); - fs.mkdirSync(localBin, { recursive: true }); - fs.mkdirSync(nemoclawDir, { recursive: true }); - fs.writeFileSync( - path.join(nemoclawDir, "onboard-session.json"), - JSON.stringify( - { - version: 1, - sessionId: "session-1", - resumable: true, - status: "complete", - mode: "interactive", - startedAt: "2026-03-31T00:00:00.000Z", - updatedAt: "2026-03-31T00:00:00.000Z", - lastStepStarted: "policies", - lastCompletedStep: "policies", - failure: null, - sandboxName: "alpha", - provider: "nvidia-prod", - model: "nvidia/nemotron-3-super-120b-a12b", - endpointUrl: null, - credentialEnv: null, - preferredInferenceApi: null, - nimContainer: null, - policyPresets: null, - metadata: { gatewayName: "nemoclaw" }, - steps: { - preflight: { status: "complete", startedAt: null, completedAt: null, error: null }, - gateway: { status: "complete", startedAt: null, completedAt: null, error: null }, - sandbox: { status: "complete", startedAt: null, completedAt: null, error: null }, - provider_selection: { - status: "complete", - startedAt: null, - completedAt: null, - error: null, - }, - inference: { status: "complete", startedAt: null, completedAt: null, error: null }, - openclaw: { status: "complete", startedAt: null, completedAt: null, error: null }, - policies: { status: "complete", startedAt: null, completedAt: null, error: null }, - }, - }, - null, - 2, - ), - { mode: 0o600 }, - ); - fs.writeFileSync( - path.join(localBin, "openshell"), - [ - "#!/usr/bin/env bash", - `marker_file=${JSON.stringify(markerFile)}`, - 'printf \'%s\\n\' "$*" >> "$marker_file"', - 'if [ "$1" = "status" ]; then', - " echo 'Server Status'", - " echo", - " echo ' Gateway: nemoclaw'", - " echo ' Status: Connected'", - " exit 0", - "fi", - 'if [ "$1" = "gateway" ] && [ "$2" = "info" ]; then', - " echo 'Gateway Info'", - " echo", - " echo ' Gateway: nemoclaw'", - " exit 0", - "fi", - 'if [ "$1" = "sandbox" ] && [ "$2" = "list" ]; then', - " echo 'NAME STATUS AGE'", - " echo 'alpha Ready 2m ago'", - " exit 0", - "fi", - 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', - " echo 'Sandbox:'", - " echo", - " echo ' Id: abc'", - " echo ' Name: alpha'", - " echo ' Namespace: openshell'", - " echo ' Phase: Ready'", - " exit 0", - "fi", - 'if [ "$1" = "sandbox" ] && [ "$2" = "connect" ] && [ "$3" = "alpha" ]; then', - " exit 0", - "fi", - 'if [ "$1" = "--version" ]; then', - " echo 'openshell 0.0.16'", - " exit 0", - "fi", - "exit 0", - ].join("\n"), - { mode: 0o755 }, - ); - - const r = runWithEnv("alpha connect", { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }); - - expect(r.code).toBe(0); - const log = fs.readFileSync(markerFile, "utf8"); - expect(log.includes("sandbox list")).toBeTruthy(); - expect(log.includes("sandbox get alpha")).toBeTruthy(); - expect(log.includes("sandbox connect alpha")).toBeTruthy(); - }); - - it("connect surfaces sandbox-not-found when recovery cannot find the requested sandbox (#2164)", () => { - const home = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-cli-connect-unknown-after-recovery-"), - ); - const localBin = path.join(home, "bin"); - const nemoclawDir = path.join(home, ".nemoclaw"); - fs.mkdirSync(localBin, { recursive: true }); - fs.mkdirSync(nemoclawDir, { recursive: true }); - fs.writeFileSync( - path.join(nemoclawDir, "onboard-session.json"), - JSON.stringify( - { - version: 1, - sessionId: "session-1", - resumable: true, - status: "complete", - mode: "interactive", - startedAt: "2026-03-31T00:00:00.000Z", - updatedAt: "2026-03-31T00:00:00.000Z", - lastStepStarted: "policies", - lastCompletedStep: "policies", - failure: null, - sandboxName: "alpha", - provider: "nvidia-prod", - model: "nvidia/nemotron-3-super-120b-a12b", - endpointUrl: null, - credentialEnv: null, - preferredInferenceApi: null, - nimContainer: null, - policyPresets: null, - metadata: { gatewayName: "nemoclaw" }, - steps: { - preflight: { status: "complete", startedAt: null, completedAt: null, error: null }, - gateway: { status: "complete", startedAt: null, completedAt: null, error: null }, - sandbox: { status: "complete", startedAt: null, completedAt: null, error: null }, - provider_selection: { - status: "complete", - startedAt: null, - completedAt: null, - error: null, - }, - inference: { status: "complete", startedAt: null, completedAt: null, error: null }, - openclaw: { status: "complete", startedAt: null, completedAt: null, error: null }, - policies: { status: "complete", startedAt: null, completedAt: null, error: null }, - }, - }, - null, - 2, - ), - { mode: 0o600 }, - ); - fs.writeFileSync( - path.join(localBin, "openshell"), - [ - "#!/usr/bin/env bash", - 'if [ "$1" = "status" ]; then', - " echo 'Server Status'", - " echo", - " echo ' Gateway: nemoclaw'", - " echo ' Status: Connected'", - " exit 0", - "fi", - 'if [ "$1" = "gateway" ] && [ "$2" = "info" ]; then', - " echo 'Gateway Info'", - " echo", - " echo ' Gateway: nemoclaw'", - " exit 0", - "fi", - 'if [ "$1" = "sandbox" ] && [ "$2" = "list" ]; then', - " echo 'No sandboxes found.'", - " exit 0", - "fi", - 'if [ "$1" = "--version" ]; then', - " echo 'openshell 0.0.16'", - " exit 0", - "fi", - "exit 0", - ].join("\n"), - { mode: 0o755 }, - ); - - const r = runWithEnv("beta connect", { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }); - - expect(r.code).toBe(1); - expect(r.out.includes("Sandbox 'beta' does not exist")).toBeTruthy(); - // Recovery from onboard-session.json restores "alpha" into the local registry, - // so the helper lists it rather than the empty-registry onboard hint. - expect(r.out.includes("Registered sandboxes: alpha")).toBeTruthy(); }); }); diff --git a/test/cli/status-gateway-lifecycle.test.ts b/test/cli/status-gateway-lifecycle.test.ts index f469f2e97a0..47458cf5137 100644 --- a/test/cli/status-gateway-lifecycle.test.ts +++ b/test/cli/status-gateway-lifecycle.test.ts @@ -1,76 +1,14 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, it, expect } from "vitest"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { describe, expect, it } from "vitest"; -import { - OPENCLAW_EXPECTED_VERSION, - execTimeout, - runWithEnv, - testTimeout, - testTimeoutOptions, - writeSandboxRegistry, -} from "./helpers"; - -describe("CLI dispatch", () => { - it( - "keeps registry entries when status hits a gateway-level transport error", - () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-gateway-error-")); - const localBin = path.join(home, "bin"); - const registryDir = path.join(home, ".nemoclaw"); - fs.mkdirSync(localBin, { recursive: true }); - fs.mkdirSync(registryDir, { recursive: true }); - fs.writeFileSync( - path.join(registryDir, "sandboxes.json"), - JSON.stringify({ - sandboxes: { - alpha: { - name: "alpha", - model: "test-model", - provider: "nvidia-prod", - gpuEnabled: false, - policies: [], - }, - }, - defaultSandbox: "alpha", - }), - { mode: 0o600 }, - ); - fs.writeFileSync( - path.join(localBin, "openshell"), - [ - "#!/usr/bin/env bash", - 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', - " echo 'Error: transport error: handshake verification failed' >&2", - " exit 1", - "fi", - "exit 0", - ].join("\n"), - { mode: 0o755 }, - ); - - const r = runWithEnv( - "alpha status", - { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }, - execTimeout(20_000), - ); - - expect(r.code).toBe(1); - expect(r.out.includes("Could not verify sandbox 'alpha'")).toBeTruthy(); - expect(r.out.includes("gateway identity drift after restart")).toBeTruthy(); - const saved = JSON.parse(fs.readFileSync(path.join(registryDir, "sandboxes.json"), "utf8")); - expect(saved.sandboxes.alpha).toBeTruthy(); - }, - testTimeout(20_000), - ); +import { execTimeout, runWithEnv, testTimeout, writeSandboxRegistry } from "./helpers"; +describe("CLI status gateway lifecycle process contracts", () => { it( "keeps status bounded when a live sandbox probe leaves child pipes open", () => { @@ -109,7 +47,7 @@ describe("CLI dispatch", () => { ); const started = Date.now(); - const r = runWithEnv( + const result = runWithEnv( "alpha status", { HOME: home, @@ -120,209 +58,13 @@ describe("CLI dispatch", () => { ); expect(Date.now() - started).toBeLessThan(execTimeout(12_000)); - expect(r.code).toBe(1); - expect(r.out).toContain("Model: test-model"); - expect(r.out).toContain("Live sandbox status probe timed out"); + expect(result.code).toBe(1); + expect(result.out).toContain("Model: test-model"); + expect(result.out).toContain("Live sandbox status probe timed out"); }, testTimeout(20_000), ); - it("recovers status after gateway runtime is reattached", () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-recover-status-")); - const localBin = path.join(home, "bin"); - const registryDir = path.join(home, ".nemoclaw"); - const stateFile = path.join(home, "sandbox-get-count"); - fs.mkdirSync(localBin, { recursive: true }); - fs.mkdirSync(registryDir, { recursive: true }); - fs.writeFileSync( - path.join(registryDir, "sandboxes.json"), - JSON.stringify({ - sandboxes: { - alpha: { - name: "alpha", - model: "test-model", - provider: "nvidia-prod", - gpuEnabled: false, - policies: [], - }, - }, - defaultSandbox: "alpha", - }), - { mode: 0o600 }, - ); - fs.writeFileSync( - path.join(localBin, "openshell"), - [ - "#!/usr/bin/env bash", - `state_file=${JSON.stringify(stateFile)}`, - 'count=$(cat "$state_file" 2>/dev/null || echo 0)', - 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', - " count=$((count + 1))", - ' echo "$count" > "$state_file"', - ' if [ "$count" -eq 1 ]; then', - " echo 'Error: transport error: Connection refused' >&2", - " exit 1", - " fi", - " echo 'Sandbox: alpha'", - " exit 0", - "fi", - 'if [ "$1" = "status" ]; then', - " echo 'Server Status'", - " echo", - " echo ' Gateway: nemoclaw'", - " echo ' Status: Connected'", - " exit 0", - "fi", - 'if [ "$1" = "gateway" ] && [ "$2" = "info" ] && [ "$3" = "-g" ] && [ "$4" = "nemoclaw" ]; then', - " echo 'Gateway Info'", - " echo", - " echo ' Gateway: nemoclaw'", - " exit 0", - "fi", - "exit 0", - ].join("\n"), - { mode: 0o755 }, - ); - - const r = runWithEnv("alpha status", { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }); - - expect(r.code).toBe(0); - expect(r.out.includes("Recovered NemoClaw gateway runtime")).toBeTruthy(); - expect(r.out.includes("Sandbox: alpha")).toBeTruthy(); - }); - - it("shows a clear local inference warning when Ollama is down", () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-local-inference-down-")); - const localBin = path.join(home, "bin"); - const registryDir = path.join(home, ".nemoclaw"); - fs.mkdirSync(localBin, { recursive: true }); - fs.mkdirSync(registryDir, { recursive: true }); - fs.writeFileSync( - path.join(registryDir, "sandboxes.json"), - JSON.stringify({ - sandboxes: { - alpha: { - name: "alpha", - model: "llama3.2:1b", - provider: "ollama-local", - gpuEnabled: false, - policies: [], - }, - }, - defaultSandbox: "alpha", - }), - { mode: 0o600 }, - ); - fs.writeFileSync( - path.join(localBin, "openshell"), - [ - "#!/usr/bin/env bash", - 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', - " echo 'Sandbox: alpha'", - " exit 0", - "fi", - 'if [ "$1" = "sandbox" ] && [ "$2" = "ssh-config" ] && [ "$3" = "alpha" ]; then', - " exit 1", - "fi", - 'if [ "$1" = "inference" ] && [ "$2" = "get" ]; then', - " echo 'Gateway inference:'", - " echo", - " echo ' Provider: ollama-local'", - " echo ' Model: llama3.2:1b'", - " exit 0", - "fi", - "exit 0", - ].join("\n"), - { mode: 0o755 }, - ); - fs.writeFileSync( - path.join(localBin, "curl"), - [ - "#!/usr/bin/env bash", - 'out=""', - 'url=""', - 'while [ "$#" -gt 0 ]; do', - ' case "$1" in', - ' -o) out="$2"; shift 2 ;;', - " -w|--connect-timeout|--max-time) shift 2 ;;", - " -s|-S|-sS|-f) shift ;;", - ' http://*|https://*) url="$1"; shift ;;', - " *) shift ;;", - " esac", - "done", - 'if [ -n "$out" ]; then : > "$out"; fi', - 'if echo "$url" | grep -q "11434/api/tags"; then', - ' printf "000"', - " exit 7", - "fi", - 'printf "000"', - "exit 7", - ].join("\n"), - { mode: 0o755 }, - ); - - const r = runWithEnv("alpha status", { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }); - - expect(r.code).toBe(0); - // #3265: backend label is qualified `Inference (ollama backend):` so the - // upcoming auth-proxy subprobe line renders in parallel. - expect(r.out).toContain("Inference (ollama backend):"); - expect(r.out).toContain("unreachable"); - expect(r.out).toContain("Start Ollama and retry"); - expect(r.out).toContain("http://127.0.0.1:11434/api/tags"); - }); - - it( - "status reports fresh shields state as not configured instead of down", - testTimeoutOptions(30_000), - () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-status-shields-default-")); - const localBin = path.join(home, "bin"); - fs.mkdirSync(localBin, { recursive: true }); - writeSandboxRegistry(home); - fs.writeFileSync( - path.join(localBin, "openshell"), - [ - "#!/usr/bin/env bash", - 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', - " echo 'Error: sandbox not found' >&2", - " exit 1", - "fi", - 'if [ "$1" = "status" ]; then', - " echo 'Server Status'", - " echo", - " echo ' Gateway: nemoclaw'", - " echo ' Status: Connected'", - " exit 0", - "fi", - 'if [ "$1" = "gateway" ] && [ "$2" = "info" ] && [ "$3" = "-g" ] && [ "$4" = "nemoclaw" ]; then', - " echo 'Gateway Info'", - " echo", - " echo ' Gateway: nemoclaw'", - " exit 0", - "fi", - "exit 0", - ].join("\n"), - { mode: 0o755 }, - ); - - const r = runWithEnv("alpha status 2>&1", { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }); - - expect(r.code).toBe(1); - expect(r.out).toContain("Permissions: not configured (default mutable state)"); - expect(r.out).not.toContain("Permissions: shields down"); - }, - ); - it("prints healthy inference only after the sandbox and gateway are verified", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-status-healthy-")); const localBin = path.join(home, "bin"); @@ -383,609 +125,22 @@ describe("CLI dispatch", () => { { mode: 0o755 }, ); - const r = runWithEnv("alpha status", { + const result = runWithEnv("alpha status", { HOME: home, PATH: `${localBin}:${process.env.PATH || ""}`, }); - expect(r.code).toBe(0); - expect(r.out).toContain("Sandbox: alpha"); - expect(r.out).toContain("Model: live-model"); - expect(r.out).toContain("Provider: nvidia-prod"); - expect(r.out).toContain("Inference:"); - expect(r.out).toContain("healthy"); - expect(r.out).not.toContain("not verified"); + expect(result.code).toBe(0); + expect(result.out).toContain("Sandbox: alpha"); + expect(result.out).toContain("Model: live-model"); + expect(result.out).toContain("Provider: nvidia-prod"); + expect(result.out).toContain("Inference:"); + expect(result.out).toContain("healthy"); + expect(result.out).not.toContain("not verified"); const calls = fs.readFileSync(markerFile, "utf8").trim().split("\n").filter(Boolean); - const sandboxGetIdx = calls.indexOf("sandbox get alpha"); - const inferenceGetIdx = calls.indexOf("inference get"); - expect(sandboxGetIdx).toBeGreaterThanOrEqual(0); - expect(inferenceGetIdx).toBeGreaterThan(sandboxGetIdx); - }); - - it("status reports the live sandbox agent version instead of cached host metadata", () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-status-agent-drift-")); - const localBin = path.join(home, "bin"); - fs.mkdirSync(localBin, { recursive: true }); - writeSandboxRegistry(home, { - model: "configured-model", - provider: "nvidia-prod", - agentVersion: "2026.5.18", - }); - fs.writeFileSync( - path.join(localBin, "openshell"), - [ - "#!/usr/bin/env bash", - 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', - " echo 'Sandbox:'", - " echo", - " echo ' Id: abc'", - " echo ' Name: alpha'", - " echo ' Namespace: openshell'", - " echo ' Phase: Ready'", - " exit 0", - "fi", - 'if [ "$1" = "sandbox" ] && [ "$2" = "ssh-config" ] && [ "$3" = "alpha" ]; then', - " echo 'Host openshell-alpha'", - " echo ' HostName 127.0.0.1'", - " exit 0", - "fi", - 'if [ "$1" = "inference" ] && [ "$2" = "get" ]; then', - " echo 'Gateway inference:'", - " echo", - " echo ' Provider: nvidia-prod'", - " echo ' Model: live-model'", - " exit 0", - "fi", - 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ] && [ "$3" = "--name" ] && [ "$4" = "alpha" ]; then', - " echo '__NEMOCLAW_SANDBOX_EXEC_STARTED__'", - " echo 'RUNNING'", - " exit 0", - "fi", - "exit 0", - ].join("\n"), - { mode: 0o755 }, - ); - fs.writeFileSync( - path.join(localBin, "ssh"), - ["#!/usr/bin/env bash", "echo 'OpenClaw 2026.3.11 (old)'", "exit 0"].join("\n"), - { mode: 0o755 }, - ); - - const r = runWithEnv("alpha status", { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }); - - expect(r.code).toBe(0); - expect(r.out).toContain("Agent: OpenClaw v2026.3.11"); - expect(r.out).toContain("Update:"); - expect(r.out).toContain(`v${OPENCLAW_EXPECTED_VERSION} available`); - expect(r.out).toContain("Run `nemoclaw alpha rebuild` to upgrade"); - expect(r.out).not.toContain("Agent: OpenClaw v2026.5.18"); - }); - - it( - "does not treat a different connected gateway as a healthy nemoclaw gateway", - () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-mixed-gateway-")); - const localBin = path.join(home, "bin"); - const registryDir = path.join(home, ".nemoclaw"); - fs.mkdirSync(localBin, { recursive: true }); - fs.mkdirSync(registryDir, { recursive: true }); - fs.writeFileSync( - path.join(registryDir, "sandboxes.json"), - JSON.stringify({ - sandboxes: { - alpha: { - name: "alpha", - model: "test-model", - provider: "nvidia-prod", - gpuEnabled: false, - policies: [], - }, - }, - defaultSandbox: "alpha", - }), - { mode: 0o600 }, - ); - fs.writeFileSync( - path.join(localBin, "openshell"), - [ - "#!/usr/bin/env bash", - 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', - " echo 'Error: transport error: Connection refused' >&2", - " exit 1", - "fi", - 'if [ "$1" = "status" ]; then', - " echo 'Server Status'", - " echo", - " echo ' Gateway: openshell'", - " echo ' Status: Connected'", - " exit 0", - "fi", - 'if [ "$1" = "gateway" ] && [ "$2" = "info" ] && [ "$3" = "-g" ] && [ "$4" = "nemoclaw" ]; then', - " echo 'Gateway Info'", - " echo", - " echo ' Gateway: nemoclaw'", - " exit 0", - "fi", - 'if [ "$1" = "gateway" ] && [ "$2" = "select" ] && [ "$3" = "nemoclaw" ]; then', - " exit 0", - "fi", - 'if [ "$1" = "gateway" ] && [ "$2" = "start" ] && [ "$3" = "--name" ] && [ "$4" = "nemoclaw" ]; then', - " exit 0", - "fi", - 'if [ "$1" = "inference" ] && [ "$2" = "get" ]; then', - " exit 0", - "fi", - "exit 0", - ].join("\n"), - { mode: 0o755 }, - ); - - const r = runWithEnv( - "alpha status", - { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }, - execTimeout(), - ); - - expect(r.code).toBe(1); - expect(r.out.includes("Recovered NemoClaw gateway runtime")).toBeFalsy(); - expect(r.out.includes("Could not verify sandbox 'alpha'")).toBeTruthy(); - expect(r.out.includes("verify the active gateway")).toBeTruthy(); - }, - testTimeout(10_000), - ); - - it( - "matches ANSI-decorated gateway transport errors when printing lifecycle hints", - () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-ansi-transport-hint-")); - const localBin = path.join(home, "bin"); - const registryDir = path.join(home, ".nemoclaw"); - fs.mkdirSync(localBin, { recursive: true }); - fs.mkdirSync(registryDir, { recursive: true }); - fs.writeFileSync( - path.join(registryDir, "sandboxes.json"), - JSON.stringify({ - sandboxes: { - alpha: { - name: "alpha", - model: "test-model", - provider: "nvidia-prod", - gpuEnabled: false, - policies: [], - }, - }, - defaultSandbox: "alpha", - }), - { mode: 0o600 }, - ); - fs.writeFileSync( - path.join(localBin, "openshell"), - [ - "#!/usr/bin/env bash", - 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', - " printf '\\033[31mError: trans\\033[0mport error: Connec\\033[33mtion refused\\033[0m\\n' >&2", - " exit 1", - "fi", - 'if [ "$1" = "status" ]; then', - " echo 'Server Status'", - " echo", - " echo ' Gateway: openshell'", - " echo ' Status: Disconnected'", - " exit 0", - "fi", - 'if [ "$1" = "gateway" ] && [ "$2" = "info" ] && [ "$3" = "-g" ] && [ "$4" = "nemoclaw" ]; then', - " printf 'Gateway Info\\n\\n Gateway: openshell\\n'", - " exit 0", - "fi", - 'if [ "$1" = "gateway" ] && [ "$2" = "select" ] && [ "$3" = "nemoclaw" ]; then', - " exit 0", - "fi", - "exit 0", - ].join("\n"), - { mode: 0o755 }, - ); - - const r = runWithEnv( - "alpha status", - { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }, - execTimeout(), - ); - - expect(r.code).toBe(1); - expect(r.out.includes("current gateway/runtime is not reachable")).toBeTruthy(); - }, - testTimeout(10_000), - ); - - it( - "matches ANSI-decorated gateway auth errors when printing lifecycle hints", - () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-ansi-auth-hint-")); - const localBin = path.join(home, "bin"); - const registryDir = path.join(home, ".nemoclaw"); - fs.mkdirSync(localBin, { recursive: true }); - fs.mkdirSync(registryDir, { recursive: true }); - fs.writeFileSync( - path.join(registryDir, "sandboxes.json"), - JSON.stringify({ - sandboxes: { - alpha: { - name: "alpha", - model: "test-model", - provider: "nvidia-prod", - gpuEnabled: false, - policies: [], - }, - }, - defaultSandbox: "alpha", - }), - { mode: 0o600 }, - ); - fs.writeFileSync( - path.join(localBin, "openshell"), - [ - "#!/usr/bin/env bash", - 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', - " printf '\\033[31mMissing gateway auth\\033[0m token\\n' >&2", - " exit 1", - "fi", - 'if [ "$1" = "status" ]; then', - " echo 'Server Status'", - " echo", - " echo ' Gateway: openshell'", - " echo ' Status: Disconnected'", - " exit 0", - "fi", - 'if [ "$1" = "gateway" ] && [ "$2" = "info" ] && [ "$3" = "-g" ] && [ "$4" = "nemoclaw" ]; then', - " printf 'Gateway Info\\n\\n Gateway: openshell\\n'", - " exit 0", - "fi", - 'if [ "$1" = "gateway" ] && [ "$2" = "select" ] && [ "$3" = "nemoclaw" ]; then', - " exit 0", - "fi", - "exit 0", - ].join("\n"), - { mode: 0o755 }, - ); - - const r = runWithEnv( - "alpha status", - { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }, - execTimeout(), - ); - - expect(r.code).toBe(1); - expect( - r.out.includes("Verify the active gateway and retry after re-establishing the runtime."), - ).toBeTruthy(); - }, - testTimeout(10_000), - ); - - it("explains unrecoverable gateway trust rotation after restart", () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-identity-drift-")); - const localBin = path.join(home, "bin"); - const registryDir = path.join(home, ".nemoclaw"); - fs.mkdirSync(localBin, { recursive: true }); - fs.mkdirSync(registryDir, { recursive: true }); - fs.writeFileSync( - path.join(registryDir, "sandboxes.json"), - JSON.stringify({ - sandboxes: { - alpha: { - name: "alpha", - model: "test-model", - provider: "nvidia-prod", - gpuEnabled: false, - policies: [], - }, - }, - defaultSandbox: "alpha", - }), - { mode: 0o600 }, - ); - fs.writeFileSync( - path.join(localBin, "openshell"), - [ - "#!/usr/bin/env bash", - 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', - " echo 'Error: transport error: handshake verification failed' >&2", - " exit 1", - "fi", - 'if [ "$1" = "status" ]; then', - " echo 'Server Status'", - " echo", - " echo ' Gateway: nemoclaw'", - " echo ' Status: Connected'", - " exit 0", - "fi", - 'if [ "$1" = "gateway" ] && [ "$2" = "info" ] && [ "$3" = "-g" ] && [ "$4" = "nemoclaw" ]; then', - " echo 'Gateway Info'", - " echo", - " echo ' Gateway: nemoclaw'", - " exit 0", - "fi", - "exit 0", - ].join("\n"), - { mode: 0o755 }, - ); - - const statusResult = runWithEnv( - "alpha status", - { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }, - execTimeout(), - ); - expect(statusResult.code).toBe(1); - expect(statusResult.out.includes("gateway trust material rotated after restart")).toBeTruthy(); - expect(statusResult.out.includes("cannot be reattached safely")).toBeTruthy(); - - const connectResult = runWithEnv("alpha connect", { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }); - expect(connectResult.code).toBe(1); - // After the auto-recovery attempt (clear stale host keys + retry), the - // fake openshell still returns the handshake error, so recovery fails. - expect(connectResult.out.includes("Could not reconnect")).toBeTruthy(); - expect(connectResult.out.includes("Recreate this sandbox")).toBeTruthy(); - }); - - it("explains when gateway metadata exists but the restarted API is still refusing connections", { - timeout: 30000, - }, () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-gateway-unreachable-")); - const localBin = path.join(home, "bin"); - const registryDir = path.join(home, ".nemoclaw"); - const markerFile = path.join(home, "openshell-calls"); - fs.mkdirSync(localBin, { recursive: true }); - fs.mkdirSync(registryDir, { recursive: true }); - fs.writeFileSync( - path.join(registryDir, "sandboxes.json"), - JSON.stringify({ - sandboxes: { - alpha: { - name: "alpha", - model: "test-model", - provider: "nvidia-prod", - gpuEnabled: false, - policies: [], - }, - }, - defaultSandbox: "alpha", - }), - { mode: 0o600 }, - ); - fs.writeFileSync( - path.join(localBin, "openshell"), - [ - "#!/usr/bin/env bash", - `printf '%s\\n' "$*" >> ${JSON.stringify(markerFile)}`, - 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', - " echo 'Error: transport error: Connection refused' >&2", - " exit 1", - "fi", - 'if [ "$1" = "status" ]; then', - " echo 'Server Status'", - " echo", - " echo ' Gateway: nemoclaw'", - " echo ' Server: https://127.0.0.1:8080'", - " echo 'Error: client error (Connect)' >&2", - " echo 'Connection refused (os error 111)' >&2", - " exit 1", - "fi", - 'if [ "$1" = "gateway" ] && [ "$2" = "info" ] && [ "$3" = "-g" ] && [ "$4" = "nemoclaw" ]; then', - " echo 'Gateway Info'", - " echo", - " echo ' Gateway: nemoclaw'", - " exit 0", - "fi", - 'if [ "$1" = "gateway" ] && [ "$2" = "select" ] && [ "$3" = "nemoclaw" ]; then', - " exit 0", - "fi", - 'if [ "$1" = "gateway" ] && [ "$2" = "start" ] && [ "$3" = "--name" ] && [ "$4" = "nemoclaw" ]; then', - " exit 0", - "fi", - "exit 0", - ].join("\n"), - { mode: 0o755 }, - ); - fs.writeFileSync( - path.join(localBin, "curl"), - [ - "#!/usr/bin/env bash", - 'out=""', - 'while [ "$#" -gt 0 ]; do', - ' case "$1" in', - ' -o) out="$2"; shift 2 ;;', - " -w|--connect-timeout|--max-time) shift 2 ;;", - " *) shift ;;", - " esac", - "done", - 'if [ -n "$out" ]; then printf "{}" > "$out"; fi', - 'printf "200"', - "exit 0", - ].join("\n"), - { mode: 0o755 }, - ); - - const statusResult = runWithEnv( - "alpha status", - { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }, - execTimeout(), - ); - expect(statusResult.code).toBe(1); - expect(statusResult.out).not.toContain("Inference: healthy"); - expect(statusResult.out).toContain( - "Inference: not verified (gateway/sandbox state not verified)", - ); - expect(fs.readFileSync(markerFile, "utf8")).not.toContain("inference get"); - expect( - statusResult.out.includes("gateway is still refusing connections after restart"), - ).toBeTruthy(); - expect( - statusResult.out.includes("Retry `openshell gateway start --name nemoclaw`"), - ).toBeTruthy(); - - const connectResult = runWithEnv("alpha connect", { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }); - expect(connectResult.code).toBe(1); - expect( - connectResult.out.includes("gateway is still refusing connections after restart"), - ).toBeTruthy(); - expect(connectResult.out.includes("If the gateway never becomes healthy")).toBeTruthy(); - }); - - it( - "explains when the named gateway is no longer configured after restart or rebuild", - () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-gateway-missing-")); - const localBin = path.join(home, "bin"); - const registryDir = path.join(home, ".nemoclaw"); - fs.mkdirSync(localBin, { recursive: true }); - fs.mkdirSync(registryDir, { recursive: true }); - fs.writeFileSync( - path.join(registryDir, "sandboxes.json"), - JSON.stringify({ - sandboxes: { - alpha: { - name: "alpha", - model: "test-model", - provider: "nvidia-prod", - gpuEnabled: false, - policies: [], - }, - }, - defaultSandbox: "alpha", - }), - { mode: 0o600 }, - ); - fs.writeFileSync( - path.join(localBin, "openshell"), - [ - "#!/usr/bin/env bash", - 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', - " echo 'Error: transport error: Connection refused' >&2", - " exit 1", - "fi", - 'if [ "$1" = "status" ]; then', - " echo 'Gateway Status'", - " echo", - " echo ' Status: No gateway configured.'", - " exit 0", - "fi", - 'if [ "$1" = "gateway" ] && [ "$2" = "info" ] && [ "$3" = "-g" ] && [ "$4" = "nemoclaw" ]; then', - " exit 1", - "fi", - 'if [ "$1" = "gateway" ] && [ "$2" = "select" ] && [ "$3" = "nemoclaw" ]; then', - " exit 0", - "fi", - 'if [ "$1" = "gateway" ] && [ "$2" = "start" ] && [ "$3" = "--name" ] && [ "$4" = "nemoclaw" ]; then', - " exit 1", - "fi", - "exit 0", - ].join("\n"), - { mode: 0o755 }, - ); - - const statusResult = runWithEnv( - "alpha status", - { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }, - execTimeout(), - ); - expect(statusResult.code).toBe(1); - expect( - statusResult.out.includes("gateway is no longer configured after restart/rebuild"), - ).toBeTruthy(); - expect(statusResult.out.includes("Start the gateway again")).toBeTruthy(); - }, - testTimeout(10_000), - ); - - it("preserves an orphan registry entry on passive status when the named gateway is healthy", () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-status-orphan-")); - const localBin = path.join(home, "bin"); - const registryDir = path.join(home, ".nemoclaw"); - fs.mkdirSync(localBin, { recursive: true }); - fs.mkdirSync(registryDir, { recursive: true }); - fs.writeFileSync( - path.join(registryDir, "sandboxes.json"), - JSON.stringify({ - sandboxes: { - alpha: { - name: "alpha", - model: "test-model", - provider: "nvidia-prod", - gpuEnabled: false, - policies: [], - }, - }, - defaultSandbox: "alpha", - }), - { mode: 0o600 }, - ); - fs.writeFileSync( - path.join(localBin, "openshell"), - [ - "#!/usr/bin/env bash", - 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ] && [ "$3" = "alpha" ]; then', - " echo 'Error: status: NotFound, message: \"sandbox not found\"' >&2", - " exit 1", - "fi", - 'if [ "$1" = "status" ]; then', - " printf 'Server Status\\n\\n Gateway: nemoclaw\\n Status: Connected\\n'", - " exit 0", - "fi", - 'if [ "$1" = "gateway" ] && [ "$2" = "info" ]; then', - " printf 'Gateway: nemoclaw\\n'", - " exit 0", - "fi", - "exit 0", - ].join("\n"), - { mode: 0o755 }, - ); - - const statusResult = runWithEnv( - "alpha status", - { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }, - execTimeout(), - ); - - expect(statusResult.code).toBe(1); - expect(statusResult.out).not.toContain("Inference: healthy"); - expect(statusResult.out).toContain( - "registered locally, but is not present in the live OpenShell gateway", - ); - expect(statusResult.out).toContain("No local registry entry was removed"); - expect(statusResult.out).not.toContain("Removed stale local registry entry"); - - const saved = JSON.parse(fs.readFileSync(path.join(registryDir, "sandboxes.json"), "utf8")); - expect(saved.sandboxes.alpha).toBeDefined(); - expect(saved.defaultSandbox).toBe("alpha"); + const sandboxGetIndex = calls.indexOf("sandbox get alpha"); + const inferenceGetIndex = calls.indexOf("inference get"); + expect(sandboxGetIndex).toBeGreaterThanOrEqual(0); + expect(inferenceGetIndex).toBeGreaterThan(sandboxGetIndex); }); }); diff --git a/test/cli/status-routing.test.ts b/test/cli/status-routing.test.ts index 3f6a064c13a..a88198e0a4a 100644 --- a/test/cli/status-routing.test.ts +++ b/test/cli/status-routing.test.ts @@ -8,118 +8,31 @@ import { describe, expect, it } from "vitest"; import { run, runWithEnv, writeSandboxRegistry } from "./helpers"; -describe("CLI status routing", () => { +describe("CLI status routing process contracts", () => { it("status --help exits 0 and shows status usage", () => { - const r = run("status --help"); - expect(r.code).toBe(0); - expect(r.out).toContain("status [--json]"); - expect(r.out).toContain("Show global sandbox and host service status"); - expect(r.out).toContain("Use ` status` for one sandbox"); - }); - - it("sandbox status --help advertises --json flag", () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-sandbox-status-help-json-")); - writeSandboxRegistry(home); - const r = runWithEnv("sandbox status alpha --help", { HOME: home }); - expect(r.code).toBe(0); - expect(r.out).toContain("--json"); - expect(r.out).toContain("$ nemoclaw sandbox status [--json]"); - expect(r.out).toContain("$ nemoclaw alpha status"); - expect(r.out).toContain("$ nemoclaw sandbox status alpha --json"); - - const alias = runWithEnv("alpha status --help", { HOME: home }); - expect(alias.code).toBe(0); - expect(alias.out).toContain("--json"); - }); - - it("status rejects unknown flags through current dispatch path", () => { - const r = run("status --bogus"); - expect(r.code).toBe(2); - expect(r.out).toContain("Nonexistent flag: --bogus"); - }); - - it("status rejects unexpected positional arguments through current dispatch path", () => { - const r = run("status bogus"); - expect(r.code).toBe(2); - expect(r.out).toContain("'nemoclaw status' shows the global sandbox/service overview"); - expect(r.out).toContain("Run: nemoclaw bogus status"); - }); - - it("status preserves --json in wrong-form sandbox status guidance", () => { - const r = run("status --json alpha"); - expect(r.code).toBe(2); - expect(r.out).toContain("Run: nemoclaw alpha status --json"); - }); - - it("status preserves --json when the flag follows the sandbox name", () => { - const r = run("status bogus --json"); - expect(r.code).toBe(2); - expect(r.out).toContain("Run: nemoclaw bogus status --json"); - }); - - it("status surfaces an unknown flag rather than the scope hint when a name follows it", () => { - const r = run("status --bogus alpha"); - expect(r.code).toBe(2); - expect(r.out).toContain("Nonexistent flag: --bogus"); - expect(r.out).not.toContain("does not take a sandbox name"); - }); - - it("status surfaces an unknown flag rather than the scope hint when it follows a name", () => { - const r = run("status alpha --bogus"); - expect(r.code).toBe(2); - expect(r.out).toContain("Nonexistent flag: --bogus"); - expect(r.out).not.toContain("does not take a sandbox name"); - }); - - it("status leaves multiple unexpected names to the strict parser", () => { - const r = run("status alpha beta"); - expect(r.code).toBe(2); - expect(r.out).toContain("Unexpected arguments: alpha, beta"); - expect(r.out).not.toContain("does not take a sandbox name"); - }); - - it("status preserves help when correcting a sandbox-like argument", () => { - const r = run("status alpha --help"); - expect(r.code).toBe(2); - expect(r.out).toContain("Run: nemoclaw alpha status --help"); - }); + const result = run("status --help"); - it.each([ - "status", - "help", - "sandbox", - "internal", - ])("status does not suggest reserved command token %s as a sandbox name", (token) => { - const r = run(`status ${token}`); - expect(r.code).toBe(2); - expect(r.out).toContain(`Unexpected argument: ${token}`); - expect(r.out).not.toContain("Run:"); - }); - - it.each([ - "status alpha --json --help", - "status alpha --help --json", - ])("status gives help precedence in combined-flag scope guidance for %s", (command) => { - const r = run(command); - expect(r.code).toBe(2); - expect(r.out).toContain("Run: nemoclaw alpha status --help"); - expect(r.out).not.toContain("Run: nemoclaw alpha status --json --help"); - }); - - it("status never emits an unsafe sandbox token in a copy-paste command", () => { - const r = run("status 'alpha;echo pwned'"); - expect(r.code).toBe(2); - expect(r.out).toContain("Unexpected argument: alpha;echo pwned"); - expect(r.out).not.toContain("Run:"); + expect(result.code).toBe(0); + expect(result.out).toContain("status [--json]"); + expect(result.out).toContain("Show global sandbox and host service status"); + expect(result.out).toContain("Use ` status` for one sandbox"); }); it("sandbox-first status rejects unexpected positional arguments through command-id dispatch", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-sandbox-status-extra-")); writeSandboxRegistry(home); - const r = runWithEnv("alpha status extra", { HOME: home }); + const result = runWithEnv("alpha status extra", { HOME: home }); + + expect(result.code).toBe(2); + expect(result.out).toContain("Unexpected argument: extra"); + }); + + it("never emits an unsafe sandbox token in a copy-paste status command", () => { + const result = run("status 'alpha;echo pwned'"); - expect(r.code).toBe(2); - expect(r.out).toContain("Unexpected argument: extra"); + expect(result.code).toBe(2); + expect(result.out).toContain("Unexpected argument: alpha;echo pwned"); + expect(result.out).not.toContain("Run:"); }); }); diff --git a/test/gateway-state-reconcile-2276.test.ts b/test/gateway-state-reconcile-2276.test.ts index 689b67e7bb6..4b3d00da50d 100644 --- a/test/gateway-state-reconcile-2276.test.ts +++ b/test/gateway-state-reconcile-2276.test.ts @@ -2,28 +2,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 // -// Regression tests for issue #2276 — "wrong active gateway" must not remove -// the local registry entry when the NemoClaw gateway is healthy but some -// other OpenShell gateway is currently active. Covers the Architect's §5 -// scenarios 1-12. (Scenario 13 is a shell-level e2e, skipped.) -// -// Updated for issue #4497 — a routine `connect` against a healthy gateway must -// no longer auto-remove the local registry entry even when the live sandbox is -// truly gone (Scenario 1, formerly destructive). `status` recommends -// `rebuild --yes` for stuck/stale sandboxes, so deleting the registry entry in -// `connect` would race that recommendation and leave `rebuild` with nothing to -// recover. Intentional purges now go through the explicit `destroy` command. -// -// Each test spawns `nemoclaw.js` as a child process with a stub `openshell` -// binary on the $PATH. The stub is configured per-scenario via a JSON -// "script" file: it records every invocation and returns canned output -// based on the current scenario state. We then assert on: -// - registry file survival (present vs removed) -// - onboard-session.json's sandboxName field (cleared vs preserved) -// - user-facing stdout/stderr messages -// - exit code -// - openshell command call log (no prompt helpers, no `gateway select` -// in forbidden scenarios). +// Cross-command regression contract for issues #2276 and #4497. Direct +// gateway-state, status, and skill-action tests own the individual lifecycle +// decisions; this file retains the one process boundary that proves a failed +// `connect` preserves enough local state for a subsequent `rebuild --yes`. +// See gateway-state-drift.test.ts, status-flow.test.ts, +// gateway-runtime-action.test.ts, skill-install.test.ts, and the typed skill +// command adapter tests for scenarios 1-12. import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; @@ -39,23 +24,9 @@ const SANDBOX_NAME = "my-assistant"; // Output fixtures that mirror real OpenShell CLI output. const GATEWAY_INFO_NEMOCLAW = "Gateway Info\n\nGateway: nemoclaw\nGateway endpoint: https://127.0.0.1:8080/\n"; -const GATEWAY_INFO_MISSING = "No gateway metadata found"; -const GATEWAY_INFO_EMPTY = ""; const STATUS_CONNECTED_NEMOCLAW = "Server Status\n\nGateway: nemoclaw\nServer: https://127.0.0.1:8080/\nStatus: Connected\n"; -const STATUS_CONNECTED_OPENSHELL = - "Server Status\n\nGateway: openshell\nServer: https://127.0.0.1:8080/\nStatus: Connected\n"; -const STATUS_CONNECTED_OTHER = - "Server Status\n\nGateway: other-gw\nServer: https://127.0.0.1:9090/\nStatus: Connected\n"; -const STATUS_REFUSED_NEMOCLAW = - "Server Status\n\nGateway: nemoclaw\nError: Connection refused (os error 111)\n"; -const STATUS_NO_GATEWAY = "Error: × No active gateway\n"; -const STATUS_EMPTY = ""; -const STATUS_MALFORMED = "??? garbage output ???"; - -const SANDBOX_GET_READY = - "Sandbox:\n\n Id: abc\n Name: my-assistant\n Namespace: openshell\n Phase: Ready\n"; const SANDBOX_GET_NOT_FOUND = "Error: × Not Found: sandbox not found"; interface ScenarioScript { @@ -69,7 +40,7 @@ interface ScenarioScript { gatewaySelect: { output: string; exit: number }; // whether `gateway select nemoclaw` flips the active gateway to nemoclaw selectFlipsActive: boolean; - // `sandbox list` output; defaults to the live sandbox for scenarios 1-12. + // `sandbox list` output; scenario 14 uses an empty list to enter stale recovery. sandboxList?: string; } @@ -80,8 +51,6 @@ interface HarnessResult { registryExists: boolean; registry: any; sessionSandboxName: string | null | undefined; - callLog: Array; - selectCalls: number; } let tmpDir: string; @@ -90,7 +59,6 @@ let homeLocalBin: string; let openshellPath: string; let stateFile: string; let scriptFile: string; -let callLogFile: string; function writeDefaultRegistry() { fs.writeFileSync( @@ -131,7 +99,6 @@ function writeDefaultSession() { function writeStubOpenshell(script: ScenarioScript) { fs.writeFileSync(scriptFile, JSON.stringify(script)); fs.writeFileSync(stateFile, JSON.stringify({})); - fs.writeFileSync(callLogFile, ""); // Inline stub — uses node as interpreter via execPath shebang. Reads // script each call so tests can tweak state between runs (not used here). @@ -139,14 +106,11 @@ function writeStubOpenshell(script: ScenarioScript) { const fs = require("fs"); const scriptPath = ${JSON.stringify(scriptFile)}; const statePath = ${JSON.stringify(stateFile)}; -const callLogPath = ${JSON.stringify(callLogFile)}; const script = JSON.parse(fs.readFileSync(scriptPath, "utf8")); const state = JSON.parse(fs.readFileSync(statePath, "utf8") || "{}"); const args = process.argv.slice(2); const requiredFeatures = "request-body-credential-rewrite websocket-credential-rewrite allow_all_known_mcp_methods"; -fs.appendFileSync(callLogPath, JSON.stringify(args) + "\\n"); - function cycle(key, list) { state[key] = (state[key] || 0) + 1; const idx = Math.min(state[key] - 1, list.length - 1); @@ -258,20 +222,6 @@ function runCli(action: string, extraEnv: Record = { } } - const callLog: Array = fs - .readFileSync(callLogFile, "utf-8") - .split("\n") - .filter(Boolean) - .map((l) => { - try { - return JSON.parse(l); - } catch { - return []; - } - }); - - const selectCalls = callLog.filter((c) => c[0] === "gateway" && c[1] === "select").length; - return { status: result.status, stdout: result.stdout || "", @@ -279,8 +229,6 @@ function runCli(action: string, extraEnv: Record = { registryExists, registry, sessionSandboxName, - callLog, - selectCalls, }; } @@ -300,7 +248,6 @@ beforeEach(() => { openshellPath = path.join(homeLocalBin, "openshell"); stateFile = path.join(tmpDir, "state.json"); scriptFile = path.join(tmpDir, "script.json"); - callLogFile = path.join(tmpDir, "calls.log"); fs.mkdirSync(homeLocalBin, { recursive: true }); fs.mkdirSync(registryDir, { recursive: true }); @@ -338,412 +285,6 @@ afterEach(() => { fs.rmSync(tmpDir, { recursive: true, force: true }); }); -// ─── Scenario 1 ─── connect is now non-destructive (#4497) ───────────────── -describe("connect with a healthy active nemoclaw gateway and a truly gone sandbox in scenario 1", () => { - it("preserves the registry entry and session, points at rebuild/destroy, and exits 1", { - timeout: TIMEOUT_MS, - }, () => { - writeStubOpenshell({ - sandboxGet: [{ output: SANDBOX_GET_NOT_FOUND, exit: 1 }], - status: [{ output: STATUS_CONNECTED_NEMOCLAW, exit: 0 }], - gatewayInfo: [{ output: GATEWAY_INFO_NEMOCLAW, exit: 0 }], - gatewaySelect: { output: "", exit: 0 }, - selectFlipsActive: false, - }); - - const r = runCli("connect"); - - assert.equal(r.status, 1, `expected exit 1, got ${r.status}\n${r.stderr}`); - assert.equal( - registrySandboxPresent(r), - true, - `expected registry entry preserved, got: ${JSON.stringify(r.registry)}`, - ); - assert.equal(r.sessionSandboxName, SANDBOX_NAME, "expected session sandboxName preserved"); - // #4497: no routine command may delete the state `rebuild` needs. - assert.doesNotMatch(r.stderr, /Removed stale local registry entry/); - assert.match(r.stderr, /registered locally, but is not present/); - assert.match(r.stderr, /preserved/); - assert.match(r.stderr, new RegExp(`${SANDBOX_NAME} rebuild --yes`)); - assert.match(r.stderr, new RegExp(`${SANDBOX_NAME} destroy`)); - }); -}); - -// ─── Scenario 2 ─── passive `status` must preserve registry state ───────── -describe("status with a healthy active nemoclaw gateway and a truly gone sandbox in scenario 2", () => { - it("reports the missing live sandbox without removing local registry state", { - timeout: TIMEOUT_MS, - }, () => { - writeStubOpenshell({ - sandboxGet: [{ output: SANDBOX_GET_NOT_FOUND, exit: 1 }], - status: [{ output: STATUS_CONNECTED_NEMOCLAW, exit: 0 }], - gatewayInfo: [{ output: GATEWAY_INFO_NEMOCLAW, exit: 0 }], - gatewaySelect: { output: "", exit: 0 }, - selectFlipsActive: false, - }); - - const r = runCli("status"); - - assert.equal(r.status, 1, `expected exit 1, got ${r.status}`); - assert.equal( - registrySandboxPresent(r), - true, - `expected registry entry preserved, got: ${JSON.stringify(r.registry)}`, - ); - assert.equal(r.sessionSandboxName, SANDBOX_NAME, "expected session sandboxName preserved"); - assert.match(r.stdout, /registered locally, but is not present/); - assert.match(r.stdout, /No local registry entry was removed/); - assert.doesNotMatch(r.stdout, /Removed stale local registry entry/); - }); -}); - -// ─── Scenario 3 ─── self-heal via gateway select succeeds ────────────────── -describe("status preserves the registry when selection succeeds and the sandbox reappears in scenario 3", () => { - it("attempts `gateway select nemoclaw`, re-queries, proceeds; registry preserved", { - timeout: TIMEOUT_MS, - }, () => { - writeStubOpenshell({ - // 1st sandbox get: NotFound (gw drifted); 2nd: Ready after select. - sandboxGet: [ - { output: SANDBOX_GET_NOT_FOUND, exit: 1 }, - { output: SANDBOX_GET_READY, exit: 0 }, - ], - // 1st status call: openshell active. 2nd: nemoclaw active. - status: [ - { output: STATUS_CONNECTED_OPENSHELL, exit: 0 }, - { output: STATUS_CONNECTED_NEMOCLAW, exit: 0 }, - ], - gatewayInfo: [{ output: GATEWAY_INFO_NEMOCLAW, exit: 0 }], - gatewaySelect: { output: "", exit: 0 }, - selectFlipsActive: true, - }); - - const r = runCli("status"); - - assert.equal( - registrySandboxPresent(r), - true, - `expected registry preserved, got: ${JSON.stringify(r.registry)}`, - ); - assert.equal(r.sessionSandboxName, SANDBOX_NAME, "expected session sandboxName preserved"); - // gateway select nemoclaw should have been invoked. - assert.ok(r.selectCalls >= 1, `expected ≥1 gateway select calls, got ${r.selectCalls}`); - }); -}); - -// ─── Scenario 4 ─── select fails → wrong_gateway_active, registry intact ─── -describe("connect when selection fails and the sandbox remains NotFound in scenario 4", () => { - it("surfaces wrong_gateway_active guidance, preserves registry, exits 1", { - timeout: TIMEOUT_MS, - }, () => { - writeStubOpenshell({ - sandboxGet: [ - { output: SANDBOX_GET_NOT_FOUND, exit: 1 }, - { output: SANDBOX_GET_NOT_FOUND, exit: 1 }, - ], - // All status probes show 'openshell' active (select "failed" to switch) - status: [{ output: STATUS_CONNECTED_OPENSHELL, exit: 0 }], - gatewayInfo: [{ output: GATEWAY_INFO_NEMOCLAW, exit: 0 }], - gatewaySelect: { output: "Error: failed to select", exit: 1 }, - selectFlipsActive: false, - }); - - const r = runCli("connect"); - - assert.equal(r.status, 1, `expected exit 1, got ${r.status}`); - assert.equal( - registrySandboxPresent(r), - true, - `registry must be preserved, got: ${JSON.stringify(r.registry)}`, - ); - assert.equal(r.sessionSandboxName, SANDBOX_NAME, "session sandboxName must be preserved"); - // User-facing guidance. - assert.match(r.stderr, /NOT been removed/); - assert.match(r.stderr, /openshell gateway select nemoclaw/); - assert.doesNotMatch(r.stderr, /Removed stale local registry entry/); - }); -}); - -// ─── Scenario 5 ─── exact #2276 repro: registry entry still present ──────── -describe("failed connect leaves the registry entry intact in scenario 5 (#2276)", () => { - it("after a failed connect triggered by drifted gateway, entry is still present", { - timeout: TIMEOUT_MS, - }, () => { - writeStubOpenshell({ - sandboxGet: [{ output: SANDBOX_GET_NOT_FOUND, exit: 1 }], - status: [{ output: STATUS_CONNECTED_OTHER, exit: 0 }], - gatewayInfo: [{ output: GATEWAY_INFO_NEMOCLAW, exit: 0 }], - gatewaySelect: { output: "", exit: 1 }, - selectFlipsActive: false, - }); - - const r = runCli("connect"); - - assert.equal(r.status, 1, `expected exit 1, got ${r.status}`); - assert.equal( - registrySandboxPresent(r), - true, - `registry must still contain '${SANDBOX_NAME}', got: ${JSON.stringify(r.registry)}`, - ); - assert.equal(r.sessionSandboxName, SANDBOX_NAME); - assert.match(r.stderr, /NOT been removed/); - assert.match(r.stderr, /openshell gateway select nemoclaw/); - }); -}); - -// ─── Scenario 6 ─── nemoclaw gateway missing + NotFound ──────────────────── -describe("connect with a missing nemoclaw gateway after restart in scenario 6", () => { - it("returns gateway_missing_after_restart, preserves registry, exits 1", { - timeout: TIMEOUT_MS, - }, () => { - writeStubOpenshell({ - sandboxGet: [{ output: SANDBOX_GET_NOT_FOUND, exit: 1 }], - status: [{ output: STATUS_NO_GATEWAY, exit: 0 }], - gatewayInfo: [{ output: GATEWAY_INFO_MISSING, exit: 1 }], - gatewaySelect: { output: "", exit: 1 }, - selectFlipsActive: false, - }); - - const r = runCli("connect"); - - assert.equal(r.status, 1); - assert.equal( - registrySandboxPresent(r), - true, - `registry must be preserved, got: ${JSON.stringify(r.registry)}`, - ); - assert.equal(r.sessionSandboxName, SANDBOX_NAME); - assert.doesNotMatch(r.stderr, /Removed stale local registry entry/); - assert.match( - r.stderr, - /(no longer configured|Start the gateway again|openshell gateway start)/i, - ); - }); -}); - -// ─── Scenario 7 ─── nemoclaw gateway unreachable + NotFound ──────────────── -describe("connect with an unreachable nemoclaw gateway after restart in scenario 7", () => { - it("returns gateway_unreachable_after_restart, preserves registry, exits 1", { - timeout: TIMEOUT_MS, - }, () => { - writeStubOpenshell({ - sandboxGet: [{ output: SANDBOX_GET_NOT_FOUND, exit: 1 }], - status: [{ output: STATUS_REFUSED_NEMOCLAW, exit: 0 }], - gatewayInfo: [{ output: GATEWAY_INFO_NEMOCLAW, exit: 0 }], - gatewaySelect: { output: "", exit: 1 }, - selectFlipsActive: false, - }); - - const r = runCli("connect"); - - assert.equal(r.status, 1); - assert.equal( - registrySandboxPresent(r), - true, - `registry must be preserved, got: ${JSON.stringify(r.registry)}`, - ); - assert.equal(r.sessionSandboxName, SANDBOX_NAME); - assert.doesNotMatch(r.stderr, /Removed stale local registry entry/); - assert.match( - r.stderr, - /(still refusing connections|openshell gateway start|verify `openshell status`)/i, - ); - }); -}); - -// ─── Scenario 8 ─── gateway info fails / unparseable ─────────────────────── -describe("gateway info failure preserves the registry with a safe default in scenario 8", () => { - it("non-zero exit on `openshell gateway info -g nemoclaw` still preserves registry", { - timeout: TIMEOUT_MS, - }, () => { - writeStubOpenshell({ - sandboxGet: [{ output: SANDBOX_GET_NOT_FOUND, exit: 1 }], - // connected to "openshell", not nemoclaw — but gateway info fails. - status: [{ output: STATUS_CONNECTED_OPENSHELL, exit: 0 }], - gatewayInfo: [{ output: GATEWAY_INFO_MISSING, exit: 1 }], - gatewaySelect: { output: "", exit: 1 }, - selectFlipsActive: false, - }); - - const r = runCli("connect"); - - assert.equal(r.status, 1); - assert.equal( - registrySandboxPresent(r), - true, - `registry must be preserved when gateway info fails, got: ${JSON.stringify(r.registry)}`, - ); - assert.equal(r.sessionSandboxName, SANDBOX_NAME); - assert.doesNotMatch(r.stderr, /Removed stale local registry entry/); - }); -}); - -// ─── Scenario 9 ─── openshell status empty / malformed ───────────────────── -describe("empty or malformed status leaves the registry untouched in scenario 9", () => { - it("preserves the registry without removal when status is empty and gateway info is missing", { - timeout: TIMEOUT_MS, - }, () => { - writeStubOpenshell({ - sandboxGet: [{ output: SANDBOX_GET_NOT_FOUND, exit: 1 }], - status: [{ output: STATUS_EMPTY, exit: 0 }], - gatewayInfo: [{ output: GATEWAY_INFO_EMPTY, exit: 0 }], - gatewaySelect: { output: "", exit: 1 }, - selectFlipsActive: false, - }); - - const r = runCli("connect"); - - assert.equal(r.status, 1); - assert.equal( - registrySandboxPresent(r), - true, - `registry must be preserved on empty status, got: ${JSON.stringify(r.registry)}`, - ); - assert.equal(r.sessionSandboxName, SANDBOX_NAME); - assert.doesNotMatch(r.stderr, /Removed stale local registry entry/); - }); - - it("preserves the registry when status and gateway info are malformed", { - timeout: TIMEOUT_MS, - }, () => { - writeStubOpenshell({ - sandboxGet: [{ output: SANDBOX_GET_NOT_FOUND, exit: 1 }], - status: [{ output: STATUS_MALFORMED, exit: 0 }], - gatewayInfo: [{ output: "garbage gateway info", exit: 0 }], - gatewaySelect: { output: "", exit: 1 }, - selectFlipsActive: false, - }); - - const r = runCli("connect"); - - assert.equal(r.status, 1); - assert.equal( - registrySandboxPresent(r), - true, - `registry must be preserved on malformed status, got: ${JSON.stringify(r.registry)}`, - ); - assert.doesNotMatch(r.stderr, /Removed stale local registry entry/); - }); -}); - -// ─── Scenario 10 ─── non-interactive mode: no prompts ────────────────────── -describe("non-interactive mode exits deterministically without prompts in scenario 10", () => { - it("NEMOCLAW_NON_INTERACTIVE=1 does not block on user input and exits 1", { - timeout: TIMEOUT_MS, - }, () => { - writeStubOpenshell({ - sandboxGet: [{ output: SANDBOX_GET_NOT_FOUND, exit: 1 }], - status: [{ output: STATUS_CONNECTED_OPENSHELL, exit: 0 }], - gatewayInfo: [{ output: GATEWAY_INFO_NEMOCLAW, exit: 0 }], - gatewaySelect: { output: "", exit: 1 }, - selectFlipsActive: false, - }); - - const r = runCli("connect", { NEMOCLAW_NON_INTERACTIVE: "1" }); - - assert.equal(r.status, 1); - assert.equal( - registrySandboxPresent(r), - true, - "registry must remain intact in non-interactive mode", - ); - assert.match(r.stderr, /NOT been removed/); - // No prompt-style "Press enter" / "? " should appear. - assert.doesNotMatch(r.stderr, /Press (enter|any key)|\?\s+\[/i); - assert.doesNotMatch(r.stdout, /Press (enter|any key)|\?\s+\[/i); - }); -}); - -// ─── Scenario 11 ─── cross-command parity: status drifts same way ────────── -describe("status gives guidance instead of removal for the wrong active gateway in scenario 11", () => { - it("drift case under `status` preserves registry and prints guidance", { - timeout: TIMEOUT_MS, - }, () => { - writeStubOpenshell({ - sandboxGet: [ - { output: SANDBOX_GET_NOT_FOUND, exit: 1 }, - { output: SANDBOX_GET_NOT_FOUND, exit: 1 }, - ], - status: [{ output: STATUS_CONNECTED_OPENSHELL, exit: 0 }], - gatewayInfo: [{ output: GATEWAY_INFO_NEMOCLAW, exit: 0 }], - gatewaySelect: { output: "", exit: 1 }, - selectFlipsActive: false, - }); - - const r = runCli("status"); - - assert.equal( - registrySandboxPresent(r), - true, - `registry must be preserved on status drift, got: ${JSON.stringify(r.registry)}`, - ); - assert.equal(r.sessionSandboxName, SANDBOX_NAME); - // status writes to stdout (console.log), not stderr. - const combined = `${r.stdout}\n${r.stderr}`; - assert.match(combined, /NOT been removed/); - assert.match(combined, /openshell gateway select nemoclaw/); - assert.doesNotMatch(combined, /Removed stale local registry entry/); - }); -}); - -// ─── Scenario 12 ─── cross-command parity: skill install drifts same way ─── -describe("skill install gives guidance instead of removal for the wrong active gateway in scenario 12", () => { - it("skill install under drift preserves registry, exits 1 with guidance", { - timeout: TIMEOUT_MS, - }, () => { - // Minimal valid skill directory. - const skillDir = path.join(tmpDir, "my-skill"); - fs.mkdirSync(skillDir, { recursive: true }); - fs.writeFileSync( - path.join(skillDir, "SKILL.md"), - "---\nname: my-skill\ndescription: test\n---\nHello\n", - ); - - writeStubOpenshell({ - sandboxGet: [{ output: SANDBOX_GET_NOT_FOUND, exit: 1 }], - status: [{ output: STATUS_CONNECTED_OPENSHELL, exit: 0 }], - gatewayInfo: [{ output: GATEWAY_INFO_NEMOCLAW, exit: 0 }], - gatewaySelect: { output: "", exit: 1 }, - selectFlipsActive: false, - }); - - const repoRoot = path.join(import.meta.dirname, ".."); - const result = spawnSync( - process.execPath, - [path.join(repoRoot, "bin", "nemoclaw.js"), SANDBOX_NAME, "skill", "install", skillDir], - { - cwd: repoRoot, - encoding: "utf-8", - timeout: TIMEOUT_MS, - env: { - ...process.env, - HOME: tmpDir, - PATH: `${homeLocalBin}:/usr/bin:/bin`, - NO_COLOR: "1", - }, - }, - ); - - const registryPath = path.join(registryDir, "sandboxes.json"); - const reg = fs.existsSync(registryPath) - ? JSON.parse(fs.readFileSync(registryPath, "utf-8")) - : null; - const sessionPath = path.join(registryDir, "onboard-session.json"); - const session = fs.existsSync(sessionPath) - ? JSON.parse(fs.readFileSync(sessionPath, "utf-8")) - : {}; - - assert.equal(result.status, 1, `expected exit 1, got ${result.status}\n${result.stderr}`); - assert.ok( - reg && reg.sandboxes && reg.sandboxes[SANDBOX_NAME], - `registry must be preserved on skill install drift, got: ${JSON.stringify(reg)}`, - ); - assert.equal(session.sandboxName, SANDBOX_NAME); - assert.match(result.stderr, /NOT been removed/); - assert.match(result.stderr, /openshell gateway select nemoclaw/); - }); -}); - // ─── Scenario 14 (#4497) ─── connect preserves enough state for rebuild ───── // End-to-end recovery contract for the REOPENED issue: a healthy gateway // reports the sandbox as gone, `connect` must NOT delete the registry entry, diff --git a/test/process-recovery-managed-controller.test.ts b/test/process-recovery-managed-controller.test.ts index 133f342c42c..735eeced42f 100644 --- a/test/process-recovery-managed-controller.test.ts +++ b/test/process-recovery-managed-controller.test.ts @@ -183,6 +183,7 @@ beta 127.0.0.1 18789 12345 running`; }, ); let healthProbeCalls = 0; + const spawnedCommands: string[] = []; process.env.NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS = "2"; process.env.NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS = "0"; @@ -190,7 +191,8 @@ beta 127.0.0.1 18789 12345 running`; try { vi.spyOn(childProcess, "spawnSync").mockImplementation( - (_command: unknown, rawArgs: unknown) => { + (command: unknown, rawArgs: unknown) => { + spawnedCommands.push(String(command)); const isHealthProbe = getSandboxExecShellCommand(rawArgs).includes("HTTP_CODE=$(curl"); healthProbeCalls += Number(isHealthProbe); return ( @@ -226,6 +228,7 @@ beta 127.0.0.1 18789 12345 running`; expectedActions.map((action) => ["beta", action]), ); expect(healthProbeCalls).toBe(1); + expect(spawnedCommands).not.toContain("ssh"); } finally { previousWaitSeconds === undefined ? delete process.env.NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS diff --git a/test/process-recovery-primitives.test.ts b/test/process-recovery-primitives.test.ts index d79547ded7f..92453090ef8 100644 --- a/test/process-recovery-primitives.test.ts +++ b/test/process-recovery-primitives.test.ts @@ -289,6 +289,51 @@ describe("executeSandboxExecCommand", () => { } }); + it("honors the sandbox-exec timeout without falling back to SSH", () => { + const childProcess = requireSource("node:child_process"); + const dockerExec = requireSource("../src/lib/adapters/docker/exec.ts"); + const privilegedExec = requireSource("../src/lib/sandbox/privileged-exec.ts"); + const timeoutError = Object.assign(new Error("timed out"), { code: "ETIMEDOUT" }); + const spawn = vi.spyOn(childProcess, "spawnSync").mockReturnValue({ + status: null, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\n", + stderr: "", + error: timeoutError, + } as never); + vi.spyOn(privilegedExec, "privilegedSandboxExecArgv").mockReturnValue([ + "exec", + "--user", + "root", + "openshell-alpha", + "sh", + "-c", + "marked-command", + ]); + const dockerSpawnSync = vi.spyOn(dockerExec, "dockerSpawnSync").mockReturnValue({ + status: null, + stdout: "", + stderr: "", + error: timeoutError, + } as never); + const previousTimeout = process.env.NEMOCLAW_SANDBOX_EXEC_TIMEOUT_MS; + process.env.NEMOCLAW_SANDBOX_EXEC_TIMEOUT_MS = "50"; + + try { + const result = withFakeOpenshellBinary(() => + executeSandboxExecCommand("alpha", "printf RUNNING"), + ); + + expect(result).toBeNull(); + expect(spawn.mock.calls.some(([command]) => command === "ssh")).toBe(false); + expect(spawn.mock.calls[0]?.[2]).toEqual(expect.objectContaining({ timeout: 50 })); + expect(dockerSpawnSync.mock.calls[0]?.[1]).toEqual(expect.objectContaining({ timeout: 50 })); + } finally { + previousTimeout === undefined + ? delete process.env.NEMOCLAW_SANDBOX_EXEC_TIMEOUT_MS + : (process.env.NEMOCLAW_SANDBOX_EXEC_TIMEOUT_MS = previousTimeout); + } + }); + it("parses stdout-framed root exec output after the startup marker", () => { const childProcess = requireSource("node:child_process"); vi.spyOn(childProcess, "spawnSync").mockReturnValue({ diff --git a/test/process-recovery.test.ts b/test/process-recovery.test.ts index 3898d8c7619..7fa3e5cb8f3 100644 --- a/test/process-recovery.test.ts +++ b/test/process-recovery.test.ts @@ -1248,7 +1248,9 @@ hermes-box 127.0.0.1 8642 12346 running`; status: 0, output: `SANDBOX BIND PORT PID STATUS\nbeta 127.0.0.1 18789 12345 running`, }); - vi.spyOn(openshellRuntime, "runOpenshell").mockReturnValue({ status: 0 } as never); + const runOpenshell = vi + .spyOn(openshellRuntime, "runOpenshell") + .mockReturnValue({ status: 0 } as never); withFakeOpenshellBinary(() => checkAndRecoverSandboxProcesses("beta", { @@ -1257,6 +1259,11 @@ hermes-box 127.0.0.1 8642 12346 running`; }), ); expect(requestGatewaySupervisorAction).not.toHaveBeenCalled(); + expect( + runOpenshell.mock.calls.some( + ([rawArgs]) => Array.isArray(rawArgs) && rawArgs[0] === "forward" && rawArgs[1] === "start", + ), + ).toBe(false); }); it("fails safe on a running Hermes gateway when the supervisor channel is unreachable", () => { diff --git a/test/sandbox-connect-inference/route-swap-repair.test.ts b/test/sandbox-connect-inference/route-swap-repair.test.ts index e20d99ff8e6..f874ad0900b 100644 --- a/test/sandbox-connect-inference/route-swap-repair.test.ts +++ b/test/sandbox-connect-inference/route-swap-repair.test.ts @@ -6,52 +6,9 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import { testTimeoutOptions } from "../helpers/timeouts"; -import { createVmRootfs, isHostWsl, runConnect, setupFixture } from "./helpers"; +import { isHostWsl, runConnect, setupFixture } from "./helpers"; describe("sandbox connect inference route swap (#1248)", () => { - it( - "skips the vLLM model preflight on connect --probe-only but keeps it for a full connect (#4585)", - testTimeoutOptions(20_000), - () => { - const fixture = setupFixture( - { - name: "my-sandbox", - model: "claude-sonnet-4-20250514", - provider: "anthropic-prod", - gpuEnabled: false, - policies: [], - }, - "anthropic-prod", - "claude-sonnet-4-20250514", - { inferenceProbeResponses: ["OK 200", "OK 200"] }, - ); - const bogus = { NEMOCLAW_VLLM_MODEL: "definitely-not-a-real-vllm-model" }; - const PREFLIGHT_HINT = "NEMOCLAW_VLLM_MODEL is consumed by"; - - // probe-only / recover never install or serve a model, so the express-vLLM - // model preflight must be skipped rather than hard-exiting the probe. - const probe = runConnect(fixture.tmpDir, fixture.sandboxName, bogus, ["--probe-only"]); - const probeOut = (probe.stdout || "") + (probe.stderr || ""); - // probe-only must proceed (not just avoid the hint): a non-zero exit would - // mean it failed for some other reason before the skipped preflight. - expect(probe.status).toBe(0); - expect(probeOut).not.toContain(PREFLIGHT_HINT); - - // A fixture remains truthful across repeated CLI invocations in one - // test: its advertised running forward keeps listening until afterEach. - const repeatedProbe = runConnect(fixture.tmpDir, fixture.sandboxName, bogus, [ - "--probe-only", - ]); - expect(repeatedProbe.status).toBe(0); - - // A full connect still runs the preflight and fails fast on the bogus value. - const full = runConnect(fixture.tmpDir, fixture.sandboxName, bogus, []); - const fullOut = (full.stdout || "") + (full.stderr || ""); - expect(full.status).toBe(1); - expect(fullOut).toContain(PREFLIGHT_HINT); - }, - ); - it( "swaps inference route when live route does not match sandbox provider", testTimeoutOptions(20_000), @@ -89,238 +46,6 @@ describe("sandbox connect inference route swap (#1248)", () => { }, ); - it( - "warns and aligns the route even in --probe-only quiet mode (#3726)", - testTimeoutOptions(20_000), - () => { - const { tmpDir, stateFile, sandboxName } = setupFixture( - { - name: "probe-diverged-sandbox", - model: "claude-sonnet-4-20250514", - provider: "anthropic-prod", - gpuEnabled: false, - policies: [], - }, - "nvidia-prod", - "nvidia/nemotron-3-super-120b-a12b", - ); - - const result = runConnect(tmpDir, sandboxName, {}, ["--probe-only"]); - expect(result.status).toBe(0); - - const state = JSON.parse(fs.readFileSync(stateFile, "utf-8")); - const combined = (result.stdout || "") + (result.stderr || ""); - expect(combined).toContain("differs from the recorded route"); - expect(combined).toContain("Aligning the gateway to anthropic-prod/claude-sonnet-4-20250514"); - expect(state.inferenceSetCalls).toContainEqual([ - "--provider", - "anthropic-prod", - "--model", - "claude-sonnet-4-20250514", - "--no-verify", - ]); - expect(state.sandboxConnectCalls).toEqual([]); - }, - ); - - it.each([ - ["null", null, null], - ["provider-only", "nvidia-prod", null], - ["model-only", null, "nvidia/test"], - ["blank-provider", " ", "nvidia/test"], - ["blank-model", "nvidia-prod", " "], - ])( - "skips inference reconciliation for %s registry entries (#5937)", - testTimeoutOptions(20_000), - (_description, provider, model) => { - const { tmpDir, stateFile, sandboxName } = setupFixture( - { - name: "legacy-sandbox", - provider, - model, - gpuEnabled: false, - policies: [], - }, - "nvidia-prod", - "nvidia/nemotron-3-super-120b-a12b", - ); - - const result = runConnect(tmpDir, sandboxName); - expect(result.status).toBe(0); - - const state = JSON.parse(fs.readFileSync(stateFile, "utf-8")); - expect(state.inferenceGetCalls).toEqual([]); - expect(state.inferenceSetCalls).toEqual([]); - }, - ); - - it( - "does not swap when live route already matches sandbox provider", - testTimeoutOptions(20_000), - () => { - const { tmpDir, stateFile, sandboxName } = setupFixture( - { - name: "matched-sandbox", - model: "nvidia/nemotron-3-super-120b-a12b", - provider: "nvidia-prod", - gpuEnabled: false, - policies: [], - }, - "nvidia-prod", - "nvidia/nemotron-3-super-120b-a12b", - ); - - const result = runConnect(tmpDir, sandboxName); - expect(result.status).toBe(0); - - const state = JSON.parse(fs.readFileSync(stateFile, "utf-8")); - expect(state.inferenceSetCalls.length).toBe(0); - }, - ); - - it( - "repairs the kubernetes sandbox DNS proxy when inference.local returns 503", - testTimeoutOptions(20_000), - () => { - const { tmpDir, stateFile, sandboxName } = setupFixture( - { - name: "stale-dns-sandbox", - model: "nvidia/nemotron-3-super-120b-a12b", - provider: "nvidia-prod", - gpuEnabled: false, - openshellDriver: "kubernetes", - policies: [], - }, - "nvidia-prod", - "nvidia/nemotron-3-super-120b-a12b", - { - inferenceProbeResponses: [ - 'BROKEN 503 {"error":"inference service unavailable"}', - "OK 200", - ], - }, - ); - - const result = runConnect(tmpDir, sandboxName); - expect(result.status).toBe(0); - - const state = JSON.parse(fs.readFileSync(stateFile, "utf-8")); - const dockerCalls = state.dockerCalls as string[][]; - const inferenceExecCalls = state.sandboxExecCalls.filter((call: string[]) => - JSON.stringify(call).includes("inference.local/v1/models"), - ); - expect(state.inferenceSetCalls.length).toBe(0); - expect(inferenceExecCalls.length).toBe(2); - expect(dockerCalls.some((call) => call.join(" ").includes("get service kube-dns"))).toBe( - true, - ); - expect(dockerCalls.some((call) => call.join(" ").includes("get endpoints kube-dns"))).toBe( - false, - ); - - const combined = (result.stdout || "") + (result.stderr || ""); - expect(combined).toContain("inference.local is unavailable inside 'stale-dns-sandbox'"); - expect(combined).toContain("inference.local route repaired"); - }, - ); - - it( - "uses the VM DNS monkeypatch without legacy DNS repair or route reset when it restores inference.local", - testTimeoutOptions(20_000), - () => { - const { tmpDir, stateFile, sandboxName } = setupFixture( - { - name: "vm-dns-sandbox", - model: "nvidia/nemotron-3-super-120b-a12b", - provider: "nvidia-prod", - gpuEnabled: false, - openshellDriver: "vm", - policies: [], - }, - "nvidia-prod", - "nvidia/nemotron-3-super-120b-a12b", - { - inferenceProbeResponses: [ - 'BROKEN 503 {"error":"inference service unavailable"}', - "OK 200", - ], - }, - ); - const rootfs = createVmRootfs(tmpDir); - - const result = runConnect(tmpDir, sandboxName, { - NEMOCLAW_FORCE_VM_DNS_MONKEYPATCH: "1", - }); - expect(result.status).toBe(0); - - const state = JSON.parse(fs.readFileSync(stateFile, "utf-8")); - expect(state.inferenceSetCalls.length).toBe(0); - expect(state.dockerCalls.length).toBe(0); - expect(fs.readFileSync(path.join(rootfs, "etc", "resolv.conf"), "utf-8")).toBe( - "nameserver 192.168.127.1\n", - ); - expect( - fs.readFileSync(path.join(rootfs, "srv", "openshell-vm-sandbox-init.sh"), "utf-8"), - ).toContain("nameserver ${GVPROXY_GATEWAY_IP}"); - - const combined = (result.stdout || "") + (result.stderr || ""); - expect(combined).toContain("Applying OpenShell VM DNS monkeypatch"); - expect(combined).toContain("inference.local route repaired"); - expect(combined).not.toContain("Reapplying OpenShell inference route"); - expect(combined).not.toContain("Repairing sandbox DNS proxy"); - }, - ); - - it( - "stops before sandbox connect when inference.local is still broken after route reset", - testTimeoutOptions(20_000), - () => { - const { tmpDir, stateFile, sandboxName } = setupFixture( - { - name: "still-broken-sandbox", - model: "nvidia/nemotron-3-super-120b-a12b", - provider: "nvidia-prod", - gpuEnabled: false, - policies: [], - }, - "nvidia-prod", - "nvidia/nemotron-3-super-120b-a12b", - { - inferenceProbeResponses: [ - 'BROKEN 503 {"error":"upstream unavailable"}', - 'BROKEN 503 {"error":"upstream unavailable"}', - 'BROKEN 503 {"error":"upstream unavailable"}', - 'BROKEN 503 {"error":"upstream unavailable"}', - 'BROKEN 503 {"error":"upstream unavailable"}', - 'BROKEN 503 {"error":"upstream unavailable"}', - 'BROKEN 503 {"error":"upstream unavailable"}', - ], - }, - ); - - const result = runConnect(tmpDir, sandboxName); - expect(result.status).toBe(1); - - const state = JSON.parse(fs.readFileSync(stateFile, "utf-8")); - expect(state.inferenceSetCalls).toEqual([ - [ - "--provider", - "nvidia-prod", - "--model", - "nvidia/nemotron-3-super-120b-a12b", - "--no-verify", - ], - ]); - expect(state.sandboxConnectCalls).toEqual([]); - - const combined = (result.stdout || "") + (result.stderr || ""); - expect(combined).toContain("inference.local is still unavailable"); - expect(combined).toContain( - "Connect is stopping because the sandbox inference route is known to be broken", - ); - }, - ); - it( "resets local Ollama routes without leaking proxy env or bearer tokens", testTimeoutOptions(20_000), diff --git a/test/support/connect-flow-test-harness.ts b/test/support/connect-flow-test-harness.ts index bd16436537d..9397938e854 100644 --- a/test/support/connect-flow-test-harness.ts +++ b/test/support/connect-flow-test-harness.ts @@ -7,6 +7,7 @@ import { createRequire } from "node:module"; import { type MockInstance, vi } from "vitest"; import type { SecretBoundaryRefusalReason } from "../../src/lib/actions/sandbox/hermes-secret-boundary-recovery"; +import type { SandboxEntry } from "../../src/lib/state/registry"; type ConnectSandbox = typeof import("../../src/lib/actions/sandbox/connect")["connectSandbox"]; @@ -19,18 +20,25 @@ requireDist(connectModulePath); delete require.cache[requireDist.resolve(connectModulePath)]; export type ConnectHarness = { + applyVmDnsMonkeypatchSpy: MockInstance; captureOpenshellSpy: MockInstance; checkAndRecoverSpy: MockInstance; connectSandbox: ConnectSandbox; ensureOllamaAuthProxySpy: MockInstance; errorSpy: MockInstance; logSpy: MockInstance; + preflightVllmSpy: MockInstance; runAutoPairSpy: MockInstance; + runOpenshellSpy: MockInstance; + runSetupDnsProxySpy: MockInstance; spawnSyncSpy: MockInstance; }; export type ConnectHarnessOptions = { agentName?: string; + inferenceGetOutput?: string; + inferenceProbeResponses?: string[]; + registryEntry?: Partial; sessionAgent?: unknown; listOutput?: string; processCheck?: { @@ -76,6 +84,7 @@ export function createConnectHarness(options: ConnectHarnessOptions = {}): Conne const runtime = requireDist("../../src/lib/adapters/openshell/runtime.js"); const resolve = requireDist("../../src/lib/adapters/openshell/resolve.js"); const agentRuntime = requireDist("../../src/lib/agent/runtime.js"); + const dns = requireDist("../../src/lib/actions/dns/index.js"); const gatewayState = requireDist("../../src/lib/actions/sandbox/gateway-state.js"); const processRecovery = requireDist("../../src/lib/actions/sandbox/process-recovery.js"); const autoPairApproval = requireDist("../../src/lib/actions/sandbox/auto-pair-approval.js"); @@ -89,13 +98,17 @@ export function createConnectHarness(options: ConnectHarnessOptions = {}): Conne const sandboxVersion = requireDist("../../src/lib/sandbox/version.js"); const registry = requireDist("../../src/lib/state/registry.js"); const sandboxSession = requireDist("../../src/lib/state/sandbox-session.js"); + const vmDnsMonkeypatch = requireDist("../../src/lib/actions/sandbox/vm-dns-monkeypatch.js"); - vi.spyOn(connectVllmPreflight, "preflightVllmModelEnvOrExit").mockImplementation(() => undefined); + const preflightVllmSpy = vi + .spyOn(connectVllmPreflight, "preflightVllmModelEnvOrExit") + .mockImplementation(() => undefined); vi.spyOn(gatewayState, "ensureLiveSandboxOrExit").mockResolvedValue({ state: "present", output: "Name: alpha\nPhase: Ready\n", }); vi.spyOn(gatewayFailureClassifier, "isDockerRuntimeDown").mockReturnValue(false); + const inferenceProbeResponses = [...(options.inferenceProbeResponses ?? [])]; const captureOpenshellSpy = vi .spyOn(runtime, "captureOpenshell") .mockImplementation((args: unknown) => { @@ -104,10 +117,25 @@ export function createConnectHarness(options: ConnectHarnessOptions = {}): Conne return { status: 0, output: options.listOutput ?? "alpha Ready" }; } if (argv[0] === "inference" && argv[1] === "get") { - return { status: 0, output: "Provider: unknown\nModel: unknown\n" }; + return { + status: 0, + output: options.inferenceGetOutput ?? "Provider: unknown\nModel: unknown\n", + }; + } + if ( + argv[0] === "sandbox" && + argv[1] === "exec" && + argv.join(" ").includes("inference.local/v1/models") + ) { + return { status: 0, output: inferenceProbeResponses.shift() ?? "OK 200" }; } return { status: 0, output: "" }; }); + const runOpenshellSpy = vi.spyOn(runtime, "runOpenshell").mockReturnValue({ status: 0 }); + const runSetupDnsProxySpy = vi.spyOn(dns, "runSetupDnsProxy").mockReturnValue({ exitCode: 0 }); + const applyVmDnsMonkeypatchSpy = vi + .spyOn(vmDnsMonkeypatch, "applyOpenShellVmDnsMonkeypatch") + .mockReturnValue({ attempted: true, changed: true, ok: true, status: "applied" }); vi.spyOn(runtime, "getOpenshellBinary").mockReturnValue("openshell"); vi.spyOn(resolve, "resolveOpenshell").mockReturnValue("/usr/bin/openshell"); vi.spyOn(sandboxSession, "getActiveSandboxSessions").mockReturnValue({ @@ -127,6 +155,9 @@ export function createConnectHarness(options: ConnectHarnessOptions = {}): Conne agent: options.agentName ?? "openclaw", provider: null, model: null, + gpuEnabled: false, + policies: [], + ...options.registryEntry, }); vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue( (options.sessionAgent ?? { name: "openclaw" }) as never, @@ -141,13 +172,17 @@ export function createConnectHarness(options: ConnectHarnessOptions = {}): Conne spawnSyncSpy.mockClear(); return { + applyVmDnsMonkeypatchSpy, captureOpenshellSpy, checkAndRecoverSpy, connectSandbox: requireDist(connectModulePath).connectSandbox, ensureOllamaAuthProxySpy, errorSpy, logSpy, + preflightVllmSpy, runAutoPairSpy, + runOpenshellSpy, + runSetupDnsProxySpy, spawnSyncSpy, }; } diff --git a/test/support/status-flow-test-harness.ts b/test/support/status-flow-test-harness.ts new file mode 100644 index 00000000000..bffc4b58240 --- /dev/null +++ b/test/support/status-flow-test-harness.ts @@ -0,0 +1,215 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createRequire } from "node:module"; + +import { type MockInstance, vi } from "vitest"; + +import type { SandboxGatewayState } from "../../src/lib/actions/sandbox/gateway-state"; +import type { SandboxStatusPreflightResult } from "../../src/lib/actions/sandbox/status-preflight"; +import type { ProviderHealthStatus } from "../../src/lib/inference/health"; + +type ShowSandboxStatus = typeof import("../../src/lib/actions/sandbox/status")["showSandboxStatus"]; + +const requireDist = createRequire(import.meta.url); +const statusModulePath = "../../src/lib/actions/sandbox/status.js"; + +// Warm the CommonJS source graph outside the first test's timeout. Each harness +// still reloads the entry module after installing its dependency spies. +requireDist(statusModulePath); +delete require.cache[requireDist.resolve(statusModulePath)]; + +export type StatusFlowHarness = { + checkAgentVersionSpy: MockInstance; + collectSandboxStatusSnapshotSpy: MockInstance; + getActiveSandboxSessionsSpy: MockInstance; + getSandboxDockerRuntimeSpy: MockInstance; + logSpy: MockInstance; + removeSandboxSpy: MockInstance; + showSandboxStatus: ShowSandboxStatus; +}; + +const baseSandboxEntry = { + name: "alpha", + model: "nvidia/nemotron", + provider: "ollama-local", + policies: ["npm", "telegram"], + hostGpuDetected: true, + gpuEnabled: true, + sandboxGpuEnabled: true, + sandboxGpuMode: "auto", + sandboxGpuDevice: "all", + sandboxGpuProof: { + status: "failed", + label: "cuInit", + detail: "CUDA initialization failed", + }, + openshellDriver: "docker", + openshellVersion: "0.1.2", + dashboardPort: 18789, + agentVersion: "0.1.0", +}; + +export type StatusFlowHarnessOptions = { + currentModel?: string; + currentProvider?: string; + inferenceHealth?: ProviderHealthStatus | null; + lookup?: SandboxGatewayState; + lookupState?: "present" | "missing"; + preflight?: SandboxStatusPreflightResult; + sandboxEntry?: Partial> & { + agent?: string | null; + agentVersion?: string | null; + }; + shieldsPosture?: { + mode: "locked" | "mutable_default" | "mutable"; + detail: string; + }; + versionCheck?: { + sandboxVersion?: string | null; + expectedVersion?: string | null; + isStale: boolean; + detectionMethod?: string; + schemeMismatch?: boolean; + verificationFailed?: boolean; + }; +}; + +export function resetStatusFlowModuleCache(): void { + delete require.cache[requireDist.resolve(statusModulePath)]; +} + +export function createStatusFlowHarness(options: StatusFlowHarnessOptions = {}): StatusFlowHarness { + resetStatusFlowModuleCache(); + + const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); + vi.spyOn(console, "error").mockImplementation(() => undefined); + + const statusPreflight = requireDist("../../src/lib/actions/sandbox/status-preflight.js"); + const statusSnapshot = requireDist("../../src/lib/actions/sandbox/status-snapshot.js"); + const dockerHealth = requireDist("../../src/lib/actions/sandbox/docker-health.js"); + const processRecovery = requireDist("../../src/lib/actions/sandbox/process-recovery.js"); + const resolve = requireDist("../../src/lib/adapters/openshell/resolve.js"); + const agentRuntime = requireDist("../../src/lib/agent/runtime.js"); + const nim = requireDist("../../src/lib/inference/nim.js"); + const sandboxVersion = requireDist("../../src/lib/sandbox/version.js"); + const shields = requireDist("../../src/lib/shields/index.js"); + const registry = requireDist("../../src/lib/state/registry.js"); + const sandboxSession = requireDist("../../src/lib/state/sandbox-session.js"); + + const lookup: SandboxGatewayState = + options.lookup ?? + (options.lookupState === "missing" + ? { + state: "missing", + output: "sandbox alpha not found", + recoveredGateway: true, + recoveryVia: "gateway reattach", + } + : { + state: "present", + output: "Name: alpha\nPhase: Ready\nEndpoint: http://127.0.0.1:18789\n", + recoveredGateway: true, + recoveryVia: "gateway reattach", + recoveredSandbox: true, + recoverySandboxVia: "docker unpause", + }); + + const sandboxEntry = { ...baseSandboxEntry, ...options.sandboxEntry }; + + vi.spyOn(registry, "getSandbox").mockReturnValue(sandboxEntry); + const removeSandboxSpy = vi.spyOn(registry, "removeSandbox").mockImplementation(() => undefined); + vi.spyOn(statusPreflight, "getSandboxStatusPreflight").mockResolvedValue( + options.preflight ?? { + failure: null, + failureLayer: null, + suppressInferenceProbe: false, + exitCode: 0, + }, + ); + const collectSandboxStatusSnapshotSpy = vi + .spyOn(statusSnapshot, "collectSandboxStatusSnapshot") + .mockResolvedValue({ + sb: sandboxEntry, + lookup, + rpcIssue: null, + currentModel: options.currentModel ?? "nvidia/nemotron-live", + currentProvider: options.currentProvider ?? "ollama-local", + inferenceHealth: + options.inferenceHealth === undefined + ? { + ok: true, + probed: true, + providerLabel: "Ollama", + endpoint: "http://127.0.0.1:11434/v1/chat/completions", + detail: "chat completions probe passed", + subprobes: [ + { + ok: false, + probed: true, + providerLabel: "Inference gateway chain", + endpoint: "http://127.0.0.1:19000/v1/chat/completions", + detail: "gateway refused connection", + probeLabel: "gateway", + failureLabel: "unreachable", + }, + ], + } + : options.inferenceHealth, + }); + const getSandboxDockerRuntimeSpy = vi + .spyOn(dockerHealth, "getSandboxDockerRuntime") + .mockReturnValue({ + containerName: "openshell-alpha", + health: "unhealthy", + paused: false, + }); + vi.spyOn(processRecovery, "isSandboxGatewayRunningForStatus").mockResolvedValue(false); + vi.spyOn(resolve, "resolveOpenshell").mockReturnValue("/usr/bin/openshell"); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ name: "openclaw" }); + vi.spyOn(agentRuntime, "getAgentDisplayName").mockReturnValue("OpenClaw"); + vi.spyOn(agentRuntime, "getGatewayCommand").mockReturnValue("openclaw daemon"); + vi.spyOn(nim, "nimStatus").mockReturnValue({ + running: true, + healthy: false, + container: "alpha-nim", + }); + vi.spyOn(nim, "nimStatusByName").mockReturnValue({ + running: false, + healthy: false, + container: null, + }); + vi.spyOn(nim, "shouldShowNimLine").mockReturnValue(true); + const checkAgentVersionSpy = vi.spyOn(sandboxVersion, "checkAgentVersion").mockReturnValue( + options.versionCheck ?? { + sandboxVersion: "0.1.0", + expectedVersion: "0.2.0", + isStale: true, + detectionMethod: "runtime", + }, + ); + vi.spyOn(shields, "getShieldsPosture").mockReturnValue( + options.shieldsPosture ?? { + mode: "mutable_default", + detail: "mutable default", + }, + ); + const getActiveSandboxSessionsSpy = vi + .spyOn(sandboxSession, "getActiveSandboxSessions") + .mockReturnValue({ + detected: true, + sessions: [{ pid: 1 }, { pid: 2 }], + }); + + logSpy.mockClear(); + + return { + checkAgentVersionSpy, + collectSandboxStatusSnapshotSpy, + getActiveSandboxSessionsSpy, + getSandboxDockerRuntimeSpy, + logSpy, + removeSandboxSpy, + showSandboxStatus: requireDist(statusModulePath).showSandboxStatus, + } satisfies StatusFlowHarness; +}