diff --git a/src/nemoclaw.ts b/src/nemoclaw.ts index 54c35f5ceff..077ce14f654 100644 --- a/src/nemoclaw.ts +++ b/src/nemoclaw.ts @@ -31,7 +31,7 @@ const { validateName, } = require("./lib/runner"); const { resolveOpenshell } = require("./lib/resolve-openshell"); -const { startGatewayForRecovery } = require("./lib/onboard"); +const { startGatewayForRecovery, pruneKnownHostsEntries } = require("./lib/onboard"); const { getCredential, deleteCredential, @@ -474,7 +474,8 @@ async function recoverRegistryEntries({ requestedSandboxName = null } = {}) { } const seeded = seedRecoveryMetadata(current, session, requestedSandboxName); - const shouldProbeLiveGateway = current.sandboxes.length > 0 || Boolean(session?.sandboxName); + const shouldProbeLiveGateway = + current.sandboxes.length > 0 || Boolean(session?.sandboxName) || Boolean(requestedSandboxName); const recoveredFromGateway = shouldProbeLiveGateway ? await recoverRegistryFromLiveGateway(seeded.metadataByName) : 0; @@ -760,15 +761,30 @@ async function ensureLiveSandboxOrExit(sandboxName) { process.exit(1); } if (lookup.state === "identity_drift") { - console.error( - ` Sandbox '${sandboxName}' is recorded locally, but the gateway trust material rotated after restart.`, - ); - if (lookup.output) { - console.error(lookup.output); + // Gateway SSH keys rotated after restart — clear stale known_hosts and retry. + console.error(" Gateway SSH identity changed after restart — clearing stale host keys..."); + const knownHostsPath = path.join(os.homedir(), ".ssh", "known_hosts"); + if (fs.existsSync(knownHostsPath)) { + try { + const kh = fs.readFileSync(knownHostsPath, "utf8"); + const cleaned = pruneKnownHostsEntries(kh); + if (cleaned !== kh) fs.writeFileSync(knownHostsPath, cleaned); + } catch { + /* best-effort cleanup */ + } + } + const retry = await getReconciledSandboxGatewayState(sandboxName); + if (retry.state === "present") { + console.error(" ✓ Reconnected after clearing stale SSH host keys."); + return retry; } + // Retry failed — fall through to error console.error( - " Existing sandbox connections cannot be reattached safely after this gateway identity change.", + ` Could not reconnect to sandbox '${sandboxName}' after clearing stale host keys.`, ); + if (retry.output) { + console.error(retry.output); + } console.error( " Recreate this sandbox with `nemoclaw onboard` once the gateway runtime is stable.", ); @@ -2392,7 +2408,7 @@ const [cmd, ...args] = process.argv.slice(2); // command, attempt recovery — the sandbox may still be live with a stale registry. if ( !registry.getSandbox(cmd) && - ["connect", "skill", "shields", "config"].includes(args[0] || "") + ["connect", "skill", "shields", "config", ""].includes(args[0] || "") ) { validateName(cmd, "sandbox name"); await recoverRegistryEntries({ requestedSandboxName: cmd }); diff --git a/test/cli.test.ts b/test/cli.test.ts index c23cc298a1d..6f8f3592f9c 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -1940,7 +1940,9 @@ describe("CLI dispatch", () => { PATH: `${localBin}:${process.env.PATH || ""}`, }); expect(connectResult.code).toBe(1); - expect(connectResult.out.includes("gateway trust material rotated after restart")).toBeTruthy(); + // After the auto-recovery attempt (clear stale host keys + retry), the + // fake openshell still returns the handshake error, so recovery fails. + expect(connectResult.out.includes("Could not reconnect")).toBeTruthy(); expect(connectResult.out.includes("Recreate this sandbox")).toBeTruthy(); }); diff --git a/test/reboot-identity-drift.test.ts b/test/reboot-identity-drift.test.ts new file mode 100644 index 00000000000..7bd09bb49dc --- /dev/null +++ b/test/reboot-identity-drift.test.ts @@ -0,0 +1,320 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Regression tests for SSH identity drift after host reboot. +// Covers: https://github.com/NVIDIA/NemoClaw/issues/2056 +// +// Simulates the post-reboot scenario where the gateway restarts with new SSH +// keys, causing "handshake verification failed" errors. Verifies: +// 1. The registry recovery gate triggers for bare `nemoclaw ` (no action) +// 2. Identity drift is detected and surfaced (current behavior) +// +// Once the fix for #2056 lands, update these tests to assert auto-recovery. + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +const tmpFixtures: string[] = []; + +afterEach(() => { + for (const dir of tmpFixtures.splice(0)) { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +/** + * Build a temp HOME with a NemoClaw registry and a fake openshell binary. + * + * @param sandboxName - name of the sandbox in the registry + * @param mode - "healthy" | "identity_drift" | "gateway_down" + */ +function setupFixture(sandboxName: string, mode: "healthy" | "identity_drift" | "gateway_down") { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-reboot-")); + tmpFixtures.push(tmpDir); + const homeLocalBin = path.join(tmpDir, ".local", "bin"); + const registryDir = path.join(tmpDir, ".nemoclaw"); + const openshellPath = path.join(homeLocalBin, "openshell"); + + fs.mkdirSync(homeLocalBin, { recursive: true }); + fs.mkdirSync(registryDir, { recursive: true }); + + fs.writeFileSync( + path.join(registryDir, "sandboxes.json"), + JSON.stringify({ + defaultSandbox: sandboxName, + sandboxes: { + [sandboxName]: { + name: sandboxName, + model: "nvidia/test-model", + provider: "nvidia-prod", + gpuEnabled: false, + policies: [], + }, + }, + }), + { mode: 0o600 }, + ); + + // Fake openshell binary that simulates post-reboot states + const handshakeError = + "ssh: handshake verification failed — gateway identity has changed since last connection"; + + let sandboxGetBehavior: string; + let statusBehavior: string; + let gatewayInfoBehavior: string; + let gatewayStartBehavior: string; + + switch (mode) { + case "healthy": + statusBehavior = `process.stdout.write("Gateway: nemoclaw\\nStatus: Connected\\n"); process.exit(0);`; + gatewayInfoBehavior = `process.stdout.write("Gateway: nemoclaw\\nGateway endpoint: https://127.0.0.1:8080\\n"); process.exit(0);`; + sandboxGetBehavior = `process.stdout.write("Sandbox:\\n\\n Id: abc\\n Name: ${sandboxName}\\n Phase: Ready\\n"); process.exit(0);`; + gatewayStartBehavior = `process.exit(0);`; + break; + case "identity_drift": + // Gateway is running but SSH keys have changed — sandbox commands fail + statusBehavior = `process.stdout.write("Gateway: nemoclaw\\nStatus: Connected\\n"); process.exit(0);`; + gatewayInfoBehavior = `process.stdout.write("Gateway: nemoclaw\\nGateway endpoint: https://127.0.0.1:8080\\n"); process.exit(0);`; + sandboxGetBehavior = `process.stderr.write("${handshakeError}\\n"); process.exit(1);`; + gatewayStartBehavior = `process.exit(0);`; + break; + case "gateway_down": + // Gateway container is not running — simulates post-reboot before recovery + statusBehavior = `process.stdout.write("No gateway configured\\n"); process.exit(1);`; + gatewayInfoBehavior = `process.stdout.write("No gateway metadata found\\n"); process.exit(1);`; + sandboxGetBehavior = `process.stderr.write("Connection refused\\n"); process.exit(1);`; + gatewayStartBehavior = `process.exit(0);`; + break; + } + + fs.writeFileSync( + openshellPath, + `#!${process.execPath} +const args = process.argv.slice(2); + +if (args[0] === "status") { + ${statusBehavior} +} + +if (args[0] === "gateway" && args[1] === "info") { + ${gatewayInfoBehavior} +} + +if (args[0] === "gateway" && args[1] === "start") { + ${gatewayStartBehavior} +} + +if (args[0] === "gateway" && args[1] === "select") { + process.exit(0); +} + +if (args[0] === "sandbox" && args[1] === "get" && args[2] === ${JSON.stringify(sandboxName)}) { + ${sandboxGetBehavior} +} + +if (args[0] === "sandbox" && args[1] === "list") { + process.stdout.write("${sandboxName}\\n"); + process.exit(0); +} + +if (args[0] === "sandbox" && args[1] === "connect") { + process.exit(0); +} + +if (args[0] === "policy" && args[1] === "get") { + process.exit(1); +} + +if (args[0] === "inference" && args[1] === "get") { + process.stdout.write("Gateway inference:\\n Provider: nvidia-prod\\n Model: nvidia/test-model\\n"); + process.exit(0); +} + +if (args[0] === "forward") { + process.exit(0); +} + +if (args[0] === "logs") { + process.exit(0); +} + +process.exit(0); +`, + { mode: 0o755 }, + ); + + return { tmpDir, sandboxName }; +} + +/** + * Simulates a cleared/corrupt registry (sandbox entry missing) where the + * gateway is still live. The recovery gate should attempt to rebuild the + * registry from the live gateway for both explicit and bare commands. + */ +function setupEmptyRegistry(sandboxName: string) { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-reboot-noreg-")); + tmpFixtures.push(tmpDir); + const homeLocalBin = path.join(tmpDir, ".local", "bin"); + const registryDir = path.join(tmpDir, ".nemoclaw"); + const openshellPath = path.join(homeLocalBin, "openshell"); + + fs.mkdirSync(homeLocalBin, { recursive: true }); + fs.mkdirSync(registryDir, { recursive: true }); + + // Empty registry — no sandboxes known + fs.writeFileSync( + path.join(registryDir, "sandboxes.json"), + JSON.stringify({ defaultSandbox: null, sandboxes: {} }), + { mode: 0o600 }, + ); + + // Fake openshell — gateway is healthy and sandbox is live + fs.writeFileSync( + openshellPath, + `#!${process.execPath} +const args = process.argv.slice(2); + +if (args[0] === "status") { + process.stdout.write("Gateway: nemoclaw\\nStatus: Connected\\n"); + process.exit(0); +} + +if (args[0] === "gateway" && args[1] === "info") { + process.stdout.write("Gateway: nemoclaw\\nGateway endpoint: https://127.0.0.1:8080\\n"); + process.exit(0); +} + +if (args[0] === "gateway" && args[1] === "select") { + process.exit(0); +} + +if (args[0] === "gateway" && args[1] === "start") { + process.exit(0); +} + +if (args[0] === "sandbox" && args[1] === "get" && args[2] === ${JSON.stringify(sandboxName)}) { + process.stdout.write("Sandbox:\\n\\n Id: abc\\n Name: ${sandboxName}\\n Phase: Ready\\n"); + process.exit(0); +} + +if (args[0] === "sandbox" && args[1] === "list") { + process.stdout.write("${sandboxName}\\n"); + process.exit(0); +} + +if (args[0] === "sandbox" && args[1] === "connect") { + process.exit(0); +} + +if (args[0] === "policy" && args[1] === "get") { + process.exit(1); +} + +if (args[0] === "inference" && args[1] === "get") { + process.stdout.write("Gateway inference:\\n Provider: nvidia-prod\\n Model: nvidia/test-model\\n"); + process.exit(0); +} + +if (args[0] === "forward") { + process.exit(0); +} + +if (args[0] === "logs") { + process.exit(0); +} + +process.exit(0); +`, + { mode: 0o755 }, + ); + + return { tmpDir, sandboxName }; +} + +function runCli(tmpDir: string, args: string[]) { + const repoRoot = path.join(import.meta.dirname, ".."); + return spawnSync(process.execPath, [path.join(repoRoot, "bin", "nemoclaw.js"), ...args], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: "/usr/bin:/bin", + NEMOCLAW_NO_CONNECT_HINT: "1", + }, + timeout: Number(process.env.NEMOCLAW_EXEC_TIMEOUT || 15_000), + }); +} + +describe("post-reboot SSH identity drift (#2056)", () => { + it( + "bare `nemoclaw ` (no action) resolves to connect and finds registry entry", + { timeout: Number(process.env.NEMOCLAW_TEST_TIMEOUT || 20_000) }, + () => { + const { tmpDir, sandboxName } = setupFixture("reboot-test", "healthy"); + const result = runCli(tmpDir, [sandboxName]); + expect(result.status).toBe(0); + }, + ); + + it( + "explicit `nemoclaw connect` works for healthy sandbox", + { timeout: Number(process.env.NEMOCLAW_TEST_TIMEOUT || 20_000) }, + () => { + const { tmpDir, sandboxName } = setupFixture("reboot-explicit", "healthy"); + const result = runCli(tmpDir, [sandboxName, "connect"]); + expect(result.status).toBe(0); + }, + ); + + it( + "identity drift is detected when gateway SSH keys have changed", + { timeout: Number(process.env.NEMOCLAW_TEST_TIMEOUT || 20_000) }, + () => { + const { tmpDir, sandboxName } = setupFixture("drift-sandbox", "identity_drift"); + const result = runCli(tmpDir, [sandboxName, "connect"]); + const combined = (result.stdout || "") + (result.stderr || ""); + expect(combined).toMatch(/identity|handshake|drift|changed/i); + expect(result.status).not.toBe(0); + }, + ); + + it( + "bare `nemoclaw ` also detects identity drift (not silent failure)", + { timeout: Number(process.env.NEMOCLAW_TEST_TIMEOUT || 20_000) }, + () => { + const { tmpDir, sandboxName } = setupFixture("drift-bare", "identity_drift"); + const result = runCli(tmpDir, [sandboxName]); + const combined = (result.stdout || "") + (result.stderr || ""); + expect(combined).not.toMatch(/unknown command/i); + expect(result.status).not.toBe(0); + }, + ); +}); + +describe("post-reboot registry recovery gate (#2056)", () => { + it( + "explicit `nemoclaw connect` recovers registry from live gateway", + { timeout: Number(process.env.NEMOCLAW_TEST_TIMEOUT || 20_000) }, + () => { + const { tmpDir, sandboxName } = setupEmptyRegistry("orphan-explicit"); + const result = runCli(tmpDir, [sandboxName, "connect"]); + expect(result.status).toBe(0); + }, + ); + + it( + "bare `nemoclaw ` recovers registry from live gateway", + { timeout: Number(process.env.NEMOCLAW_TEST_TIMEOUT || 20_000) }, + () => { + const { tmpDir, sandboxName } = setupEmptyRegistry("orphan-bare"); + const result = runCli(tmpDir, [sandboxName]); + const combined = (result.stdout || "") + (result.stderr || ""); + expect(combined).not.toMatch(/unknown command/i); + expect(result.status).toBe(0); + }, + ); +});