From 0efbd66bdbf7ef49e8ff8d271b70351a8b3cd43b Mon Sep 17 00:00:00 2001 From: Kevin Jones Date: Wed, 1 Apr 2026 16:14:41 -0400 Subject: [PATCH 1/3] fix: harden spark startup and destroy handling --- bin/nemoclaw.js | 33 ++++++++- scripts/nemoclaw-start.sh | 36 ++++++++-- test/cli.test.js | 130 ++++++++++++++++++++++++++++++++++++ test/nemoclaw-start.test.js | 8 +++ 4 files changed, 201 insertions(+), 6 deletions(-) diff --git a/bin/nemoclaw.js b/bin/nemoclaw.js index e45c80cc92..768fc714ef 100755 --- a/bin/nemoclaw.js +++ b/bin/nemoclaw.js @@ -128,6 +128,20 @@ function hasNoLiveSandboxes() { return parseLiveSandboxNames(liveList.output).size === 0; } +function isMissingSandboxDeleteResult(output = "") { + return /sandbox .* not found|sandbox .* not present|sandbox does not exist|no such sandbox/i.test( + stripAnsi(output), + ); +} + +function getSandboxDeleteOutcome(deleteResult) { + const output = `${deleteResult.stdout || ""}${deleteResult.stderr || ""}`.trim(); + return { + output, + alreadyGone: deleteResult.status !== 0 && isMissingSandboxDeleteResult(output), + }; +} + function parseVersionFromText(value = "") { const match = String(value || "").match(/([0-9]+\.[0-9]+\.[0-9]+)/); return match ? match[1] : null; @@ -1107,17 +1121,32 @@ async function sandboxDestroy(sandboxName, args = []) { else nim.stopNimContainer(sandboxName); console.log(` Deleting sandbox '${sandboxName}'...`); - const deleteResult = runOpenshell(["sandbox", "delete", sandboxName], { ignoreError: true }); + const deleteResult = runOpenshell(["sandbox", "delete", sandboxName], { + ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + }); + const { output: deleteOutput, alreadyGone } = getSandboxDeleteOutcome(deleteResult); + + if (deleteResult.status !== 0 && !alreadyGone) { + if (deleteOutput) { + console.error(` ${deleteOutput}`); + } + console.error(` Failed to destroy sandbox '${sandboxName}'.`); + process.exit(deleteResult.status || 1); + } const removed = registry.removeSandbox(sandboxName); if ( - deleteResult.status === 0 && + (deleteResult.status === 0 || alreadyGone) && removed && registry.listSandboxes().sandboxes.length === 0 && hasNoLiveSandboxes() ) { cleanupGatewayAfterLastSandbox(); } + if (alreadyGone) { + console.log(` Sandbox '${sandboxName}' was already absent from the live gateway.`); + } console.log(` ${G}✓${R} Sandbox '${sandboxName}' destroyed`); } diff --git a/scripts/nemoclaw-start.sh b/scripts/nemoclaw-start.sh index fbdebda209..bdb29903f3 100755 --- a/scripts/nemoclaw-start.sh +++ b/scripts/nemoclaw-start.sh @@ -54,10 +54,38 @@ elif [ "${NEMOCLAW_CAPS_DROPPED:-}" != "1" ]; then echo "[SECURITY WARNING] capsh not available — running with default capabilities" >&2 fi -# Filter out self-invocation: openshell sandbox create passes "nemoclaw-start" -# as the command, but since this script is now the ENTRYPOINT, receiving our -# own name as $1 would cause infinite recursion via the NEMOCLAW_CMD exec path. -# Only strip from $1 — later args with this name are legitimate user arguments. +# Normalize the sandbox-create bootstrap wrapper. Onboard launches the +# container as `env CHAT_UI_URL=... nemoclaw-start`, but this script is already +# the ENTRYPOINT. If we treat that wrapper as a real command, the root path will +# try `gosu sandbox env ... nemoclaw-start`, which fails on Spark/arm64 when +# no-new-privileges blocks gosu. Consume only the self-wrapper form and promote +# the env assignments into the current process. +if [ "${1:-}" = "env" ]; then + _raw_args=("$@") + _self_wrapper_index="" + for ((i = 1; i < ${#_raw_args[@]}; i += 1)); do + case "${_raw_args[$i]}" in + *=*) ;; + nemoclaw-start | /usr/local/bin/nemoclaw-start) + _self_wrapper_index="$i" + break + ;; + *) + break + ;; + esac + done + if [ -n "$_self_wrapper_index" ]; then + for ((i = 1; i < _self_wrapper_index; i += 1)); do + export "${_raw_args[$i]}" + done + set -- "${_raw_args[@]:$((_self_wrapper_index + 1))}" + fi +fi + +# Filter out direct self-invocation too. Since this script is the ENTRYPOINT, +# receiving our own name as $1 would otherwise recurse via the NEMOCLAW_CMD +# exec path. Only strip from $1 — later args with this name are legitimate. case "${1:-}" in nemoclaw-start | /usr/local/bin/nemoclaw-start) shift ;; esac diff --git a/test/cli.test.js b/test/cli.test.js index 241e1d3bd8..08311f28ea 100644 --- a/test/cli.test.js +++ b/test/cli.test.js @@ -398,6 +398,136 @@ describe("CLI dispatch", () => { } }); + it("fails destroy when openshell sandbox delete returns a real error", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-destroy-failure-")); + const localBin = path.join(home, "bin"); + const registryDir = path.join(home, ".nemoclaw"); + const openshellLog = path.join(home, "openshell.log"); + fs.mkdirSync(localBin, { recursive: true }); + fs.mkdirSync(registryDir, { recursive: true }); + fs.writeFileSync( + path.join(registryDir, "sandboxes.json"), + JSON.stringify({ + sandboxes: { + alpha: { + name: "alpha", + model: "test-model", + provider: "nvidia-prod", + gpuEnabled: false, + policies: [], + }, + }, + defaultSandbox: "alpha", + }), + { mode: 0o600 }, + ); + fs.writeFileSync( + path.join(localBin, "openshell"), + [ + "#!/bin/sh", + `log_file=${JSON.stringify(openshellLog)}`, + 'printf \'%s\\n\' "$*" >> "$log_file"', + 'if [ "$1" = "sandbox" ] && [ "$2" = "delete" ]; then', + ' echo "transport error: gateway unavailable" >&2', + " exit 1", + "fi", + "exit 0", + ].join("\n"), + { mode: 0o755 }, + ); + + const r = runWithEnv("alpha destroy --yes", { + HOME: home, + PATH: `${localBin}:${process.env.PATH || ""}`, + }); + + expect(r.code).toBe(1); + expect(r.out).toContain("transport error: gateway unavailable"); + expect(r.out).toContain("Failed to destroy sandbox 'alpha'."); + expect(r.out).not.toContain("Sandbox 'alpha' destroyed"); + + const registryAfter = JSON.parse( + fs.readFileSync(path.join(registryDir, "sandboxes.json"), "utf8"), + ); + expect(registryAfter.sandboxes.alpha).toBeTruthy(); + expect(fs.readFileSync(openshellLog, "utf8")).toContain("sandbox delete alpha"); + expect(fs.readFileSync(openshellLog, "utf8")).not.toContain("gateway destroy -g nemoclaw"); + }); + + it("treats an already-missing sandbox as destroyed and clears the stale registry entry", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-destroy-missing-")); + const localBin = path.join(home, "bin"); + const registryDir = path.join(home, ".nemoclaw"); + const openshellLog = path.join(home, "openshell.log"); + const bashLog = path.join(home, "bash.log"); + fs.mkdirSync(localBin, { recursive: true }); + fs.mkdirSync(registryDir, { recursive: true }); + fs.writeFileSync( + path.join(registryDir, "sandboxes.json"), + JSON.stringify({ + sandboxes: { + alpha: { + name: "alpha", + model: "test-model", + provider: "nvidia-prod", + gpuEnabled: false, + policies: [], + }, + }, + defaultSandbox: "alpha", + }), + { mode: 0o600 }, + ); + fs.writeFileSync( + path.join(localBin, "openshell"), + [ + "#!/bin/sh", + `log_file=${JSON.stringify(openshellLog)}`, + 'if [ "$1" = "sandbox" ] && [ "$2" = "delete" ]; then', + ' printf \'%s\\n\' "$*" >> "$log_file"', + ' echo "sandbox alpha not found" >&2', + " exit 1", + "fi", + 'if [ "$1" = "sandbox" ] && [ "$2" = "list" ]; then', + ' printf "NAME STATUS\\n" >> "$log_file"', + ' printf "NAME STATUS\\n"', + " exit 0", + "fi", + 'printf \'%s\\n\' "$*" >> "$log_file"', + "exit 0", + ].join("\n"), + { mode: 0o755 }, + ); + fs.writeFileSync( + path.join(localBin, "bash"), + [ + "#!/bin/sh", + `log_file=${JSON.stringify(bashLog)}`, + 'printf \'%s\\n\' "$*" >> "$log_file"', + "exit 0", + ].join("\n"), + { mode: 0o755 }, + ); + + const r = runWithEnv("alpha destroy --yes", { + HOME: home, + PATH: `${localBin}:${process.env.PATH || ""}`, + }); + + expect(r.code).toBe(0); + expect(r.out).toContain("already absent from the live gateway"); + expect(r.out).toContain("Sandbox 'alpha' destroyed"); + + const registryAfter = JSON.parse( + fs.readFileSync(path.join(registryDir, "sandboxes.json"), "utf8"), + ); + expect(registryAfter.sandboxes.alpha).toBeFalsy(); + expect(fs.readFileSync(openshellLog, "utf8")).toContain("sandbox delete alpha"); + expect(fs.readFileSync(openshellLog, "utf8")).toContain("forward stop 18789"); + expect(fs.readFileSync(openshellLog, "utf8")).toContain("gateway destroy -g nemoclaw"); + expect(fs.readFileSync(bashLog, "utf8")).toContain("docker volume ls -q --filter"); + }); + it("passes plain logs through without the tail flag", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-logs-plain-")); const localBin = path.join(home, "bin"); diff --git a/test/nemoclaw-start.test.js b/test/nemoclaw-start.test.js index d54ac97096..ba69acd64b 100644 --- a/test/nemoclaw-start.test.js +++ b/test/nemoclaw-start.test.js @@ -58,4 +58,12 @@ describe("nemoclaw-start non-root fallback", () => { expect(line).toContain(">&2"); } }); + + it("unwraps the sandbox-create env self-wrapper before building NEMOCLAW_CMD", () => { + const src = fs.readFileSync(START_SCRIPT, "utf-8"); + + expect(src).toContain('if [ "${1:-}" = "env" ]; then'); + expect(src).toContain('export "${_raw_args[$i]}"'); + expect(src).toContain('set -- "${_raw_args[@]:$((_self_wrapper_index + 1))}"'); + }); }); From 3f698e785842689706bdbf57e4c1c5531a05c312 Mon Sep 17 00:00:00 2001 From: Kevin Jones Date: Wed, 1 Apr 2026 17:13:44 -0400 Subject: [PATCH 2/3] fix: align destroy missing-sandbox matching --- bin/nemoclaw.js | 2 +- test/cli.test.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/nemoclaw.js b/bin/nemoclaw.js index b8ce91c353..ad47dbe267 100755 --- a/bin/nemoclaw.js +++ b/bin/nemoclaw.js @@ -129,7 +129,7 @@ function hasNoLiveSandboxes() { } function isMissingSandboxDeleteResult(output = "") { - return /sandbox .* not found|sandbox .* not present|sandbox does not exist|no such sandbox/i.test( + return /NotFound|sandbox not found|sandbox .* not found|sandbox .* not present|sandbox does not exist|no such sandbox/i.test( stripAnsi(output), ); } diff --git a/test/cli.test.js b/test/cli.test.js index d3111b95e7..b36f3d22e8 100644 --- a/test/cli.test.js +++ b/test/cli.test.js @@ -499,7 +499,7 @@ describe("CLI dispatch", () => { `log_file=${JSON.stringify(openshellLog)}`, 'if [ "$1" = "sandbox" ] && [ "$2" = "delete" ]; then', ' printf \'%s\\n\' "$*" >> "$log_file"', - ' echo "sandbox alpha not found" >&2', + ' echo "Error: status: NotFound, message: \\"sandbox not found\\"" >&2', " exit 1", "fi", 'if [ "$1" = "sandbox" ] && [ "$2" = "list" ]; then', From 84f8dcf5bdbfe90e6ca34e50b6e6b41513a2a3af Mon Sep 17 00:00:00 2001 From: Kevin Jones Date: Wed, 1 Apr 2026 19:47:27 -0400 Subject: [PATCH 3/3] fix: match spaced not-found sandbox errors --- bin/nemoclaw.js | 4 ++-- test/cli.test.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/nemoclaw.js b/bin/nemoclaw.js index ad47dbe267..ba21a2aa40 100755 --- a/bin/nemoclaw.js +++ b/bin/nemoclaw.js @@ -129,7 +129,7 @@ function hasNoLiveSandboxes() { } function isMissingSandboxDeleteResult(output = "") { - return /NotFound|sandbox not found|sandbox .* not found|sandbox .* not present|sandbox does not exist|no such sandbox/i.test( + return /\bNotFound\b|\bNot Found\b|sandbox not found|sandbox .* not found|sandbox .* not present|sandbox does not exist|no such sandbox/i.test( stripAnsi(output), ); } @@ -390,7 +390,7 @@ function getSandboxGatewayState(sandboxName) { if (result.status === 0) { return { state: "present", output }; } - if (/NotFound|sandbox not found/i.test(output)) { + if (/\bNotFound\b|\bNot Found\b|sandbox not found/i.test(output)) { return { state: "missing", output }; } if ( diff --git a/test/cli.test.js b/test/cli.test.js index b36f3d22e8..0e6fc6645a 100644 --- a/test/cli.test.js +++ b/test/cli.test.js @@ -499,7 +499,7 @@ describe("CLI dispatch", () => { `log_file=${JSON.stringify(openshellLog)}`, 'if [ "$1" = "sandbox" ] && [ "$2" = "delete" ]; then', ' printf \'%s\\n\' "$*" >> "$log_file"', - ' echo "Error: status: NotFound, message: \\"sandbox not found\\"" >&2', + ' echo "Error: status: Not Found, message: \\"sandbox not found\\"" >&2', " exit 1", "fi", 'if [ "$1" = "sandbox" ] && [ "$2" = "list" ]; then',