From 678ff2ebe730293d2bd0cde6416c0240b8ca538d Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Fri, 14 Aug 2026 13:31:09 +0000 Subject: [PATCH 1/5] fix(gateway): name a gateway start command that exists Signed-off-by: Tinson Lai --- ci/source-architecture-budget.json | 12 +- docs/deployment/deploy-to-headless-server.mdx | 2 +- .../recover-rebuild-sandboxes.mdx | 5 +- docs/reference/commands.mdx | 3 +- docs/reference/troubleshooting.mdx | 2 +- src/lib/actions/credentials/list.ts | 3 +- src/lib/actions/sandbox/connect.ts | 8 +- .../actions/sandbox/doctor-system-checks.ts | 5 +- .../sandbox/gateway-state-hints.test.ts | 30 ++- src/lib/actions/sandbox/gateway-state.ts | 15 +- src/lib/actions/sandbox/policy-channel.ts | 9 +- src/lib/actions/sandbox/status-flow.test.ts | 2 +- .../sandbox/status-lookup-rendering.ts | 7 +- src/lib/credentials/command-support.ts | 3 +- src/lib/gateway-start-guidance.test.ts | 39 ++++ src/lib/gateway-start-guidance.ts | 47 +++++ src/lib/inventory/index.ts | 5 +- src/lib/onboard.ts | 198 +++--------------- src/lib/onboard/docker-cdi.ts | 4 +- src/lib/onboard/gateway-recovery.test.ts | 37 +--- src/lib/onboard/gateway-recovery.ts | 49 ++--- .../gateway-start-failure-integration.test.ts | 50 ++--- src/lib/onboard/gateway-start-failure.test.ts | 19 -- src/lib/onboard/gateway-start-failure.ts | 17 -- src/lib/onboard/gateway.test.ts | 69 ------ src/lib/onboard/gateway.ts | 170 --------------- src/lib/onboard/machine/handlers/gateway.ts | 4 +- src/lib/onboard/preflight-gateway-reuse.ts | 6 +- src/lib/validation.ts | 4 +- .../cli/credentials-cli-command.test.ts | 2 +- test/sandbox-status-json-stdout.test.ts | 4 +- 31 files changed, 232 insertions(+), 598 deletions(-) create mode 100644 src/lib/gateway-start-guidance.test.ts create mode 100644 src/lib/gateway-start-guidance.ts delete mode 100644 src/lib/onboard/gateway.test.ts delete mode 100644 src/lib/onboard/gateway.ts diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 65f8265f9bd..84a816aa564 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -11,7 +11,7 @@ "src/lib/adapters/openshell/runtime.ts": 54, "src/lib/adapters/openshell/timeouts.ts": 38, "src/lib/agent/defs.ts": 33, - "src/lib/cli/branding.ts": 86, + "src/lib/cli/branding.ts": 87, "src/lib/cli/nemoclaw-oclif-command.ts": 106, "src/lib/cli/terminal-style.ts": 43, "src/lib/core/json-types.ts": 37, @@ -24,24 +24,24 @@ "src/lib/inference/web-search.ts": 21, "src/lib/messaging/channels/index.ts": 25, "src/lib/onboard/gateway-binding.ts": 51, - "src/lib/runner.ts": 88, + "src/lib/runner.ts": 87, "src/lib/security/redact.ts": 53, "src/lib/state/onboard-session.ts": 36, "src/lib/state/registry.ts": 101, "src/lib/state/state-root.ts": 21, "src/lib/subprocess-env.ts": 24, - "src/lib/validation.ts": 25 + "src/lib/validation.ts": 24 } }, "fanOut": { "defaultMax": 20, "maxByFile": { "src/lib/actions/inference-set.ts": 32, - "src/lib/actions/sandbox/connect.ts": 39, + "src/lib/actions/sandbox/connect.ts": 40, "src/lib/actions/sandbox/destroy.ts": 29, "src/lib/actions/sandbox/doctor.ts": 30, "src/lib/actions/sandbox/status-snapshot.ts": 20, - "src/lib/actions/sandbox/policy-channel.ts": 29, + "src/lib/actions/sandbox/policy-channel.ts": 30, "src/lib/actions/sandbox/process-recovery.ts": 21, "src/lib/actions/sandbox/rebuild-pipeline.ts": 28, "src/lib/actions/sandbox/snapshot.ts": 40, @@ -56,7 +56,7 @@ }, "allowedCycles": [], "maxRootFiles": { - "src/lib/onboard": 308, + "src/lib/onboard": 307, "src/lib/actions": 19, "src/lib/actions/sandbox": 183, "src/lib/state": 38, diff --git a/docs/deployment/deploy-to-headless-server.mdx b/docs/deployment/deploy-to-headless-server.mdx index ea345ee132d..7be308851fb 100644 --- a/docs/deployment/deploy-to-headless-server.mdx +++ b/docs/deployment/deploy-to-headless-server.mdx @@ -455,7 +455,7 @@ $$nemoclaw headless-agent recover ``` `recover`, `start`, and `connect --probe-only` do not restart the shared host OpenShell gateway. -If they report a host gateway RPC error, follow the printed `openshell status`, named gateway start, or onboarding guidance. +If they report a host gateway RPC error, follow the printed host gateway recovery guidance. diff --git a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx index 6a6d6aa7284..fb7ac992419 100644 --- a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx +++ b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx @@ -206,7 +206,10 @@ Deployment ordering remains responsible for external changes that OpenShell and These commands do not restart, replace, or reap the shared host OpenShell gateway process. If the host gateway RPC returns an error while these sandbox-scoped commands run, they surface the error with explicit next-step guidance and exit rather than swapping the shared gateway out from under other sandboxes. -To repair the host gateway itself, use `openshell status`, `openshell gateway start --name `, or run `$$nemoclaw onboard` again. +The OpenShell CLI has no command that starts a gateway, and `openshell status` only reports the gateway state. +If NemoClaw starts the gateway on your host, run `$$nemoclaw onboard` again to repair the host gateway itself. +If a deployment outside NemoClaw owns the gateway process, start the gateway with that deployment, then run `openshell gateway select `. +NemoClaw names the applicable command when a sandbox-scoped command reports that the host gateway is down. Other workflows, including onboarding, rebuild, and `doctor --fix`, can explicitly recover the named host gateway when their operation requires it. `$$nemoclaw gateway restart` instead restarts only the supported agent gateway inside the named sandbox. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index ac710dc11da..1557929280a 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -3941,7 +3941,8 @@ $$nemoclaw status --json When at least one sandbox is registered and the named NemoClaw gateway is unreachable, unhealthy, or attached to a different sandbox, the command prints a `gateway: down [state] (reason)` line between the sandbox list and the host-service list. The command classifies the failing layer when possible: the named gateway port is not accepting connections, the named gateway is running but not Connected, the active OpenShell gateway points at a different name, or the named gateway is not configured at all. -It then suggests `$$nemoclaw onboard --resume` or equivalent managed-gateway recovery guidance. +It then prints the gateway start guidance for your host. +That guidance names `$$nemoclaw onboard` where NemoClaw starts the gateway, and the owning deployment plus `openshell gateway select ` where it does not. It exits with code `1` so shell scripts and CI can detect the degraded state from `$?`. For `--json`, the structured output includes `gatewayHealth`, and the exit code is set after the report is generated. A clean machine with no registered sandboxes keeps the legacy `0` exit because no gateway is expected to be configured yet. diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 0c53b560624..3415beabbe2 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2979,7 +2979,7 @@ When shared gateway cleanup would be unsafe, follow the targeted destroy or gate ### `unresolvable CDI devices nvidia.com/gpu=all` during gateway start -Recent NVIDIA Container Toolkit installs configure the Docker daemon for Container Device Interface (CDI) device injection, which OpenShell's `gateway start --gpu` then auto-selects. +Recent NVIDIA Container Toolkit installs configure the Docker daemon for Container Device Interface (CDI) device injection, which a GPU-enabled gateway start then auto-selects. If no `nvidia.com/gpu` CDI spec has been generated on the host yet, gateway start fails with `Docker responded with status code 500: CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all`. Outside Station Express, the standard NemoClaw installer detects this gap before onboarding, first tries to enable the NVIDIA CDI refresh systemd units, and can fall back to generating the spec directly with `nvidia-ctk`. Station Express never falls back to direct CDI generation. diff --git a/src/lib/actions/credentials/list.ts b/src/lib/actions/credentials/list.ts index 2a728dabbd4..2a77365f701 100644 --- a/src/lib/actions/credentials/list.ts +++ b/src/lib/actions/credentials/list.ts @@ -6,6 +6,7 @@ import { OPENSHELL_OPERATION_TIMEOUT_MS } from "../../adapters/openshell/timeout import { CLI_NAME } from "../../cli/branding"; import { recoverGatewayOrExit } from "../../credentials/command-support"; import { parseGatewayProviderNames } from "../../credentials/provider-list"; +import { gatewayStartGuidance } from "../../gateway-start-guidance"; export type CredentialsListResult = { exitCode: number; @@ -32,7 +33,7 @@ export async function runCredentialsListAction(): Promise if (result.status !== 0) { return fail([ " Could not query OpenShell gateway. Is it running?", - ` Run 'openshell gateway start --name nemoclaw' or '${CLI_NAME} onboard' first.`, + ` ${gatewayStartGuidance()}`, ]); } diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index 4909092860c..326fe0c6c4e 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -20,6 +20,7 @@ import { D, G, R, YW } from "../../cli/terminal-style"; import { spawnExitCode } from "../../core/process-exit"; import { shellQuote } from "../../core/shell-quote"; import { getNamedGatewayLifecycleState } from "../../gateway-runtime-action"; +import { gatewayStartGuidance } from "../../gateway-start-guidance"; import { formatInferenceRouteDriftForDisplay, parseGatewayInference, @@ -382,11 +383,8 @@ function failConnectReadinessGatewayUnavailable(sandboxName: string, detailOutpu printGatewayLifecycleHint(detailOutput, sandboxName, console.error); } console.error(" Recovery:"); - console.error( - ` 1. Run: openshell gateway start --name ${getSandboxTargetGatewayName(sandboxName)}`, - ); - console.error(` 2. If the gateway cannot be restarted, run: ${CLI_NAME} onboard`); - console.error(` 3. Retry: ${CLI_NAME} ${sandboxName} connect`); + console.error(` 1. ${gatewayStartGuidance(getSandboxTargetGatewayName(sandboxName))}`); + console.error(` 2. Retry: ${CLI_NAME} ${sandboxName} connect`); process.exit(1); } diff --git a/src/lib/actions/sandbox/doctor-system-checks.ts b/src/lib/actions/sandbox/doctor-system-checks.ts index 14843f2c385..814c90fedd0 100644 --- a/src/lib/actions/sandbox/doctor-system-checks.ts +++ b/src/lib/actions/sandbox/doctor-system-checks.ts @@ -5,6 +5,7 @@ import path from "node:path"; import { buildValidatedCurlCommandArgs } from "../../adapters/http/curl-args"; import { stripAnsi } from "../../adapters/openshell/client"; import { CLI_NAME } from "../../cli/branding"; +import { gatewayStartGuidance } from "../../gateway-start-guidance"; import { GATEWAY_PORT, OLLAMA_PORT } from "../../core/ports"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES, @@ -39,9 +40,7 @@ function gatewayContainerCheck( label: "Docker container", status: running && healthy ? "ok" : "fail", detail: `${containerName} ${running ? "running" : "stopped"} (${health}; ${image})`, - hint: running - ? undefined - : `restart the gateway with \`openshell gateway start --name ${options.gatewayName ?? "nemoclaw"}\``, + hint: running ? undefined : gatewayStartGuidance(options.gatewayName ?? "nemoclaw"), }; } diff --git a/src/lib/actions/sandbox/gateway-state-hints.test.ts b/src/lib/actions/sandbox/gateway-state-hints.test.ts index 20da18b7412..3c24ff9f729 100644 --- a/src/lib/actions/sandbox/gateway-state-hints.test.ts +++ b/src/lib/actions/sandbox/gateway-state-hints.test.ts @@ -100,7 +100,8 @@ describe("printGatewayLifecycleHint multi-instance hints", () => { const combined = lines.join("\n"); expect(combined).not.toContain("sandbox has no spec"); - expect(combined).toContain("openshell gateway start"); + expect(combined).toContain("no longer configured or its metadata/runtime has been lost"); + expect(combined).toContain("Start the gateway again with `nemoclaw onboard`."); }); it.each([ @@ -385,4 +386,31 @@ describe("printGatewayLifecycleHint multi-instance hints", () => { errorSpy.mockRestore(); exitSpy.mockRestore(); }); + + it("names a command that exists when a sandbox-scoped command observes a stopped gateway", async () => { + captureOpenshellSpy.mockReturnValue({ + status: 1, + output: "transport error\ntcp connect error\nConnection refused (os error 61)", + }); + const lines: string[] = []; + const errorSpy = vi.spyOn(console, "error").mockImplementation((line = "") => { + lines.push(String(line)); + }); + const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new Error(`process.exit(${code ?? 0})`); + }) as never); + + await expect( + gatewayState.ensureLiveSandboxOrExit("instance-a", { gatewayRecovery: "observe" }), + ).rejects.toThrow("process.exit(1)"); + + const output = lines.join("\n"); + expect(output).toContain("This sandbox-scoped command will not restart the shared host gateway"); + expect(output).toContain("Start the gateway again with `nemoclaw onboard`."); + expect(output).not.toContain("openshell gateway start"); + expect(recoverNamedGatewayRuntimeSpy).not.toHaveBeenCalled(); + expect(exitSpy).toHaveBeenCalledWith(1); + errorSpy.mockRestore(); + exitSpy.mockRestore(); + }); }); diff --git a/src/lib/actions/sandbox/gateway-state.ts b/src/lib/actions/sandbox/gateway-state.ts index 9df1b4281b0..2dd93faacff 100644 --- a/src/lib/actions/sandbox/gateway-state.ts +++ b/src/lib/actions/sandbox/gateway-state.ts @@ -10,6 +10,7 @@ import { getNamedGatewayLifecycleState, recoverNamedGatewayRuntime, } from "../../gateway-runtime-action"; +import { gatewayStartGuidance } from "../../gateway-start-guidance"; import { assertNoOpenShellGatewayEndpointOverride } from "../../openshell-gateway-endpoint-guard"; import { isTerminalSandboxPhase, parseSandboxPhase } from "../../state/gateway"; import { selectSandboxOwningGateway } from "./gateway-select"; @@ -479,9 +480,7 @@ export function printGatewayLifecycleHint( writer( ` The selected ${CLI_DISPLAY_NAME} gateway is no longer configured or its metadata/runtime has been lost.`, ); - writer( - ` Start the gateway again with \`openshell gateway start --name ${targetGatewayName}\` before expecting existing sandboxes to reconnect.`, - ); + writer(` ${gatewayStartGuidance(targetGatewayName)}`); writer( " If the gateway has to be rebuilt from scratch, recreate the affected sandbox afterward.", ); @@ -496,7 +495,7 @@ export function printGatewayLifecycleHint( ); writer(" This usually means the gateway runtime did not come back cleanly after the restart."); writer( - ` Retry \`openshell gateway start --name ${targetGatewayName}\`; if it stays in this state, rebuild the gateway before expecting existing sandboxes to reconnect.`, + ` ${gatewayStartGuidance(targetGatewayName)} If the gateway stays in this state, rebuild it before expecting existing sandboxes to reconnect.`, ); return; } @@ -783,7 +782,7 @@ export async function ensureLiveSandboxOrExit( console.error(lookup.output); } console.error( - ` Retry \`openshell gateway start --name ${getSandboxTargetGatewayName(sandboxName)}\` and verify \`openshell status\` is healthy before reconnecting.`, + ` ${gatewayStartGuidance(getSandboxTargetGatewayName(sandboxName))} Check that \`openshell status\` reports the gateway healthy before reconnecting.`, ); console.error( " If the gateway never becomes healthy, rebuild the gateway and then recreate the affected sandbox.", @@ -799,7 +798,7 @@ export async function ensureLiveSandboxOrExit( } printGatewayLifecycleHint(lookup.output, sandboxName); console.error( - ` This sandbox-scoped command will not restart the shared host gateway. Run \`openshell status\` and \`openshell gateway start --name ${getSandboxTargetGatewayName(sandboxName)}\` before retrying.`, + ` This sandbox-scoped command will not restart the shared host gateway. ${gatewayStartGuidance(getSandboxTargetGatewayName(sandboxName))} Then retry this command.`, ); process.exit(1); } @@ -810,9 +809,7 @@ export async function ensureLiveSandboxOrExit( if (lookup.output) { console.error(lookup.output); } - console.error( - ` Start the gateway again with \`openshell gateway start --name ${getSandboxTargetGatewayName(sandboxName)}\` before retrying.`, - ); + console.error(` ${gatewayStartGuidance(getSandboxTargetGatewayName(sandboxName))}`); console.error( " If the gateway had to be rebuilt from scratch, recreate the affected sandbox afterward.", ); diff --git a/src/lib/actions/sandbox/policy-channel.ts b/src/lib/actions/sandbox/policy-channel.ts index 8194e1b42c4..36904758281 100644 --- a/src/lib/actions/sandbox/policy-channel.ts +++ b/src/lib/actions/sandbox/policy-channel.ts @@ -19,6 +19,7 @@ import { parsePolicyAddOptions, } from "../../domain/policy-channel"; import { recoverNamedGatewayRuntime } from "../../gateway-runtime-action"; +import { gatewayStartGuidance } from "../../gateway-start-guidance"; import { type ChannelManifest, createBuiltInChannelManifestRegistry, @@ -886,8 +887,8 @@ async function applyChannelAddToGatewayAndRegistry( console.error( ` Could not reach the ${CLI_DISPLAY_NAME} OpenShell gateway. Tokens were staged`, ); - console.error(" in env for this run only — re-run after starting the gateway, or run"); - console.error(` 'openshell gateway start --name ${gatewayName}' manually.`); + console.error(" in env for this run only. Rerun after starting the gateway."); + console.error(` ${gatewayStartGuidance(gatewayName)}`); process.exit(1); } try { @@ -946,9 +947,7 @@ async function applyChannelRemoveToGatewayAndRegistry( console.error( ` Could not reach the ${CLI_DISPLAY_NAME} OpenShell gateway to delete the bridge.`, ); - console.error( - ` Re-run after starting the gateway, or run 'openshell gateway start --name ${gatewayName}'.`, - ); + console.error(` ${gatewayStartGuidance(gatewayName)} Then rerun this command.`); if (!bestEffort) process.exit(1); gatewayReachable = false; residual.push("gateway-providers"); diff --git a/src/lib/actions/sandbox/status-flow.test.ts b/src/lib/actions/sandbox/status-flow.test.ts index 14347d87b6a..3ff698f91da 100644 --- a/src/lib/actions/sandbox/status-flow.test.ts +++ b/src/lib/actions/sandbox/status-flow.test.ts @@ -548,7 +548,7 @@ describe("showSandboxStatus flow", () => { expect(output).not.toContain("Inference: healthy"); expect(output).toContain("Inference: not verified (gateway/sandbox state not verified)"); expect(output).toContain("gateway is still refusing connections after restart"); - expect(output).toContain("Retry `openshell gateway start --name nemoclaw`"); + expect(output).toContain("Start the gateway again with `nemoclaw onboard`."); expect(output).toContain("If the gateway never becomes healthy"); expect(harness.collectSandboxStatusSnapshotSpy).toHaveBeenCalledWith("alpha", { preflight: { diff --git a/src/lib/actions/sandbox/status-lookup-rendering.ts b/src/lib/actions/sandbox/status-lookup-rendering.ts index 910c1510d78..7b4c929ce0a 100644 --- a/src/lib/actions/sandbox/status-lookup-rendering.ts +++ b/src/lib/actions/sandbox/status-lookup-rendering.ts @@ -3,6 +3,7 @@ import { CLI_DISPLAY_NAME, CLI_NAME } from "../../cli/branding"; import { D, R } from "../../cli/terminal-style"; +import { gatewayStartGuidance } from "../../gateway-start-guidance"; import { isTerminalSandboxPhase } from "../../state/gateway"; import { getSandboxDockerRuntime } from "./docker-health"; import { isDockerRuntimeDown, printDockerRuntimeDownGuidance } from "./gateway-failure-classifier"; @@ -170,7 +171,7 @@ async function printGatewayUnreachableAfterRestartLookupStatus({ console.log(lookup.output); } console.log( - ` Retry \`openshell gateway start --name ${getSandboxTargetGatewayName(sandboxName)}\` and verify \`openshell status\` is healthy before reconnecting.`, + ` ${gatewayStartGuidance(getSandboxTargetGatewayName(sandboxName))} Check that \`openshell status\` reports the gateway healthy before reconnecting.`, ); console.log( " If the gateway never becomes healthy, rebuild the gateway and then recreate the affected sandbox.", @@ -191,9 +192,7 @@ async function printGatewayMissingAfterRestartLookupStatus({ if (lookup.output) { console.log(lookup.output); } - console.log( - ` Start the gateway again with \`openshell gateway start --name ${getSandboxTargetGatewayName(sandboxName)}\` before retrying.`, - ); + console.log(` ${gatewayStartGuidance(getSandboxTargetGatewayName(sandboxName))}`); console.log( " If the gateway had to be rebuilt from scratch, recreate the affected sandbox afterward.", ); diff --git a/src/lib/credentials/command-support.ts b/src/lib/credentials/command-support.ts index afbaf52d41b..ca56ce08ccf 100644 --- a/src/lib/credentials/command-support.ts +++ b/src/lib/credentials/command-support.ts @@ -4,6 +4,7 @@ import { recoverNamedGatewayRuntime } from "../actions/global"; import { CLI_DISPLAY_NAME, CLI_NAME } from "../cli/branding"; import { GATEWAY_PORT } from "../core/ports"; +import { gatewayStartGuidance } from "../gateway-start-guidance"; import { resolveGatewayName } from "../onboard/gateway-binding"; import { resolveGatewayCredentialMutationAuthority } from "../onboard/gateway-teardown-authority"; @@ -29,7 +30,7 @@ export function credentialsGatewayRecoveryFailureLines(kind: "query" | "reach"): const action = kind === "query" ? "query" : "reach"; return [ ` Could not ${action} the ${CLI_DISPLAY_NAME} OpenShell gateway. Is it running?`, - ` Run 'openshell gateway start --name nemoclaw' or '${CLI_NAME} onboard' first.`, + ` ${gatewayStartGuidance(resolveGatewayName(GATEWAY_PORT))}`, ]; } diff --git a/src/lib/gateway-start-guidance.test.ts b/src/lib/gateway-start-guidance.test.ts new file mode 100644 index 00000000000..a8619b368b2 --- /dev/null +++ b/src/lib/gateway-start-guidance.test.ts @@ -0,0 +1,39 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { gatewayStartGuidance, resolveGatewayLauncher } from "./gateway-start-guidance"; +import { resolveCurrentOpenShellComputePlan } from "./onboard/compute/plan"; + +describe("gatewayStartGuidance", () => { + it("names the NemoClaw command where NemoClaw launches the gateway", () => { + expect(gatewayStartGuidance("nemoclaw", "nemoclaw")).toBe( + "Start the gateway again with `nemoclaw onboard`.", + ); + }); + + it("names the owning deployment and gateway selection where NemoClaw does not launch it", () => { + const guidance = gatewayStartGuidance("nemoclaw-8091", "openshell"); + + expect(guidance).toContain("does not start the 'nemoclaw-8091' gateway on this host"); + expect(guidance).toContain("openshell gateway select nemoclaw-8091"); + }); + + it("never names a gateway lifecycle command the OpenShell CLI does not have", () => { + for (const launcher of ["nemoclaw", "openshell"] as const) { + expect(gatewayStartGuidance("nemoclaw", launcher)).not.toContain("openshell gateway start"); + } + }); + + it("reads the launcher the current runtime provider records", () => { + expect(resolveGatewayLauncher({ gatewayLauncher: "openshell" })).toBe("openshell"); + expect(resolveGatewayLauncher()).toBe(resolveCurrentOpenShellComputePlan().gatewayLauncher); + }); + + it("resolves the launcher from the plan when the caller supplies no override", () => { + expect(gatewayStartGuidance("nemoclaw")).toBe( + gatewayStartGuidance("nemoclaw", resolveGatewayLauncher()), + ); + }); +}); diff --git a/src/lib/gateway-start-guidance.ts b/src/lib/gateway-start-guidance.ts new file mode 100644 index 00000000000..b9919d202b8 --- /dev/null +++ b/src/lib/gateway-start-guidance.ts @@ -0,0 +1,47 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { CLI_DISPLAY_NAME, CLI_NAME } from "./cli/branding"; +import type { OpenShellGatewayLauncher } from "./onboard/compute/plan"; + +export type { OpenShellGatewayLauncher }; + +/** + * Read the launcher the current runtime provider records. The provider registry + * is loaded on demand so guidance stays importable from the credential and + * inventory command paths without pulling the onboarding graph into them. + */ +export function resolveGatewayLauncher(plan?: { + gatewayLauncher: OpenShellGatewayLauncher; +}): OpenShellGatewayLauncher { + if (plan) return plan.gatewayLauncher; + const { resolveCurrentOpenShellComputePlan } = + require("./onboard/compute/plan") as typeof import("./onboard/compute/plan"); + return resolveCurrentOpenShellComputePlan().gatewayLauncher; +} + +/** + * Name the component that starts the gateway again. The OpenShell CLI has no + * command that starts a gateway, so naming one sends the operator to a + * remediation that cannot run. The launcher the runtime provider records + * selects the branch. `nemoclaw` means NemoClaw starts the gateway process. + * `openshell` means the deployment that created the gateway process still owns + * starting it. Callers that know which gateway failed pass its name so the + * printed selection command is copyable. + */ +export function gatewayStartGuidance( + gatewayName?: string, + launcher: OpenShellGatewayLauncher = resolveGatewayLauncher(), +): string { + if (launcher === "nemoclaw") { + return `Start the gateway again with \`${CLI_NAME} onboard\`.`; + } + const subject = gatewayName ? `the '${gatewayName}' gateway` : "the OpenShell gateway"; + const select = gatewayName + ? `openshell gateway select ${gatewayName}` + : "openshell gateway select"; + return ( + `${CLI_DISPLAY_NAME} does not start ${subject} on this host. ` + + `Start it with the deployment that owns the gateway process, then run \`${select}\`.` + ); +} diff --git a/src/lib/inventory/index.ts b/src/lib/inventory/index.ts index b5b2950c7f8..224fc478bc5 100644 --- a/src/lib/inventory/index.ts +++ b/src/lib/inventory/index.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { CLI_NAME } from "../cli/branding"; +import { gatewayStartGuidance } from "../gateway-start-guidance"; import type { GatewayInference } from "../inference/config"; import { getActiveChannelIdsFromPlan } from "../messaging/plan-validation"; import type { GatewayOwnerDescription } from "../onboard/gateway-ownership"; @@ -590,9 +591,7 @@ export function showStatusCommand(deps: ShowStatusCommandDeps): void { log(""); const detail = health.reason ? ` (${health.reason})` : ""; log(` gateway: down [${health.state}]${detail}`); - log( - ` Run 'openshell gateway start --name nemoclaw' or 'nemoclaw onboard --resume' to recover.`, - ); + log(` ${gatewayStartGuidance()}`); process.exitCode = 1; } } diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index f30348a8cbe..4e29cb2b85c 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -159,7 +159,6 @@ const crypto = require("node:crypto"); const fs = require("fs"); const os = require("os"); const path = require("path"); -const pRetry = require("p-retry"); const runner: typeof import("./runner") = require("./runner"); const { ROOT, SCRIPTS, redact, run, runCapture, runCaptureEx, runFile, validateName } = runner; const braveProviderProfile: typeof import("./onboard/brave-provider-profile") = require("./onboard/brave-provider-profile"); @@ -521,11 +520,8 @@ const { trackChildExit } = require("./onboard/child-exit-tracker") as typeof import("./onboard/child-exit-tracker"); const { reportDockerDriverGatewayStartFailure: reportGatewayFailure } = require("./onboard/docker-driver-gateway-failure") as typeof import("./onboard/docker-driver-gateway-failure"); -const { - createFinalGatewayStartFailureHandler, - normalizeGatewayStartError, - reportLegacyGatewayStartResultFailure, -} = require("./onboard/gateway-start-failure") as typeof import("./onboard/gateway-start-failure"); +const { createFinalGatewayStartFailureHandler, normalizeGatewayStartError } = + require("./onboard/gateway-start-failure") as typeof import("./onboard/gateway-start-failure"); const dockerDriverGatewayEnv: typeof import("./onboard/docker-driver-gateway-env") = require("./onboard/docker-driver-gateway-env"); const dockerDriverGatewayRuntimeMarker: typeof import("./onboard/docker-driver-gateway-runtime-marker") = @@ -565,12 +561,12 @@ const sandboxCreateFailureDiagnostics: typeof import("./onboard/sandbox-create-f import type { CurlProbeResult } from "./adapters/http/probe"; import type { AgentDefinition } from "./agent/defs"; +import { gatewayStartGuidance } from "./gateway-start-guidance"; import type { WebSearchConfig } from "./inference/web-search"; import { hydrateMessagingChannelConfig, type MessagingChannelConfig, } from "./messaging-channel-config"; -import { streamGatewayStart } from "./onboard/gateway"; import * as gatewayAuthorityCheckpoint from "./onboard/gateway-authority-checkpoint"; import { createGatewayHostRuntime } from "./onboard/gateway-host-runtime"; import { @@ -735,7 +731,6 @@ const { runOpenshell, runCaptureOpenshell, captureOpenshell, - getGatewayPortArg, getDockerDriverGatewayEndpointArg, } = createOpenshellCliHelpers({ getCachedBinary: () => OPENSHELL_BIN, @@ -1549,7 +1544,11 @@ async function preflight( // ── Step 2: Gateway ────────────────────────────────────────────── -/** Start the OpenShell gateway with retry logic and post-start health polling. */ +/** + * Start or reuse the OpenShell gateway for the current runtime provider. Only + * the Docker-driver provider starts a gateway process. Every other provider + * reuses a gateway that its own deployment started. + */ async function startGatewayWithOptions( _gpu: ReturnType, { @@ -1600,150 +1599,25 @@ async function startGatewayWithOptions( return; } console.log( - ` Gateway metadata reports healthy but ${getGatewayLocalEndpoint()}/ is not responding. Starting a fresh gateway...`, + ` Gateway metadata reports healthy but ${getGatewayLocalEndpoint()}/ is not responding.`, ); } if (hasStaleGateway(gatewaySnapshot.gwInfo)) { - console.log(" Stale gateway detected — attempting restart without destroy..."); - } - - try { - const { execFileSync } = require("child_process"); - execFileSync("ssh-keygen", ["-R", `openshell-${GATEWAY_NAME}`], { stdio: "ignore" }); - } catch { - /* ssh-keygen -R may fail if entry doesn't exist — safe to ignore */ - } - const knownHostsPath = path.join(os.homedir(), ".ssh", "known_hosts"); - try { - const kh = fs.readFileSync(knownHostsPath, "utf8"); - const cleaned = pruneKnownHostsEntries(kh); - if (cleaned !== kh) fs.writeFileSync(knownHostsPath, cleaned); - } catch { - /* best-effort cleanup — ignore absent/read/write errors */ + console.log(" Stale gateway detected."); } - const gwArgs = ["--name", GATEWAY_NAME, "--port", getGatewayPortArg()]; - if (gpuPassthrough) { - gwArgs.push("--gpu"); - } - const gatewayEnv = getGatewayStartEnv(); - if (gatewayEnv.OPENSHELL_CLUSTER_IMAGE) { - console.log(` Using pinned OpenShell gateway image: ${gatewayEnv.OPENSHELL_CLUSTER_IMAGE}`); - } - - const retries = exitOnFailure ? 2 : 0; - let dockerUnreachable = false; - try { - await pRetry( - async () => { - const startResult = await streamGatewayStart( - openshellShellCommand(["gateway", "start", ...gwArgs]), - { - ...process.env, - ...gatewayEnv, - }, - ); - if (startResult.status !== 0) { - const failure = reportLegacyGatewayStartResultFailure( - startResult.output || "", - console.log, - ); - if (failure.kind === "docker_unreachable") { - dockerUnreachable = true; - throw new pRetry.AbortError("Docker daemon is not reachable (gateway cannot start)."); - } - } - console.log(" Waiting for gateway health..."); - const healthWait = getGatewayHealthWaitConfig( - startResult.status, - getGatewayClusterContainerState(), - ); - if (healthWait.extended) { - console.log( - ` Gateway container is still ${healthWait.containerState}; allowing up to ${ - healthWait.count * healthWait.interval - }s for first-time startup.`, - ); - } - if ( - await waitForGatewayHealth({ - attachGatewayMetadataIfNeeded, - gatewayClusterHealthcheckPassed, - gatewayName: GATEWAY_NAME, - healthPollCount: healthWait.count, - healthPollIntervalSeconds: healthWait.interval, - isGatewayHealthy, - isGatewayHttpReady: (signal) => - isGatewayHttpReady(undefined, undefined, undefined, signal), - repairGatewayBootstrapSecrets, - runCaptureOpenshell, - sleepSeconds, - }) - ) { - return; - } - - const waitLimit = formatGatewayHealthWaitLimit(healthWait.count, healthWait.interval); - throw new Error(`Gateway failed within the configured ${waitLimit}.`); - }, - { - retries, - minTimeout: 10_000, - factor: 3, - onFailedAttempt: (err: { attemptNumber: number; retriesLeft: number }) => { - console.log( - ` Gateway start attempt ${err.attemptNumber} failed. ${err.retriesLeft} retries left...`, - ); - if (err.retriesLeft > 0 && exitOnFailure) { - destroyGateway(); - } - }, - }, - ); - } catch (error) { - if (exitOnFailure) handleFinalGatewayStartFailure({ retries, dockerUnreachable }); - throw normalizeGatewayStartError(error); - } - - console.log(" ✓ Gateway is healthy"); - - // CoreDNS fix — k3s-inside-Docker has broken DNS forwarding on all platforms. - const runtime = getContainerRuntime(); - if (shouldPatchCoredns(runtime)) { - console.log(" Patching CoreDNS DNS forwarding..."); - run(["bash", path.join(SCRIPTS, "fix-coredns.sh"), GATEWAY_NAME], { - ignoreError: true, - }); - const corednsReady = waitUntil(() => { - const check = runCaptureOpenshell( - [ - "doctor", - "exec", - "--", - "kubectl", - "get", - "pods", - "-n", - "kube-system", - "-l", - "k8s-app=kube-dns", - "-o", - 'jsonpath={range .items[*]}{.status.phase}{" "}{range .status.containerStatuses[*]}{.ready}{" "}{end}{end}', - ], - { ignoreError: true }, - ); - return check.includes("Running") && check.includes("true") && !check.includes("false"); - }, 10); - if (!corednsReady) { - console.warn( - " CoreDNS did not report ready within timeout; continuing may cause DNS flakiness.", - ); - } - } - runOpenshell(["gateway", "select", GATEWAY_NAME], { ignoreError: true }); - process.env.OPENSHELL_GATEWAY = GATEWAY_NAME; + // Reuse is the only startup this runtime provider has. The OpenShell CLI has + // no command that starts a gateway, so the gateway process belongs to + // whichever deployment created it. Only the `openshell` launcher reaches + // here, and its guidance already opens with this same sentence, so print the + // guidance alone and keep the sentence for the thrown error. + const unstartableGateway = `${cliDisplayName()} does not start the '${GATEWAY_NAME}' gateway on this host.`; + console.error(` ${gatewayStartGuidance(GATEWAY_NAME)}`); + if (exitOnFailure) process.exit(1); + throw normalizeGatewayStartError(new Error(unstartableGateway)); } + async function startDockerDriverGateway({ exitOnFailure = true, skipSandboxBridgeReachability = false, @@ -1944,7 +1818,6 @@ async function startGateway( async function startGatewayForRecovery(options = {}): Promise { return require("./onboard/gateway-recovery").startGatewayForRecovery(options, { assertGatewayStartAllowed, - getGatewayStartEnv, runCaptureOpenshell, runOpenshell, startGatewayWithOptions, @@ -1992,35 +1865,13 @@ async function recoverGatewayRuntime() { } runOpenshell(["gateway", "select", GATEWAY_NAME], { ignoreError: true }); - let status = runCaptureOpenshell(["status"], { ignoreError: true }); + const status = runCaptureOpenshell(["status"], { ignoreError: true }); if (status.includes("Connected") && isSelectedGateway(status) && (await isGatewayHttpReady())) { process.env.OPENSHELL_GATEWAY = GATEWAY_NAME; return true; } - const startResult = runOpenshell( - ["gateway", "start", "--name", GATEWAY_NAME, "--port", getGatewayPortArg()], - { - ignoreError: true, - env: getGatewayStartEnv(), - suppressOutput: true, - }, - ); - if (startResult.status !== 0) { - const diagnostic = compactText( - redact(`${startResult.stderr || ""} ${startResult.stdout || ""}`), - ); - console.error(` Gateway restart failed (exit ${startResult.status}).`); - if (diagnostic) { - console.error(` ${diagnostic.slice(0, 240)}`); - } - } - runOpenshell(["gateway", "select", GATEWAY_NAME], { ignoreError: true }); - - const recoveryWait = getGatewayHealthWaitConfig( - startResult.status ?? 0, - getGatewayClusterContainerState(), - ); + const recoveryWait = getGatewayHealthWaitConfig(0, getGatewayClusterContainerState()); const recoveryPollCount = recoveryWait.extended ? recoveryWait.count : envInt("NEMOCLAW_HEALTH_POLL_COUNT", 10); @@ -2039,7 +1890,10 @@ async function recoverGatewayRuntime() { runCaptureOpenshell, sleepSeconds, }); - if (!healthy) return false; + if (!healthy) { + console.error(` ${gatewayStartGuidance(GATEWAY_NAME)}`); + return false; + } process.env.OPENSHELL_GATEWAY = GATEWAY_NAME; if (shouldPatchCoredns(getContainerRuntime())) { diff --git a/src/lib/onboard/docker-cdi.ts b/src/lib/onboard/docker-cdi.ts index f305629192e..cf9d0246dbc 100644 --- a/src/lib/onboard/docker-cdi.ts +++ b/src/lib/onboard/docker-cdi.ts @@ -466,7 +466,7 @@ export function explainStaleCdiReason(mismatch: string | undefined): string { const isLeftover = flaggedFilePath && flaggedFilePath !== NVIDIA_CDI_REFRESH_SPEC_PATH; return ( `An NVIDIA CDI device node no longer matches the live device (${detail}). ` + - "OpenShell's `gateway start --gpu` injects devices from the CDI spec, so a stale " + + "A GPU-enabled gateway injects devices from the CDI spec, so a stale " + "device number points the container at the wrong device and CUDA init fails " + "(`CUDA unknown error`). The nvidia-cdi-refresh service keeps " + `${NVIDIA_CDI_REFRESH_SPEC_PATH} current on driver/toolkit changes` + @@ -503,7 +503,7 @@ export function explainNvidiaCdiRepairReason(assessment: NvidiaCdiRepairAssessme ); } reasons.push( - "OpenShell's `gateway start --gpu` can fail until the CDI spec is refreshed and verified.", + "A GPU-enabled gateway start can fail until the CDI spec is refreshed and verified.", ); return reasons.join(" "); } diff --git a/src/lib/onboard/gateway-recovery.test.ts b/src/lib/onboard/gateway-recovery.test.ts index a24111674e6..1bdf58ea7e5 100644 --- a/src/lib/onboard/gateway-recovery.test.ts +++ b/src/lib/onboard/gateway-recovery.test.ts @@ -35,7 +35,6 @@ function createDeps(overrides: Partial = {}): GatewayRecove return { assertGatewayStartAllowed: vi.fn(), getGatewayClusterContainerState: () => "missing", - getGatewayStartEnv: () => ({ OPENSHELL_DRIVERS: "docker" }), runCaptureOpenshell: vi.fn(() => "Disconnected"), runOpenshell: vi.fn(() => ({ status: 0 })), getContainerRuntime: () => "docker", @@ -78,22 +77,13 @@ describe("gateway recovery", () => { ); expect(deps.startGatewayWithOptions).not.toHaveBeenCalled(); - expect(deps.runOpenshell).toHaveBeenNthCalledWith( - 1, - ["gateway", "start", "--name", "nemoclaw-8090", "--port", "8090"], - { - ignoreError: true, - env: { - OPENSHELL_DRIVERS: "docker", - OPENSHELL_SERVER_PORT: "8090", - OPENSHELL_SSH_GATEWAY_PORT: "8090", - }, - suppressOutput: true, - }, - ); - expect(deps.runOpenshell).toHaveBeenNthCalledWith(2, ["gateway", "select", "nemoclaw-8090"], { + expect(deps.runOpenshell).toHaveBeenNthCalledWith(1, ["gateway", "select", "nemoclaw-8090"], { ignoreError: true, }); + expect(deps.runOpenshell).not.toHaveBeenCalledWith( + expect.arrayContaining(["gateway", "start"]), + expect.anything(), + ); }); it("derives the canonical gateway name when only a non-default port is supplied", async () => { @@ -104,16 +94,9 @@ describe("gateway recovery", () => { "Gateway 'nemoclaw-8091' did not become ready", ); - expect(deps.runOpenshell).toHaveBeenNthCalledWith( - 1, - ["gateway", "start", "--name", "nemoclaw-8091", "--port", "8091"], - expect.objectContaining({ - env: expect.objectContaining({ - OPENSHELL_SERVER_PORT: "8091", - OPENSHELL_SSH_GATEWAY_PORT: "8091", - }), - }), - ); + expect(deps.runOpenshell).toHaveBeenNthCalledWith(1, ["gateway", "select", "nemoclaw-8091"], { + ignoreError: true, + }); }); it("polls until the configured recovery deadline and reports it in the timeout (#3768)", async () => { @@ -313,8 +296,8 @@ describe("gateway lifecycle authority during recovery", () => { }), }); - // The cross-port, non-default-name target is the branch that reaches a raw - // `openshell gateway start` without going through startGatewayWithOptions. + // The cross-port, non-default-name target is the branch that reselects the + // gateway without going through startGatewayWithOptions. await expect( startGatewayForRecovery({ gatewayName: "nemoclaw-8090", gatewayPort: 8090 }, deps), ).rejects.toThrow(ownershipError); diff --git a/src/lib/onboard/gateway-recovery.ts b/src/lib/onboard/gateway-recovery.ts index b542e94b00d..c48a2a535b8 100644 --- a/src/lib/onboard/gateway-recovery.ts +++ b/src/lib/onboard/gateway-recovery.ts @@ -20,6 +20,7 @@ import { validateGatewayPort, } from "../core/ports"; import { sleepSeconds, waitUntilAsync } from "../core/wait"; +import { gatewayStartGuidance } from "../gateway-start-guidance"; import { shouldPatchCoredns } from "../platform"; import { run, SCRIPTS } from "../runner"; import { isGatewayHealthy } from "../state/gateway"; @@ -64,7 +65,6 @@ export type GatewayRecoveryDeps = { target: { gatewayName: string; gatewayPort: number }, ): void; getGatewayClusterContainerState?(gatewayName: string): string; - getGatewayStartEnv(): Record; runCaptureOpenshell(args: string[], opts?: RunCaptureOpenshellOptions): string; runOpenshell(args: string[], opts?: RunOpenshellOptions): GatewayStartResult; startGatewayWithOptions(gpu: never, options: { exitOnFailure: false }): Promise; @@ -124,17 +124,6 @@ function resolveGatewayRecoveryTarget(options: StartGatewayForRecoveryOptions = return { gatewayName, gatewayPort }; } -function getGatewayStartEnvForPort( - gatewayPort: number, - getGatewayStartEnv: GatewayRecoveryDeps["getGatewayStartEnv"], -): Record { - return { - ...getGatewayStartEnv(), - OPENSHELL_SERVER_PORT: String(gatewayPort), - OPENSHELL_SSH_GATEWAY_PORT: String(gatewayPort), - }; -} - function getDefaultGatewayClusterContainerState(gatewayName: string): string { const state = dockerContainerInspectFormat( "{{.State.Status}}{{if .State.Health}} {{.State.Health.Status}}{{end}}", @@ -174,19 +163,10 @@ async function startTargetGatewayForRecovery( { gatewayName, gatewayPort }: { gatewayName: string; gatewayPort: number }, deps: GatewayRecoveryDeps, ): Promise { - const gatewayPortArg = String(gatewayPort); - const startResult = deps.runOpenshell( - ["gateway", "start", "--name", gatewayName, "--port", gatewayPortArg], - { - ignoreError: true, - env: getGatewayStartEnvForPort(gatewayPort, deps.getGatewayStartEnv), - suppressOutput: true, - }, - ); deps.runOpenshell(["gateway", "select", gatewayName], { ignoreError: true }); const recoveryWait = getGatewayHealthWaitConfig( - startResult.status ?? 0, + 0, (deps.getGatewayClusterContainerState ?? getDefaultGatewayClusterContainerState)(gatewayName), ); const recoveryPollCount = recoveryWait.extended @@ -243,7 +223,7 @@ async function startTargetGatewayForRecovery( ? formatGatewayHealthWaitLimit(recoveryPollCount, recoveryPollInterval) : `${formatReadinessDeadline(waitBudgetMs)} recovery deadline (${recoveryPollInterval}s poll interval)`; throw new Error( - `Gateway '${gatewayName}' did not become ready within the configured ${waitLimit}`, + `Gateway '${gatewayName}' did not become ready within the configured ${waitLimit}. ${gatewayStartGuidance(gatewayName)}`, ); } @@ -252,18 +232,19 @@ export async function startGatewayForRecovery( deps: GatewayRecoveryDeps, ): Promise { const target = resolveGatewayRecoveryTarget(options); - // Guard every recovery branch, including the cross-port / non-default-name - // path below that reaches `openshell gateway start` without going through - // startGatewayWithOptions. Resolve and bind the requested target first: the - // process-global gateway can name a different port during sandbox recovery. + // Guard every recovery branch. The cross-port / non-default-name path below + // bypasses startGatewayWithOptions. It reselects an already-running gateway + // and waits for health instead of starting a gateway process. Resolve and + // bind the requested target first, because the process-global gateway can + // name a different port during sandbox recovery. deps.assertGatewayStartAllowed(false, target); const linuxDockerDriverEnabled = ( deps.isLinuxDockerDriverGatewayEnabled ?? isLinuxDockerDriverGatewayEnabled )(); // The Docker-driver Linux startup path (startGatewayWithOptions → // startDockerDriverGateway) restores the runtime-marker, package-managed - // registration, and sandbox-bridge reachability — none of which a plain - // `openshell gateway start` produces. Route through it whenever the + // registration, and sandbox-bridge reachability, and it is the only path + // that starts a gateway process at all. Route through it whenever the // recovery target matches the current process's GATEWAY_PORT (the common // case where the user re-runs with the same NEMOCLAW_GATEWAY_PORT). if (target.gatewayPort === GATEWAY_PORT) { @@ -273,11 +254,11 @@ export async function startGatewayForRecovery( } // Cross-port recovery on a Linux Docker-driver gateway cannot share this // process's module-globals: startDockerDriverGateway captures the port at - // load time, so a plain `openshell gateway start` would skip the - // runtime-marker / package registration / sandbox-bridge setup and leave - // the host in a half-recovered state. Fail closed instead and direct the - // operator to re-run with the matching NEMOCLAW_GATEWAY_PORT so the - // docker-driver path re-stamps the per-port artefacts. + // load time, so the reselect path below would skip the runtime-marker / + // package registration / sandbox-bridge setup and leave the host in a + // half-recovered state. Fail closed instead and direct the operator to + // re-run with the matching NEMOCLAW_GATEWAY_PORT so the docker-driver path + // re-stamps the per-port artefacts. if (linuxDockerDriverEnabled && target.gatewayPort !== GATEWAY_PORT) { throw new Error( `Cross-port recovery for Linux Docker-driver gateway '${target.gatewayName}' is not safe from a process bound to port ${GATEWAY_PORT}. ` + diff --git a/src/lib/onboard/gateway-start-failure-integration.test.ts b/src/lib/onboard/gateway-start-failure-integration.test.ts index 1ecf6989590..44ea4af2183 100644 --- a/src/lib/onboard/gateway-start-failure-integration.test.ts +++ b/src/lib/onboard/gateway-start-failure-integration.test.ts @@ -1,9 +1,9 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 // -// Integration tests for the docker-unreachable abort path that -// startGatewayWithOptions() takes when `openshell gateway start` reports -// the Docker daemon is not reachable. See src/lib/onboard.ts:2233. +// Integration tests for the docker-unreachable abort path the Docker-driver +// gateway start takes when the gateway log reports the Docker daemon is not +// reachable. // // This helper-level suite preserves the former shell regression, which was // structurally a Node-process unit test of startGateway() with a PATH-shimmed @@ -28,7 +28,6 @@ import { classifyGatewayStartFailure } from "../validation"; import { createFinalGatewayStartFailureHandler, printDockerDaemonRecovery, - reportLegacyGatewayStartResultFailure, } from "./gateway-start-failure"; // The production binding itself remains covered by @@ -41,10 +40,9 @@ const handleFinalGatewayStartFailure = createFinalGatewayStartFailureHandler({ cleanupGateway: () => undefined, }); -// Real signatures the legacy script's fake openshell binary emitted from -// `gateway start` to simulate Colima-stopped (macOS) and dockerd-stopped -// (Linux). These are the wire format the call site sees from -// streamGatewayStart()'s `output` field. +// Real signatures a stopped Docker daemon produces on macOS (Colima) and Linux +// (dockerd). These are the wire format the call site sees in the gateway log +// tail it classifies. const DARWIN_DOCKER_UNREACHABLE_OUTPUT = [ "Error: Failed to create Docker client.", "Socket not found: /var/run/docker.sock", @@ -177,38 +175,28 @@ describe("startGatewayWithOptions docker-unreachable abort (#2347)", () => { // ── Layer 2: composition test — the exact sequence the call site uses ──── // - // startGatewayWithOptions does, on `streamGatewayStart()` failure: + // The Docker-driver start path classifies the gateway log tail and routes a + // docker_unreachable verdict to the recovery message, while any other + // verdict falls through to the regular failure handler: // - // const failure = reportLegacyGatewayStartResultFailure(output, log); - // if (failure.kind === "docker_unreachable") { - // dockerUnreachable = true; - // throw new pRetry.AbortError(...); - // } - // } catch { - // if (exitOnFailure) { - // handleFinalGatewayStartFailure({ retries, dockerUnreachable }); - // } - // throw new Error("Gateway failed to start"); - // } + // const failure = classifyGatewayStartFailure(tail); + // if (failure.kind === "docker_unreachable") { ... } + // handleFinalGatewayStartFailure({ retries, dockerUnreachable }); // // The composition test exercises the same helpers in the same order and // confirms the chain bottoms out at exitProcess(1) with the recovery // message printed. describe("composition classifies, handles the final state, and exits 1", () => { - let capturedClassifyLog: string[]; let capturedPrintError: string[]; - function runComposition(streamGatewayStartOutput: string): { + function runComposition(gatewayLogTail: string): { thrown: unknown; exitCode: number | null; } { - capturedClassifyLog = []; capturedPrintError = []; - const failure = reportLegacyGatewayStartResultFailure(streamGatewayStartOutput, (m) => - capturedClassifyLog.push(m), - ); + const failure = classifyGatewayStartFailure(gatewayLogTail); let dockerUnreachable = false; if (failure.kind === "docker_unreachable") { @@ -244,10 +232,6 @@ describe("startGatewayWithOptions docker-unreachable abort (#2347)", () => { expect(thrown).toBeInstanceOf(Error); expect(exitCode).toBe(1); expect(capturedPrintError.join("\n")).toContain("Docker daemon is not running"); - // The classification helper logs the original output as a breadcrumb; - // the legacy script asserts on this output too via its `[INFO] node - // exit code` log lines. - expect(capturedClassifyLog.join("\n")).toContain("Gateway start returned before healthy"); }); it("composes through the docker-unreachable path on the Linux dockerd signature", () => { @@ -263,13 +247,9 @@ describe("startGatewayWithOptions docker-unreachable abort (#2347)", () => { // up). If this test ever flips, the call-site classifier has been made // too aggressive and would silence real gateway failures behind the // Docker-recovery message. - capturedClassifyLog = []; capturedPrintError = []; - const failure = reportLegacyGatewayStartResultFailure( - " k3s: failed to bootstrap helm chart after 90s\n", - (m) => capturedClassifyLog.push(m), - ); + const failure = classifyGatewayStartFailure(" k3s: failed to bootstrap helm chart after 90s\n"); expect(failure.kind).toBe("unknown"); diff --git a/src/lib/onboard/gateway-start-failure.test.ts b/src/lib/onboard/gateway-start-failure.test.ts index 746cef99e24..4be52a82c87 100644 --- a/src/lib/onboard/gateway-start-failure.test.ts +++ b/src/lib/onboard/gateway-start-failure.test.ts @@ -7,7 +7,6 @@ import { classifyGatewayStartFailure } from "../validation"; import { createFinalGatewayStartFailureHandler, normalizeGatewayStartError, - reportLegacyGatewayStartResultFailure, } from "./gateway-start-failure"; describe("normalizeGatewayStartError", () => { @@ -100,24 +99,6 @@ describe("classifyGatewayStartFailure", () => { }); }); -describe("reportLegacyGatewayStartResultFailure", () => { - it("classifies Docker-unreachable output after stripping ANSI sequences (#2347)", () => { - const log = vi.fn(); - const output = [ - "\x1b[31mError: Failed to create Docker client.\x1b[0m", - "\x1b[33mSocket not found: /var/run/docker.sock\x1b[0m", - ].join("\n"); - - expect(reportLegacyGatewayStartResultFailure(output, log)).toEqual({ - kind: "docker_unreachable", - }); - expect(log).toHaveBeenCalledWith( - expect.stringContaining("Gateway start returned before healthy"), - ); - expect(log.mock.calls[0][0]).not.toContain("\x1b"); - }); -}); - describe("createFinalGatewayStartFailureHandler", () => { it("normalizes diagnostics before redacting secrets split by terminal control bytes", () => { const printed: string[] = []; diff --git a/src/lib/onboard/gateway-start-failure.ts b/src/lib/onboard/gateway-start-failure.ts index c911f894cb5..a50a73b2ea2 100644 --- a/src/lib/onboard/gateway-start-failure.ts +++ b/src/lib/onboard/gateway-start-failure.ts @@ -3,7 +3,6 @@ import { compactText } from "../core/url-utils"; import { redact } from "../security/redact"; -import { classifyGatewayStartFailure } from "../validation"; import { isPortableExperimentalProfile } from "./experimental/portable-profile"; import { onboardResumeRecoveryCommand } from "./resume-hint"; @@ -28,22 +27,6 @@ export function normalizeGatewayStartError(error: unknown): Error { return error instanceof Error ? error : new Error(String(error)); } -export function reportLegacyGatewayStartResultFailure( - output: string, - log: (message: string) => void, -) { - const cleanedOutput = String(output || "").replace(ANSI_RE, ""); - const lines = redact(cleanedOutput) - .split("\n") - .map((l) => compactText(l)) - .filter(Boolean) - .map((l) => ` ${l}`); - if (lines.length > 0) { - log(` Gateway start returned before healthy:\n${lines.join("\n")}`); - } - return classifyGatewayStartFailure(cleanedOutput); -} - export function printDockerDaemonRecovery( printError: (message?: string) => void, platform: NodeJS.Platform = process.platform, diff --git a/src/lib/onboard/gateway.test.ts b/src/lib/onboard/gateway.test.ts deleted file mode 100644 index db87f0cbaf9..00000000000 --- a/src/lib/onboard/gateway.test.ts +++ /dev/null @@ -1,69 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { EventEmitter } from "node:events"; - -import { beforeEach, describe, expect, it, vi } from "vitest"; - -const spawnMock = vi.hoisted(() => vi.fn()); - -vi.mock("node:child_process", () => ({ - spawn: spawnMock, -})); - -vi.mock("../runner", () => ({ - ROOT: "/repo/root", -})); - -import { streamGatewayStart } from "./gateway"; - -class FakeChild extends EventEmitter { - stdout = new EventEmitter(); - stderr = new EventEmitter(); - kill = vi.fn(); -} - -describe("streamGatewayStart", () => { - beforeEach(() => { - spawnMock.mockReset(); - vi.spyOn(console, "log").mockImplementation(() => {}); - }); - - it("bounds retained diagnostics while preserving initial and terminal evidence", async () => { - const child = new FakeChild(); - spawnMock.mockReturnValue(child); - const resultPromise = streamGatewayStart("openshell gateway start", { - NEMOCLAW_GATEWAY_START_TIMEOUT: "60", - }); - - child.stdout.emit("data", Buffer.from("initial failure signature\n")); - for (let index = 0; index < 8; index += 1) { - child.stdout.emit("data", Buffer.from(`${String(index)}:${"x".repeat(64 * 1024)}\n`)); - } - child.stderr.emit("data", Buffer.from("terminal diagnostic\n")); - child.emit("close", 1); - - const result = await resultPromise; - expect(result.status).toBe(1); - expect(result.output).toContain("initial failure signature"); - expect(result.output).toContain("terminal diagnostic"); - expect(result.output).toContain("transcript characters omitted"); - expect(result.output.length).toBeLessThan(257 * 1024); - }); - - it("does not combine partial stdout and stderr lines", async () => { - const child = new FakeChild(); - spawnMock.mockReturnValue(child); - const resultPromise = streamGatewayStart("openshell gateway start", { - NEMOCLAW_GATEWAY_START_TIMEOUT: "60", - }); - - child.stdout.emit("data", Buffer.from("stdout-part")); - child.stderr.emit("data", Buffer.from("stderr-line\n")); - child.stdout.emit("data", Buffer.from("-end\n")); - child.emit("close", 0); - - const result = await resultPromise; - expect(result.output.split("\n")).toEqual(["stderr-line", "stdout-part-end"]); - }); -}); diff --git a/src/lib/onboard/gateway.ts b/src/lib/onboard/gateway.ts deleted file mode 100644 index d2c67d40f1b..00000000000 --- a/src/lib/onboard/gateway.ts +++ /dev/null @@ -1,170 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { spawn } from "node:child_process"; - -import { BoundedLineDecoder, BoundedTextTranscript } from "../core/bounded-line-transcript"; - -const { envInt }: typeof import("./env") = require("./env"); -const { ROOT } = require("../runner") as typeof import("../runner"); - -const GATEWAY_TRANSCRIPT_MAX_CHARS = 256 * 1024; -const GATEWAY_TRANSCRIPT_HEAD_CHARS = 64 * 1024; -const GATEWAY_MAX_PENDING_LINE_CHARS = 64 * 1024; - -/** Spawn `openshell gateway start` and stream its output with progress heartbeats. */ -export function streamGatewayStart( - command: string, - env: NodeJS.ProcessEnv = process.env, -): Promise<{ status: number; output: string }> { - const child = spawn("bash", ["-lc", command], { - cwd: ROOT, - env, - stdio: ["ignore", "pipe", "pipe"], - }); - - const transcript = new BoundedTextTranscript({ - maxChars: GATEWAY_TRANSCRIPT_MAX_CHARS, - headChars: GATEWAY_TRANSCRIPT_HEAD_CHARS, - }); - let settled = false; - let resolvePromise: (value: { status: number; output: string }) => void; - let lastPrintedLine = ""; - let currentPhase = "cluster"; - let lastHeartbeatBucket = -1; - let lastOutputAt = Date.now(); - const startedAt = Date.now(); - - function getDisplayWidth(): number { - return Math.max(60, Number(process.stdout.columns || 100)); - } - - function trimDisplayLine(line: string): string { - const width = getDisplayWidth(); - const maxLen = Math.max(40, width - 4); - if (line.length <= maxLen) return line; - return `${line.slice(0, Math.max(0, maxLen - 3))}...`; - } - - function printProgressLine(line: string): void { - const display = trimDisplayLine(line); - if (display !== lastPrintedLine) { - console.log(display); - lastPrintedLine = display; - } - } - - function elapsedSeconds(): number { - return Math.max(0, Math.floor((Date.now() - startedAt) / 1000)); - } - - function setPhase(nextPhase: string | null): void { - if (!nextPhase || nextPhase === currentPhase) return; - currentPhase = nextPhase; - const phaseLine = - nextPhase === "install" - ? " Installing OpenShell components..." - : nextPhase === "pod" - ? " Starting OpenShell gateway pod..." - : nextPhase === "health" - ? " Waiting for gateway health..." - : " Starting gateway cluster..."; - printProgressLine(phaseLine); - } - - function classifyLine(line: string): string | null { - if (/ApplyJob|helm-install-openshell|Applying HelmChart/i.test(line)) return "install"; - if ( - /openshell-0|Observed pod startup duration|MountVolume\.MountDevice succeeded/i.test(line) - ) { - return "pod"; - } - if (/Gateway .* ready\.?$/i.test(line)) return "health"; - return null; - } - - function flushLine(rawLine: string): void { - const line = rawLine.replace(/\r/g, "").trimEnd(); - if (!line) return; - transcript.appendLine(line); - lastOutputAt = Date.now(); - const nextPhase = classifyLine(line); - if (nextPhase) setPhase(nextPhase); - } - - const stdoutDecoder = new BoundedLineDecoder({ - maxPendingChars: GATEWAY_MAX_PENDING_LINE_CHARS, - onLine: flushLine, - }); - const stderrDecoder = new BoundedLineDecoder({ - maxPendingChars: GATEWAY_MAX_PENDING_LINE_CHARS, - onLine: flushLine, - }); - - function finish(status: number): void { - if (settled) return; - settled = true; - stdoutDecoder.end(); - stderrDecoder.end(); - clearInterval(heartbeatTimer); - resolvePromise({ status, output: transcript.toString() }); - } - - child.stdout.on("data", (chunk) => { - if (!settled) stdoutDecoder.write(chunk); - }); - child.stderr.on("data", (chunk) => { - if (!settled) stderrDecoder.write(chunk); - }); - - printProgressLine(" Starting gateway cluster..."); - const heartbeatTimer = setInterval(() => { - if (settled) return; - const elapsed = elapsedSeconds(); - const bucket = Math.floor(elapsed / 10); - if (bucket === lastHeartbeatBucket) return; - if (Date.now() - lastOutputAt < 3000 && elapsed < 10) return; - const heartbeatLine = - currentPhase === "install" - ? ` Still installing OpenShell components... (${elapsed}s elapsed)` - : currentPhase === "pod" - ? ` Still starting OpenShell gateway pod... (${elapsed}s elapsed)` - : currentPhase === "health" - ? ` Still waiting for gateway health... (${elapsed}s elapsed)` - : ` Still starting gateway cluster... (${elapsed}s elapsed)`; - printProgressLine(heartbeatLine); - lastHeartbeatBucket = bucket; - }, 5000); - heartbeatTimer.unref?.(); - - // Hard timeout to prevent indefinite hangs if the openshell process - // never exits (e.g. Docker daemon unresponsive, k3s restart loop). (#1830) - // On timeout, send SIGTERM and let the `close` event resolve the promise - // so the child has actually exited before the caller proceeds to retry. - const gatewayStartTimeout = envInt("NEMOCLAW_GATEWAY_START_TIMEOUT", 600) * 1000; - let killedByTimeout = false; - const killTimer = setTimeout(() => { - killedByTimeout = true; - transcript.appendLine("[NemoClaw] Gateway start timed out - killing process."); - child.kill("SIGTERM"); - setTimeout(() => { - if (!settled) child.kill("SIGKILL"); - }, 10_000).unref?.(); - }, gatewayStartTimeout); - killTimer.unref?.(); - - return new Promise<{ status: number; output: string }>((resolve) => { - resolvePromise = resolve; - child.on("error", (error: Error) => { - clearTimeout(killTimer); - const detail = error?.message || String(error); - transcript.appendLine(detail); - finish(1); - }); - child.on("close", (code: number | null) => { - clearTimeout(killTimer); - const exitCode = killedByTimeout ? 1 : (code ?? 1); - finish(exitCode); - }); - }); -} diff --git a/src/lib/onboard/machine/handlers/gateway.ts b/src/lib/onboard/machine/handlers/gateway.ts index 94e545979b6..224f9bdb45a 100644 --- a/src/lib/onboard/machine/handlers/gateway.ts +++ b/src/lib/onboard/machine/handlers/gateway.ts @@ -142,8 +142,8 @@ async function handleGatewayStatePhase({ } else if (containerState === "stopped") { // #4187: a stopped legacy `openshell-cluster-*` container after a host // VM stop/start still holds the PVC volume. Attempt non-destructive - // recovery (openshell gateway start) before any destructive path so we - // never delete the k3s local-path PVC backing data. + // gateway recovery before any destructive path so we never delete the + // k3s local-path PVC backing data. console.log( " Gateway container is stopped (likely host or Docker restart). Attempting non-destructive recovery...", ); diff --git a/src/lib/onboard/preflight-gateway-reuse.ts b/src/lib/onboard/preflight-gateway-reuse.ts index c07b6b3489b..86701dfbd72 100644 --- a/src/lib/onboard/preflight-gateway-reuse.ts +++ b/src/lib/onboard/preflight-gateway-reuse.ts @@ -61,9 +61,9 @@ export async function reconcilePreflightGatewayReuseState( } else if (containerState === "stopped") { // #4187: a stopped legacy `openshell-cluster-*` container after a host VM // stop/start still holds the k3s local-path PVC volume. Attempt - // non-destructive recovery (openshell gateway start) before any - // destructive cleanup path so we never delete the PVC backing data and - // silently provision a fresh, empty workspace. + // non-destructive gateway recovery before any destructive cleanup path so + // we never delete the PVC backing data and silently provision a fresh, + // empty workspace. console.log( " Gateway container is stopped (likely host or Docker restart). Attempting non-destructive recovery...", ); diff --git a/src/lib/validation.ts b/src/lib/validation.ts index 5f500ab93ac..4471cc4477c 100644 --- a/src/lib/validation.ts +++ b/src/lib/validation.ts @@ -216,8 +216,8 @@ export function planSandboxCreateRecovery( } /** - * Classify a non-zero `openshell gateway start` result so the onboard retry - * loop can short-circuit on unrecoverable failures. + * Classify a failed gateway start so the onboard retry loop can short-circuit + * on unrecoverable failures. * * The classifier identifies failures for which callers can select a supported * recovery instead of generic retry or health-wait behavior. diff --git a/test/package-contract/cli/credentials-cli-command.test.ts b/test/package-contract/cli/credentials-cli-command.test.ts index 5fa11c40350..d426c29a75e 100644 --- a/test/package-contract/cli/credentials-cli-command.test.ts +++ b/test/package-contract/cli/credentials-cli-command.test.ts @@ -232,7 +232,7 @@ describe("credentials oclif commands", () => { ); expect(output.stderr).toContain("Could not query OpenShell gateway"); - expect(output.stderr).toContain("openshell gateway start --name nemoclaw"); + expect(output.stderr).toContain("Start the gateway again with `nemoclaw onboard`."); }); it("records gateway recovery failures without calling provider list", async () => { diff --git a/test/sandbox-status-json-stdout.test.ts b/test/sandbox-status-json-stdout.test.ts index 766da3f23a8..c26c5808326 100644 --- a/test/sandbox-status-json-stdout.test.ts +++ b/test/sandbox-status-json-stdout.test.ts @@ -8,8 +8,8 @@ import { getSandboxStatusReport } from "../src/lib/actions/sandbox/status-snapsh // `sandbox status --json` builds a machine-readable report through // getSandboxStatusReport, which reconciles the gateway. When the gateway needs // recovery, the reconcile path prints human progress to stdout (step(), -// streamGatewayStart, "Waiting for gateway health...", etc.). We inject a -// reconcile that writes that progress and assert the --json report builder +// "Waiting for gateway health...", and so on). We inject a reconcile that +// writes that progress and assert the --json report builder // keeps stdout clean; otherwise the JSON document on stdout is unparseable. // Writes go through process.stdout.write directly (what console.log delegates // to), so the test targets the exact stream the builder must keep clean. From 204b18fb2c4d2a39efa93b959dffcdb498f79f28 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 14 Aug 2026 09:53:57 -0700 Subject: [PATCH 2/5] fix(gateway): complete launcher-specific recovery guidance Signed-off-by: Prekshi Vyas --- ci/source-architecture-budget.json | 2 +- .../recover-rebuild-sandboxes.mdx | 2 +- docs/reference/commands.mdx | 5 +- src/lib/actions/sandbox/gateway-state.ts | 2 +- src/lib/gateway-start-guidance.test.ts | 6 + src/lib/gateway-start-guidance.ts | 5 +- .../gateway-start-failure-integration.test.ts | 164 +----------------- ...onboard-gateway-docker-unreachable.test.ts | 148 ---------------- test/onboard.test.ts | 104 ++--------- test/strict-tool-call-probe.test.ts | 6 +- 10 files changed, 40 insertions(+), 404 deletions(-) delete mode 100644 test/onboard-gateway-docker-unreachable.test.ts diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 2ffdee8a65f..418bef80137 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -24,7 +24,7 @@ "src/lib/inference/web-search.ts": 21, "src/lib/messaging/channels/index.ts": 25, "src/lib/onboard/gateway-binding.ts": 52, - "src/lib/runner.ts": 88, + "src/lib/runner.ts": 87, "src/lib/security/redact.ts": 53, "src/lib/state/onboard-session.ts": 36, "src/lib/state/registry.ts": 101, diff --git a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx index fb7ac992419..953feb985c3 100644 --- a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx +++ b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx @@ -209,7 +209,7 @@ If the host gateway RPC returns an error while these sandbox-scoped commands run The OpenShell CLI has no command that starts a gateway, and `openshell status` only reports the gateway state. If NemoClaw starts the gateway on your host, run `$$nemoclaw onboard` again to repair the host gateway itself. If a deployment outside NemoClaw owns the gateway process, start the gateway with that deployment, then run `openshell gateway select `. -NemoClaw names the applicable command when a sandbox-scoped command reports that the host gateway is down. +NemoClaw prints the applicable recovery guidance when a sandbox-scoped command reports that the host gateway is down. Other workflows, including onboarding, rebuild, and `doctor --fix`, can explicitly recover the named host gateway when their operation requires it. `$$nemoclaw gateway restart` instead restarts only the supported agent gateway inside the named sandbox. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index ce1917e4522..c139919cbf7 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -3951,8 +3951,9 @@ $$nemoclaw status --json When at least one sandbox is registered and the named NemoClaw gateway is unreachable, unhealthy, or attached to a different sandbox, the command prints a `gateway: down [state] (reason)` line between the sandbox list and the host-service list. The command classifies the failing layer when possible: the named gateway port is not accepting connections, the named gateway is running but not Connected, the active OpenShell gateway points at a different name, or the named gateway is not configured at all. -It then prints the gateway start guidance for your host. -That guidance names `$$nemoclaw onboard` where NemoClaw starts the gateway, and the owning deployment plus `openshell gateway select ` where it does not. +It then prints the gateway recovery guidance for your host. +That guidance names `$$nemoclaw onboard` when NemoClaw starts the gateway process. +When another deployment owns that process, the guidance directs you to start it with the owning deployment and run `openshell gateway select `. It exits with code `1` so shell scripts and CI can detect the degraded state from `$?`. For `--json`, the structured output includes `gatewayHealth`, and the exit code is set after the report is generated. A clean machine with no registered sandboxes keeps the legacy `0` exit because no gateway is expected to be configured yet. diff --git a/src/lib/actions/sandbox/gateway-state.ts b/src/lib/actions/sandbox/gateway-state.ts index 2dd93faacff..00b0936d257 100644 --- a/src/lib/actions/sandbox/gateway-state.ts +++ b/src/lib/actions/sandbox/gateway-state.ts @@ -491,7 +491,7 @@ export function printGatewayLifecycleHint( gatewayNamePattern(targetGatewayName).test(cleanOutput) ) { writer( - " The selected NemoClaw gateway exists in metadata, but its API is refusing connections after restart.", + " The target OpenShell gateway exists in metadata, but its API is refusing connections after restart.", ); writer(" This usually means the gateway runtime did not come back cleanly after the restart."); writer( diff --git a/src/lib/gateway-start-guidance.test.ts b/src/lib/gateway-start-guidance.test.ts index a8619b368b2..ffdf0adfc60 100644 --- a/src/lib/gateway-start-guidance.test.ts +++ b/src/lib/gateway-start-guidance.test.ts @@ -20,6 +20,12 @@ describe("gatewayStartGuidance", () => { expect(guidance).toContain("openshell gateway select nemoclaw-8091"); }); + it("marks the required gateway argument when the caller does not know its name", () => { + expect(gatewayStartGuidance(undefined, "openshell")).toContain( + "openshell gateway select ", + ); + }); + it("never names a gateway lifecycle command the OpenShell CLI does not have", () => { for (const launcher of ["nemoclaw", "openshell"] as const) { expect(gatewayStartGuidance("nemoclaw", launcher)).not.toContain("openshell gateway start"); diff --git a/src/lib/gateway-start-guidance.ts b/src/lib/gateway-start-guidance.ts index b9919d202b8..6361cfdd5b3 100644 --- a/src/lib/gateway-start-guidance.ts +++ b/src/lib/gateway-start-guidance.ts @@ -27,7 +27,8 @@ export function resolveGatewayLauncher(plan?: { * selects the branch. `nemoclaw` means NemoClaw starts the gateway process. * `openshell` means the deployment that created the gateway process still owns * starting it. Callers that know which gateway failed pass its name so the - * printed selection command is copyable. + * printed selection command is copyable. Other callers print a template that + * marks the required gateway argument. */ export function gatewayStartGuidance( gatewayName?: string, @@ -39,7 +40,7 @@ export function gatewayStartGuidance( const subject = gatewayName ? `the '${gatewayName}' gateway` : "the OpenShell gateway"; const select = gatewayName ? `openshell gateway select ${gatewayName}` - : "openshell gateway select"; + : "openshell gateway select "; return ( `${CLI_DISPLAY_NAME} does not start ${subject} on this host. ` + `Start it with the deployment that owns the gateway process, then run \`${select}\`.` diff --git a/src/lib/onboard/gateway-start-failure-integration.test.ts b/src/lib/onboard/gateway-start-failure-integration.test.ts index 44ea4af2183..68e0ba78a8a 100644 --- a/src/lib/onboard/gateway-start-failure-integration.test.ts +++ b/src/lib/onboard/gateway-start-failure-integration.test.ts @@ -12,27 +12,20 @@ // Original regression: NemoClaw #2347. // Owning migration issue: NemoClaw #4355. // -// Coverage strategy: prove the helper-level contract through two layers: -// -// 1. Unit tests of the already-exported helpers (printDockerDaemonRecovery, -// handleFinalGatewayStartFailure with dockerUnreachable=true). -// 2. A composition test that runs the same helper sequence the call site -// uses (classify → handleFinal → exitProcess(1)). -// -// The caller-level process regression that drives startGateway() through a -// PATH-shimmed openshell binary lives in -// test/onboard-gateway-docker-unreachable.test.ts. +// Coverage strategy: prove the exported helper contract here. The production +// Docker-driver composition is exercised through +// docker-driver-gateway-failure.test.ts instead of copying its control flow +// into this test. import { describe, expect, it, vi } from "vitest"; -import { classifyGatewayStartFailure } from "../validation"; import { createFinalGatewayStartFailureHandler, printDockerDaemonRecovery, } from "./gateway-start-failure"; // The production binding itself remains covered by -// test/gateway-final-failure-cleanup.test.ts. These helper and composition -// checks only need the production factory, and should not load onboard.ts's +// test/gateway-final-failure-cleanup.test.ts. These helper checks only need the +// production factory, and should not load onboard.ts's // full dependency graph for every source-test worker. const handleFinalGatewayStartFailure = createFinalGatewayStartFailureHandler({ getGatewayName: () => "nemoclaw", @@ -40,17 +33,6 @@ const handleFinalGatewayStartFailure = createFinalGatewayStartFailureHandler({ cleanupGateway: () => undefined, }); -// Real signatures a stopped Docker daemon produces on macOS (Colima) and Linux -// (dockerd). These are the wire format the call site sees in the gateway log -// tail it classifies. -const DARWIN_DOCKER_UNREACHABLE_OUTPUT = [ - "Error: Failed to create Docker client.", - "Socket not found: /var/run/docker.sock", -].join("\n"); - -const LINUX_DOCKER_UNREACHABLE_OUTPUT = - "Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?"; - describe("startGatewayWithOptions docker-unreachable abort (#2347)", () => { // ── Layer 1: unit tests of the platform-branching recovery message ──────── @@ -173,138 +155,4 @@ describe("startGatewayWithOptions docker-unreachable abort (#2347)", () => { }); }); - // ── Layer 2: composition test — the exact sequence the call site uses ──── - // - // The Docker-driver start path classifies the gateway log tail and routes a - // docker_unreachable verdict to the recovery message, while any other - // verdict falls through to the regular failure handler: - // - // const failure = classifyGatewayStartFailure(tail); - // if (failure.kind === "docker_unreachable") { ... } - // handleFinalGatewayStartFailure({ retries, dockerUnreachable }); - // - // The composition test exercises the same helpers in the same order and - // confirms the chain bottoms out at exitProcess(1) with the recovery - // message printed. - - describe("composition classifies, handles the final state, and exits 1", () => { - let capturedPrintError: string[]; - - function runComposition(gatewayLogTail: string): { - thrown: unknown; - exitCode: number | null; - } { - capturedPrintError = []; - - const failure = classifyGatewayStartFailure(gatewayLogTail); - - let dockerUnreachable = false; - if (failure.kind === "docker_unreachable") { - dockerUnreachable = true; - } - - let exitCode: number | null = null; - let thrown: unknown = null; - try { - handleFinalGatewayStartFailure({ - retries: 2, - dockerUnreachable, - printError: (m = "") => capturedPrintError.push(m), - collectDiagnostics: () => { - throw new Error("collectDiagnostics must not be called on docker_unreachable"); - }, - cleanupGateway: () => { - throw new Error("cleanupGateway must not be called on docker_unreachable"); - }, - exitProcess: ((code: number) => { - exitCode = code; - throw new Error(`__exit(${code})`); - }) as (code: number) => never, - }); - } catch (err) { - thrown = err; - } - return { thrown, exitCode }; - } - - it("composes through the docker-unreachable path on the macOS Colima signature", () => { - const { thrown, exitCode } = runComposition(DARWIN_DOCKER_UNREACHABLE_OUTPUT); - expect(thrown).toBeInstanceOf(Error); - expect(exitCode).toBe(1); - expect(capturedPrintError.join("\n")).toContain("Docker daemon is not running"); - }); - - it("composes through the docker-unreachable path on the Linux dockerd signature", () => { - const { thrown, exitCode } = runComposition(LINUX_DOCKER_UNREACHABLE_OUTPUT); - expect(thrown).toBeInstanceOf(Error); - expect(exitCode).toBe(1); - expect(capturedPrintError.join("\n")).toContain("Docker daemon is not running"); - }); - - it("does NOT trigger the docker-unreachable path on unrelated start output (negative control)", () => { - // A genuinely-broken-but-not-Docker-unreachable failure must still reach - // the regular failure path (which DOES collect diagnostics and clean - // up). If this test ever flips, the call-site classifier has been made - // too aggressive and would silence real gateway failures behind the - // Docker-recovery message. - capturedPrintError = []; - - const failure = classifyGatewayStartFailure(" k3s: failed to bootstrap helm chart after 90s\n"); - - expect(failure.kind).toBe("unknown"); - - let collectCalls = 0; - let cleanupCalls = 0; - let exitCode: number | null = null; - try { - handleFinalGatewayStartFailure({ - retries: 2, - dockerUnreachable: false, - printError: (m = "") => capturedPrintError.push(m), - collectDiagnostics: () => { - collectCalls += 1; - return ""; - }, - cleanupGateway: () => { - cleanupCalls += 1; - }, - exitProcess: ((code: number) => { - exitCode = code; - throw new Error(`__exit(${code})`); - }) as (code: number) => never, - }); - } catch { - // expected - } - expect(collectCalls).toBeGreaterThan(0); - expect(cleanupCalls).toBeGreaterThan(0); - // The non-Docker-unreachable branch does NOT print the Docker daemon - // recovery message. - expect(capturedPrintError.join("\n")).not.toContain("Docker daemon is not running"); - // exitCode is left null here because the test's exitProcess throws - // and the surrounding handleFinal swallows other branches' exits via - // its caller — the assertion that matters is that diagnostics + cleanup - // happened. - void exitCode; - }); - }); - - // ── Sanity: classifyGatewayStartFailure recognises both signatures ───── - // (Already covered in gateway-start-failure.test.ts; this is a pinning - // assertion for the two strings the legacy script generated, kept here so - // the retirement leaves no implicit reference to those byte sequences.) - - describe("classifyGatewayStartFailure pinning for legacy-script signatures", () => { - it("classifies the macOS Colima signature as docker_unreachable", () => { - expect(classifyGatewayStartFailure(DARWIN_DOCKER_UNREACHABLE_OUTPUT)).toEqual({ - kind: "docker_unreachable", - }); - }); - - it("classifies the Linux dockerd signature as docker_unreachable", () => { - expect(classifyGatewayStartFailure(LINUX_DOCKER_UNREACHABLE_OUTPUT)).toEqual({ - kind: "docker_unreachable", - }); - }); - }); }); diff --git a/test/onboard-gateway-docker-unreachable.test.ts b/test/onboard-gateway-docker-unreachable.test.ts deleted file mode 100644 index 66c35340127..00000000000 --- a/test/onboard-gateway-docker-unreachable.test.ts +++ /dev/null @@ -1,148 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import assert from "node:assert/strict"; -import { spawnSync } from "node:child_process"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { describe, it } from "vitest"; -import { testTimeoutOptions } from "./helpers/timeouts"; - -describe("startGateway Docker-unreachable fallback (#2347)", () => { - it("fast-fails before health polling or generic cleanup", testTimeoutOptions(20_000), () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gateway-docker-down-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "gateway-docker-down.cjs"); - const tracePath = path.join(tmpDir, "openshell.trace"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - - fs.mkdirSync(fakeBin, { recursive: true }); - fs.writeFileSync(tracePath, ""); - fs.writeFileSync( - path.join(fakeBin, "openshell"), - `#!/usr/bin/env bash -set -euo pipefail -trace="\${NEMOCLAW_FAKE_OPENSHELL_TRACE:?}" -printf "%s\\n" "$*" >> "$trace" - -if [[ "$*" == "--version" ]]; then - printf "openshell 0.0.44\\n" - exit 0 -fi -if [[ "$*" == "gateway --help" ]]; then - printf "Commands: start select info destroy remove\\n" - exit 0 -fi -if [[ "$*" == *"gateway"*"start"* ]]; then - printf "__GATEWAY_START__\\n" >> "$trace" - printf "Error: Failed to create Docker client.\\n" - printf "Socket not found: /var/run/docker.sock\\n" - exit 1 -fi -if [[ "$*" == *"status"* || "$*" == *"gateway"*"info"* ]]; then - printf "HEALTH POLL REACHED\\n" - exit 0 -fi -if [[ "$*" == *"doctor"*"logs"* ]]; then - printf "DOCTOR LOGS REACHED\\n" - exit 0 -fi -if [[ "$*" == *"gateway"*"select"* || "$*" == *"gateway"*"destroy"* || "$*" == *"gateway"*"remove"* ]]; then - exit 0 -fi -exit 0 -`, - { mode: 0o755 }, - ); - - const script = ` -const mod = require("module"); -const origLoad = mod._load; -mod._load = function(req, parent, isMain) { - if (req === "p-retry") { - const pRetry = async (fn, opts) => { - try { - return await fn({ attemptNumber: 1, retriesLeft: 0 }); - } catch (e) { - if (!(e instanceof pRetry.AbortError) && opts && opts.onFailedAttempt) { - opts.onFailedAttempt(Object.assign(e, { attemptNumber: 1, retriesLeft: 0 })); - } - throw e; - } - }; - pRetry.AbortError = class AbortError extends Error {}; - return pRetry; - } - return origLoad.call(this, req, parent, isMain); -}; -Object.defineProperty(process, "platform", { value: "darwin" }); -Object.defineProperty(process, "arch", { value: "x64" }); -const { startGateway } = require(${onboardPath}); -startGateway(null).catch(() => {}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_FAKE_OPENSHELL_TRACE: tracePath, - NEMOCLAW_HEALTH_POLL_COUNT: "5", - NEMOCLAW_NON_INTERACTIVE: "1", - }, - }); - - assert.equal(result.status, 1, `unexpected exit code; stderr:\n${result.stderr}`); - assert.ok( - result.stderr.includes("Docker daemon is not running"), - `expected Docker recovery guidance in stderr:\n${result.stderr}`, - ); - assert.ok( - result.stderr.includes("colima start"), - `expected macOS Docker start hint in stderr:\n${result.stderr}`, - ); - assert.ok( - !result.stdout.includes("Waiting for gateway health"), - `health polling should not start after Docker-unreachable output:\n${result.stdout}`, - ); - assert.ok( - !result.stdout.includes("HEALTH POLL REACHED"), - `gateway status/info probes should not run after Docker-unreachable output:\n${result.stdout}`, - ); - assert.ok( - !result.stderr.includes("Cleaning up failed gateway state"), - `Docker-unreachable failure should skip generic cleanup:\n${result.stderr}`, - ); - assert.ok( - !result.stderr.includes("openshell doctor logs"), - `Docker-unreachable failure should skip generic diagnostics:\n${result.stderr}`, - ); - - const trace = fs.readFileSync(tracePath, "utf8"); - assert.ok(trace.includes("__GATEWAY_START__"), `gateway start marker missing:\n${trace}`); - const postGatewayStartCommands = trace - .split("__GATEWAY_START__\n") - .slice(1) - .join("__GATEWAY_START__\n") - .split("\n") - .map((line) => line.trim()) - .filter(Boolean); - const forbiddenPostStartCommands = postGatewayStartCommands.filter( - (line) => - line === "status" || - /\bgateway\b.*\binfo\b/.test(line) || - /\bdoctor\b.*\blogs\b/.test(line) || - /\bgateway\b.*\b(?:destroy|remove)\b/.test(line), - ); - assert.deepEqual( - forbiddenPostStartCommands, - [], - `forbidden openshell commands ran after Docker-unreachable gateway start:\n${trace}`, - ); - }); -}); diff --git a/test/onboard.test.ts b/test/onboard.test.ts index f0fde47a706..13c04f30bc3 100644 --- a/test/onboard.test.ts +++ b/test/onboard.test.ts @@ -222,12 +222,12 @@ describe("onboard helpers", () => { }); it( - "prints doctor logs automatically when gateway fails to start (#1605)", + "prints owning-deployment guidance when NemoClaw does not launch the gateway (#9120)", testTimeoutOptions(20_000), () => { - // Intentional process-contract coverage: this case verifies the real child exit status and - // stdout/stderr handling across the Node -> shell -> OpenShell adapter boundary. The - // setupInference cases below are unit-shaped and run directly through typed dependencies. + // Intentional process-contract coverage: this case verifies the real + // child exit status and stderr guidance across the Node -> OpenShell + // adapter boundary. const repoRoot = path.join(import.meta.dirname, ".."); const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gateway-diag-")); const fakeBin = path.join(tmpDir, "bin"); @@ -235,59 +235,25 @@ describe("onboard helpers", () => { const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); fs.mkdirSync(fakeBin, { recursive: true }); - // Fake openshell: - // gateway start — emits ANSI color codes + \r\n (mirrors real gateway output), exits 1 - // doctor logs — emits ANSI sequences, an OOMKilled message, and a fake nvapi- credential - // to exercise ANSI stripping and redaction in the doctor-log path + // No gateway metadata is available, so the external-launcher branch + // prints recovery guidance and exits. fs.writeFileSync( path.join(fakeBin, "openshell"), `#!/usr/bin/env bash -if [[ "$*" == *"doctor"*"logs"* ]]; then - printf "\\033[31mERROR\\033[0m k3s cluster crashed: OOMKilled\\r\\n" - printf " Container nemoclaw_k3s ran out of memory\\r\\n" - printf " Gateway auth token: nvapi-fakecredential-9999\\r\\n" - exit 0 -fi -if [[ "$*" == "gateway --help" ]]; then - printf "Commands: start destroy\\n" - exit 0 -fi -if [[ "$*" == *"gateway"*"start"* ]]; then - printf "\\033[33mDeploying\\033[0m gateway nemoclaw...\\r\\n" - printf "\\r\\nWaiting for gateway health...\\r\\n" - exit 1 -fi exit 1 `, { mode: 0o755 }, ); - // Script runs in a child process: patching p-retry to be immediate avoids the - // 10 s + 30 s minTimeout delays, and NEMOCLAW_HEALTH_POLL_COUNT=0 skips the - // health-poll loop so the function throws "Gateway failed to start" on the - // first attempt. With exitOnFailure:true the catch block should auto-print - // doctor logs to stderr and then call process.exit(1). + // FreeBSD selects the external gateway launcher branch in the current + // provider plan. const script = ` -const mod = require("module"); -const origLoad = mod._load; -mod._load = function(req, parent, isMain) { - if (req === "p-retry") { - return async (fn, opts) => { - try { - return await fn({ attemptNumber: 1, retriesLeft: 0 }); - } catch (e) { - if (opts && opts.onFailedAttempt) { - opts.onFailedAttempt(Object.assign(e, { attemptNumber: 1, retriesLeft: 0 })); - } - throw e; - } - }; - } - return origLoad.call(this, req, parent, isMain); -}; Object.defineProperty(process, "platform", { value: "freebsd" }); const { startGateway } = require(${onboardPath}); -startGateway(null).catch(() => {}); +startGateway(null).catch((error) => { + console.error(error); + process.exitCode = 1; +}); `; fs.writeFileSync(scriptPath, script); @@ -307,48 +273,10 @@ startGateway(null).catch(() => {}); // The process exits 1 because startGateway calls process.exit(1) on failure. assert.equal(result.status, 1, `unexpected exit code; stderr:\n${result.stderr}`); - // Fix 3: doctor logs are auto-printed to stderr. - assert.ok( - result.stderr.includes("Gateway logs:"), - `expected "Gateway logs:" header in stderr:\n${result.stderr}`, - ); - assert.ok( - result.stderr.includes("OOMKilled"), - `expected doctor log output in stderr:\n${result.stderr}`, - ); - - // ANSI sequences must be stripped from both stdout (gateway start output) and - // stderr (doctor logs). A raw \x1b in the output means the regex failed. - assert.ok( - !result.stdout.includes("\x1b"), - `unexpected ANSI escape in stdout:\n${result.stdout}`, - ); - assert.ok( - !result.stderr.includes("\x1b"), - `unexpected ANSI escape in stderr:\n${result.stderr}`, - ); - - // Credentials in doctor logs must be redacted, never printed verbatim. - assert.ok( - !result.stderr.includes("nvapi-fakecredential-9999"), - `credential leaked verbatim in stderr:\n${result.stderr}`, - ); - - // Fix 2: the \r\n -> \naiting rendering artifact must not appear. - assert.ok( - !result.stdout.includes("\naiting"), - `\\naiting artifact present in stdout:\n${result.stdout}`, - ); - - // Fix 1: gateway start output is printed per-line under the header, not as - // one collapsed blob. "Deploying" and "Waiting" must appear on separate lines. - const gatewayLines = result.stdout - .split("\n") - .filter((l) => l.includes("Deploying") || l.includes("Waiting")); - assert.ok( - gatewayLines.length >= 2, - `expected "Deploying" and "Waiting" on separate lines in stdout:\n${result.stdout}`, - ); + assert.match(result.stderr, /does not start the 'nemoclaw' gateway on this host/u); + assert.match(result.stderr, /deployment that owns the gateway process/u); + assert.match(result.stderr, /openshell gateway select nemoclaw/u); + assert.doesNotMatch(result.stderr, /openshell gateway start/u); }, ); diff --git a/test/strict-tool-call-probe.test.ts b/test/strict-tool-call-probe.test.ts index 80fe445cf1b..fe5b3d9fd01 100644 --- a/test/strict-tool-call-probe.test.ts +++ b/test/strict-tool-call-probe.test.ts @@ -20,9 +20,9 @@ import { testTimeoutOptions } from "./helpers/timeouts"; // // Why subprocess: the validation path drives `curl` via spawnSync with a // tight process timeout. Driving the entire scenario set through a fresh -// source-hooked child mirrors the legacy script (and #5119's -// onboard-gateway-docker-unreachable.test.ts) and keeps the behavior under -// test identical to production runtime conditions — bypassing Vitest's +// source-hooked child mirrors the legacy script and the caller-level +// onboarding process tests. It keeps the behavior under test identical to +// production runtime conditions — bypassing Vitest's // worker pool, fetch shim, and signal handling, all of which can interfere // with the in-process curl subprocess used by validateOpenAiLikeSelection. // From ad46a97a3967a741e057a5fb8b9d242c05c1aaec Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 14 Aug 2026 10:16:26 -0700 Subject: [PATCH 3/5] fix(gateway): honor declared lifecycle authority in guidance Signed-off-by: Prekshi Vyas --- ci/source-architecture-budget.json | 4 +- docs/get-started/prerequisites.mdx | 2 +- docs/reference/cli-selection-guide.mdx | 4 +- docs/reference/commands.mdx | 2 +- src/lib/gateway-start-guidance.test.ts | 34 +++- src/lib/gateway-start-guidance.ts | 54 +++++- src/lib/onboard.ts | 13 +- .../gateway-start-failure-integration.test.ts | 158 ------------------ src/lib/onboard/gateway-start-failure.test.ts | 48 +++--- src/lib/onboard/gateway-start-failure.ts | 82 --------- test/e2e/e2e-cloud-experimental/check-docs.sh | 2 +- test/gateway-final-failure-cleanup.test.ts | 115 ------------- 12 files changed, 109 insertions(+), 409 deletions(-) delete mode 100644 src/lib/onboard/gateway-start-failure-integration.test.ts delete mode 100644 test/gateway-final-failure-cleanup.test.ts diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 418bef80137..26d55a168d5 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -17,7 +17,7 @@ "src/lib/core/json-types.ts": 37, "src/lib/core/ports.ts": 89, "src/lib/core/shell-quote.ts": 28, - "src/lib/core/url-utils.ts": 28, + "src/lib/core/url-utils.ts": 27, "src/lib/core/wait.ts": 35, "src/lib/credentials/store.ts": 46, "src/lib/inference/config.ts": 30, @@ -25,7 +25,7 @@ "src/lib/messaging/channels/index.ts": 25, "src/lib/onboard/gateway-binding.ts": 52, "src/lib/runner.ts": 87, - "src/lib/security/redact.ts": 53, + "src/lib/security/redact.ts": 52, "src/lib/state/onboard-session.ts": 36, "src/lib/state/registry.ts": 101, "src/lib/state/state-root.ts": 21, diff --git a/docs/get-started/prerequisites.mdx b/docs/get-started/prerequisites.mdx index a11a69a1a6f..61b53b8d317 100644 --- a/docs/get-started/prerequisites.mdx +++ b/docs/get-started/prerequisites.mdx @@ -79,7 +79,7 @@ docker info For NemoClaw-managed environments, use `$$nemoclaw onboard` when you need to create or recreate the OpenShell gateway or sandbox. -Avoid `openshell self-update`, `npm update -g openshell`, `openshell gateway start --recreate`, or `openshell sandbox create` directly unless you intend to manage OpenShell separately and then rerun `$$nemoclaw onboard`. +Avoid `openshell self-update`, `npm update -g openshell`, or `openshell sandbox create` directly unless you intend to manage OpenShell separately and then rerun `$$nemoclaw onboard`. diff --git a/docs/reference/cli-selection-guide.mdx b/docs/reference/cli-selection-guide.mdx index c10be5e3314..85179290c36 100644 --- a/docs/reference/cli-selection-guide.mdx +++ b/docs/reference/cli-selection-guide.mdx @@ -167,8 +167,8 @@ This section covers common decisions when using the NemoClaw CLI and the OpenShe Use `$$nemoclaw onboard`. It starts the OpenShell gateway when needed, registers providers, builds the selected agent sandbox image, applies NemoClaw policy choices, and creates the sandbox. -Avoid running `openshell gateway start --recreate` or `openshell sandbox create` directly for NemoClaw-managed sandboxes. -Those commands do not update NemoClaw's registry, session metadata, workspace-preservation flow, or agent-specific configuration. +The OpenShell CLI does not provide a command that starts a gateway. +Avoid running `openshell sandbox create` directly for NemoClaw-managed sandboxes because it does not update NemoClaw's registry, session metadata, workspace-preservation flow, or agent-specific configuration. ### Connect to the Sandbox diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index c139919cbf7..912120c06fb 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -542,7 +542,7 @@ Only `--fresh` also discards the saved onboarding session; the refresh environme For NemoClaw-managed environments, use `$$nemoclaw onboard` when you need to create or recreate the OpenShell gateway or sandbox. -Avoid `openshell self-update`, `npm update -g openshell`, `openshell gateway start --recreate`, or `openshell sandbox create` directly unless you intend to manage OpenShell separately and then rerun `$$nemoclaw onboard`. +Avoid `openshell self-update`, `npm update -g openshell`, or `openshell sandbox create` directly unless you intend to manage OpenShell separately and then rerun `$$nemoclaw onboard`. Use `--fresh` to ignore any saved onboarding session and restart the wizard from scratch. This is useful after an interrupted `$$nemoclaw onboard` run when you want to discard saved state instead of continuing it with `--resume`. diff --git a/src/lib/gateway-start-guidance.test.ts b/src/lib/gateway-start-guidance.test.ts index ffdf0adfc60..8b8e4705269 100644 --- a/src/lib/gateway-start-guidance.test.ts +++ b/src/lib/gateway-start-guidance.test.ts @@ -4,8 +4,22 @@ import { describe, expect, it } from "vitest"; import { gatewayStartGuidance, resolveGatewayLauncher } from "./gateway-start-guidance"; +import type { GatewayManagementDeclaration } from "./onboard/gateway-management"; import { resolveCurrentOpenShellComputePlan } from "./onboard/compute/plan"; +const externallySupervisedDeclaration: GatewayManagementDeclaration = { + version: 1, + mode: "externally-supervised", + endpoint: "http://127.0.0.1:8080", + stateDir: "/var/lib/openshell", + supervisor: { + kind: "systemd-system", + serviceName: "openshell-gateway.service", + execPath: "/usr/bin/openshell-gateway", + }, + requiredCapabilities: [], +}; + describe("gatewayStartGuidance", () => { it("names the NemoClaw command where NemoClaw launches the gateway", () => { expect(gatewayStartGuidance("nemoclaw", "nemoclaw")).toBe( @@ -33,13 +47,27 @@ describe("gatewayStartGuidance", () => { }); it("reads the launcher the current runtime provider records", () => { - expect(resolveGatewayLauncher({ gatewayLauncher: "openshell" })).toBe("openshell"); - expect(resolveGatewayLauncher()).toBe(resolveCurrentOpenShellComputePlan().gatewayLauncher); + expect( + resolveGatewayLauncher({ plan: { gatewayLauncher: "openshell" }, declaration: null }), + ).toBe("openshell"); + expect(resolveGatewayLauncher({ declaration: null })).toBe( + resolveCurrentOpenShellComputePlan().gatewayLauncher, + ); + }); + + it("honors external lifecycle authority on a Docker-managed runtime", () => { + expect( + resolveGatewayLauncher({ + gatewayName: "nemoclaw", + plan: { gatewayLauncher: "nemoclaw" }, + declaration: externallySupervisedDeclaration, + }), + ).toBe("openshell"); }); it("resolves the launcher from the plan when the caller supplies no override", () => { expect(gatewayStartGuidance("nemoclaw")).toBe( - gatewayStartGuidance("nemoclaw", resolveGatewayLauncher()), + gatewayStartGuidance("nemoclaw", resolveGatewayLauncher({ gatewayName: "nemoclaw" })), ); }); }); diff --git a/src/lib/gateway-start-guidance.ts b/src/lib/gateway-start-guidance.ts index 6361cfdd5b3..500b47018a3 100644 --- a/src/lib/gateway-start-guidance.ts +++ b/src/lib/gateway-start-guidance.ts @@ -2,19 +2,57 @@ // SPDX-License-Identifier: Apache-2.0 import { CLI_DISPLAY_NAME, CLI_NAME } from "./cli/branding"; +import type { GatewayManagementDeclaration } from "./onboard/gateway-management"; import type { OpenShellGatewayLauncher } from "./onboard/compute/plan"; export type { OpenShellGatewayLauncher }; +export interface ResolveGatewayLauncherOptions { + gatewayName?: string; + plan?: { gatewayLauncher: OpenShellGatewayLauncher }; + /** Validated declaration supplied by tests or an in-process caller. */ + declaration?: GatewayManagementDeclaration | null; +} + /** - * Read the launcher the current runtime provider records. The provider registry - * is loaded on demand so guidance stays importable from the credential and - * inventory command paths without pulling the onboarding graph into them. + * Resolve the component that owns startup for one gateway. A validated + * lifecycle declaration is authoritative over the runtime provider: an + * externally supervised gateway stays owned by its deployment even on a + * platform whose default provider lets NemoClaw launch gateways. Modules are + * loaded on demand so credential and inventory commands do not pull in the + * onboarding graph merely to render a healthy result. */ -export function resolveGatewayLauncher(plan?: { - gatewayLauncher: OpenShellGatewayLauncher; -}): OpenShellGatewayLauncher { - if (plan) return plan.gatewayLauncher; +export function resolveGatewayLauncher( + options: ResolveGatewayLauncherOptions = {}, +): OpenShellGatewayLauncher { + const gatewayManagement = + require("./onboard/gateway-management") as typeof import("./onboard/gateway-management"); + const declaration = Object.hasOwn(options, "declaration") + ? (options.declaration ?? null) + : (() => { + const loaded = gatewayManagement.loadGatewayManagementDeclaration(); + if (!loaded.ok) { + throw gatewayManagement.invalidGatewayManagementDeclarationError(loaded.reason); + } + return loaded.declaration; + })(); + if (declaration) { + const { isExternallySupervised, resolveGatewayOwner } = + require("./onboard/gateway-ownership") as typeof import("./onboard/gateway-ownership"); + const endpoint = declaration.endpoint ? new URL(declaration.endpoint) : null; + const gatewayPort = endpoint + ? Number(endpoint.port || (endpoint.protocol === "https:" ? 443 : 80)) + : 0; + const owner = resolveGatewayOwner({ + gatewayName: options.gatewayName ?? "", + gatewayPort, + declaration, + hasPackagedService: false, + }); + if (isExternallySupervised(owner)) return "openshell"; + } + + if (options.plan) return options.plan.gatewayLauncher; const { resolveCurrentOpenShellComputePlan } = require("./onboard/compute/plan") as typeof import("./onboard/compute/plan"); return resolveCurrentOpenShellComputePlan().gatewayLauncher; @@ -32,7 +70,7 @@ export function resolveGatewayLauncher(plan?: { */ export function gatewayStartGuidance( gatewayName?: string, - launcher: OpenShellGatewayLauncher = resolveGatewayLauncher(), + launcher: OpenShellGatewayLauncher = resolveGatewayLauncher({ gatewayName }), ): string { if (launcher === "nemoclaw") { return `Start the gateway again with \`${CLI_NAME} onboard\`.`; diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 4e29cb2b85c..1d79dd18b90 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -520,7 +520,7 @@ const { trackChildExit } = require("./onboard/child-exit-tracker") as typeof import("./onboard/child-exit-tracker"); const { reportDockerDriverGatewayStartFailure: reportGatewayFailure } = require("./onboard/docker-driver-gateway-failure") as typeof import("./onboard/docker-driver-gateway-failure"); -const { createFinalGatewayStartFailureHandler, normalizeGatewayStartError } = +const { normalizeGatewayStartError } = require("./onboard/gateway-start-failure") as typeof import("./onboard/gateway-start-failure"); const dockerDriverGatewayEnv: typeof import("./onboard/docker-driver-gateway-env") = require("./onboard/docker-driver-gateway-env"); @@ -1213,16 +1213,6 @@ function destroyGateway( }); } -const handleFinalGatewayStartFailure = createFinalGatewayStartFailureHandler({ - getGatewayName: () => GATEWAY_NAME, - collectDiagnostics: () => - runCaptureOpenshell(["doctor", "logs", "--name", GATEWAY_NAME], { - ignoreError: true, - timeout: 10_000, - }), - cleanupGateway: destroyGateway, -}); - function getGatewayClusterContainerState(): string { const containerName = getGatewayClusterContainerName(GATEWAY_NAME); const state = dockerContainerInspectFormat( @@ -4182,7 +4172,6 @@ module.exports = { isDockerDriverGatewayHttpReady, isGatewayHttpReady, waitForGatewayHttpReady, - handleFinalGatewayStartFailure, getNavigationChoice, getSandboxInferenceConfig, getInstalledOpenshellVersion, diff --git a/src/lib/onboard/gateway-start-failure-integration.test.ts b/src/lib/onboard/gateway-start-failure-integration.test.ts deleted file mode 100644 index 68e0ba78a8a..00000000000 --- a/src/lib/onboard/gateway-start-failure-integration.test.ts +++ /dev/null @@ -1,158 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -// -// Integration tests for the docker-unreachable abort path the Docker-driver -// gateway start takes when the gateway log reports the Docker daemon is not -// reachable. -// -// This helper-level suite preserves the former shell regression, which was -// structurally a Node-process unit test of startGateway() with a PATH-shimmed -// openshell binary, not a sandbox-lifecycle E2E. -// -// Original regression: NemoClaw #2347. -// Owning migration issue: NemoClaw #4355. -// -// Coverage strategy: prove the exported helper contract here. The production -// Docker-driver composition is exercised through -// docker-driver-gateway-failure.test.ts instead of copying its control flow -// into this test. - -import { describe, expect, it, vi } from "vitest"; -import { - createFinalGatewayStartFailureHandler, - printDockerDaemonRecovery, -} from "./gateway-start-failure"; - -// The production binding itself remains covered by -// test/gateway-final-failure-cleanup.test.ts. These helper checks only need the -// production factory, and should not load onboard.ts's -// full dependency graph for every source-test worker. -const handleFinalGatewayStartFailure = createFinalGatewayStartFailureHandler({ - getGatewayName: () => "nemoclaw", - collectDiagnostics: () => "", - cleanupGateway: () => undefined, -}); - -describe("startGatewayWithOptions docker-unreachable abort (#2347)", () => { - // ── Layer 1: unit tests of the platform-branching recovery message ──────── - - describe("printDockerDaemonRecovery platform branches", () => { - it("prints the macOS/colima recovery hint when platform=darwin", () => { - const printed: string[] = []; - printDockerDaemonRecovery((message = "") => printed.push(message), "darwin"); - const joined = printed.join("\n"); - expect(joined).toContain("Docker daemon is not running"); - expect(joined).toContain("colima start"); - expect(joined).not.toContain("systemctl"); - }); - - it("prints the Linux/systemctl recovery hint when platform=linux", () => { - const printed: string[] = []; - printDockerDaemonRecovery((message = "") => printed.push(message), "linux"); - const joined = printed.join("\n"); - expect(joined).toContain("Docker daemon is not running"); - expect(joined).toContain("sudo systemctl start docker"); - expect(joined).not.toContain("colima start"); - }); - - it("prints a platform-neutral fallback hint on other platforms", () => { - const printed: string[] = []; - printDockerDaemonRecovery((message = "") => printed.push(message), "win32"); - const joined = printed.join("\n"); - expect(joined).toContain("Docker daemon is not running"); - expect(joined).toContain("Start the Docker daemon"); - expect(joined).not.toContain("colima start"); - expect(joined).not.toContain("systemctl"); - }); - - it("prints the rootless-Podman resume hint when portable=true (#9035)", () => { - const printed: string[] = []; - printDockerDaemonRecovery((message = "") => printed.push(message), "linux", true); - const joined = printed.join("\n"); - expect(joined).toContain("rootless Podman API service is not reachable"); - expect(joined).toContain("Start Podman"); - expect(joined).toContain("nemoclaw onboard --resume"); - expect(joined).not.toContain("nemoclaw onboard --experimental-profile portable"); - expect(joined).not.toContain("sudo systemctl start docker"); - expect(joined).not.toContain("colima start"); - expect(joined).toContain("--resume"); - }); - }); - - // ── Layer 1: handleFinalGatewayStartFailure dockerUnreachable branch ───── - // - // Proves three things at once: - // - exitProcess(1) is called → covers the legacy script's NODE_EXIT==1 - // assertion. - // - collectDiagnostics is NEVER called → covers the legacy script's - // `!grep "openshell doctor logs"` assertion (the script's assertion 7). - // - cleanupGateway is NEVER called → covers the legacy script's implicit - // contract that destroyGateway is not invoked on Docker-unreachable - // (preserving any prior good gateway state for the user). - // - printError is invoked with the recovery guidance → composition with - // printDockerDaemonRecovery. - - describe("handleFinalGatewayStartFailure({dockerUnreachable: true})", () => { - it("calls exitProcess(1) and skips diagnostics + cleanup", () => { - const printError = vi.fn(); - const collectDiagnostics = vi.fn(() => "should-never-be-collected"); - const cleanupGateway = vi.fn(); - const exitProcess = vi.fn((code: number) => { - // Throw so the function's `: never` signature is honored from the - // test's perspective without actually terminating the process. - throw new Error(`__exitProcess(${code})`); - }) as (code: number) => never; - - expect(() => - handleFinalGatewayStartFailure({ - retries: 2, - dockerUnreachable: true, - printError, - collectDiagnostics, - cleanupGateway, - exitProcess, - }), - ).toThrow(/__exitProcess\(1\)/); - - expect(exitProcess).toHaveBeenCalledTimes(1); - expect(exitProcess).toHaveBeenCalledWith(1); - // The crucial behavioural difference from the non-Docker-unreachable - // path: no doctor logs are collected and no cleanup is attempted. - expect(collectDiagnostics).not.toHaveBeenCalled(); - expect(cleanupGateway).not.toHaveBeenCalled(); - - const printed = printError.mock.calls.map((c) => String(c[0] ?? "")).join("\n"); - expect(printed).toContain("Docker daemon is not running"); - }); - - it("DOES collect diagnostics and clean up when dockerUnreachable=false (negative control)", () => { - // Guards against a future refactor that accidentally short-circuits the - // non-Docker-unreachable branch as well. - const printError = vi.fn(); - const collectDiagnostics = vi.fn(() => ""); - const cleanupGateway = vi.fn(); - const exitProcess = vi.fn(() => { - throw new Error("__exitProcess"); - }) as (code: number) => never; - - try { - handleFinalGatewayStartFailure({ - retries: 2, - dockerUnreachable: false, - printError, - collectDiagnostics, - cleanupGateway, - exitProcess, - }); - } catch { - // expected — handleFinal still calls exitProcess on the unhealthy - // (non-Docker-unreachable) branch by way of the surrounding caller; - // here the function returns normally if exitProcess does not throw. - } - - expect(collectDiagnostics).toHaveBeenCalled(); - expect(cleanupGateway).toHaveBeenCalled(); - }); - }); - -}); diff --git a/src/lib/onboard/gateway-start-failure.test.ts b/src/lib/onboard/gateway-start-failure.test.ts index 4be52a82c87..abf26de3467 100644 --- a/src/lib/onboard/gateway-start-failure.test.ts +++ b/src/lib/onboard/gateway-start-failure.test.ts @@ -1,13 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it, vi } from "vitest"; +import { describe, expect, it } from "vitest"; import { classifyGatewayStartFailure } from "../validation"; -import { - createFinalGatewayStartFailureHandler, - normalizeGatewayStartError, -} from "./gateway-start-failure"; +import { normalizeGatewayStartError, printDockerDaemonRecovery } from "./gateway-start-failure"; describe("normalizeGatewayStartError", () => { it("preserves the original deadline-aware Error instance (#3768)", () => { @@ -99,28 +96,31 @@ describe("classifyGatewayStartFailure", () => { }); }); -describe("createFinalGatewayStartFailureHandler", () => { - it("normalizes diagnostics before redacting secrets split by terminal control bytes", () => { +describe("printDockerDaemonRecovery", () => { + it.each([ + ["darwin", "colima start", "systemctl"], + ["linux", "sudo systemctl start docker", "colima start"], + ["win32", "Start the Docker daemon", "systemctl"], + ] as const)("prints the %s recovery command", (platform, expected, excluded) => { + const printed: string[] = []; + + printDockerDaemonRecovery((message = "") => printed.push(message), platform); + + const output = printed.join("\n"); + expect(output).toContain("Docker daemon is not running"); + expect(output).toContain(expected); + expect(output).not.toContain(excluded); + }); + + it("prints the rootless Podman resume command for the portable profile (#9035)", () => { const printed: string[] = []; - const handleFailure = createFinalGatewayStartFailureHandler({ - getGatewayName: () => "nemoclaw-test", - collectDiagnostics: () => "NVIDIA_API_KEY=ghp_abcde\r\x1b[31mfghijklmno\x1b[0m", - cleanupGateway: vi.fn(), - }); - expect(() => - handleFailure({ - retries: 0, - printError: (message = "") => printed.push(message), - exitProcess: (code): never => { - throw new Error(`exit ${code}`); - }, - }), - ).toThrow("exit 1"); + printDockerDaemonRecovery((message = "") => printed.push(message), "linux", true); const output = printed.join("\n"); - expect(output).not.toContain("\x1b"); - expect(output).not.toContain("fghijklmno"); - expect(output).toMatch(/NVIDIA_API_KEY=ghp_\*+/); + expect(output).toContain("rootless Podman API service is not reachable"); + expect(output).toContain("Start Podman"); + expect(output).toContain("nemoclaw onboard --resume"); + expect(output).not.toContain("sudo systemctl start docker"); }); }); diff --git a/src/lib/onboard/gateway-start-failure.ts b/src/lib/onboard/gateway-start-failure.ts index a50a73b2ea2..5367fa0f927 100644 --- a/src/lib/onboard/gateway-start-failure.ts +++ b/src/lib/onboard/gateway-start-failure.ts @@ -1,28 +1,9 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { compactText } from "../core/url-utils"; -import { redact } from "../security/redact"; import { isPortableExperimentalProfile } from "./experimental/portable-profile"; import { onboardResumeRecoveryCommand } from "./resume-hint"; -const ANSI_RE = /\x1B(?:\[[0-?]*[ -/]*[@-~]|\][^\x07]*(?:\x07|\x1B\\)|[@-_])/g; - -export type FinalGatewayStartFailureOptions = { - retries: number; - dockerUnreachable?: boolean; - collectDiagnostics?: () => string | null | undefined; - cleanupGateway?: () => void; - exitProcess?: (code: number) => never; - printError?: (message?: string) => void; -}; - -export type FinalGatewayStartFailureDeps = { - getGatewayName(): string; - collectDiagnostics(): string | null | undefined; - cleanupGateway(): void; -}; - export function normalizeGatewayStartError(error: unknown): Error { return error instanceof Error ? error : new Error(String(error)); } @@ -50,66 +31,3 @@ export function printDockerDaemonRecovery( printError(" Start the Docker daemon."); } } - -export function createFinalGatewayStartFailureHandler(deps: FinalGatewayStartFailureDeps) { - return function handleFinalGatewayStartFailure({ - retries, - dockerUnreachable = false, - collectDiagnostics = deps.collectDiagnostics, - cleanupGateway = deps.cleanupGateway, - exitProcess = (code) => process.exit(code), - printError = (message = "") => console.error(message), - }: FinalGatewayStartFailureOptions): never { - if (dockerUnreachable) { - printDockerDaemonRecovery(printError); - return exitProcess(1); - } - - const gatewayName = deps.getGatewayName(); - printError(` Gateway failed to start after ${retries + 1} attempts.`); - printError(" Gateway state preserved until diagnostics are collected."); - printError(""); - - try { - const normalizedLogs = String(collectDiagnostics() || "") - .replace(/\r/g, "") - .replace(ANSI_RE, ""); - const logs = redact(normalizedLogs); - if (logs) { - printError(" Gateway logs:"); - for (const line of logs.split("\n").filter(Boolean)) { - printError(` ${line}`); - } - printError(""); - } - } catch { - // doctor logs unavailable — continue to best-effort cleanup and manual instructions - } - - printError(" Cleaning up failed gateway state..."); - try { - cleanupGateway(); - printError(" Cleanup attempted."); - } catch (error) { - const message = compactText(error instanceof Error ? error.message : String(error)); - printError(message ? ` Cleanup attempt failed: ${message}` : " Cleanup attempt failed."); - } - printError(""); - printError(" Diagnostic command attempted before cleanup:"); - printError(` openshell doctor logs --name ${gatewayName}`); - printError(" openshell doctor check"); - printError(""); - printError(" If gateway cleanup did not complete, run:"); - printError(` openshell gateway remove ${gatewayName}`); - if (process.platform === "linux") { - printError( - " sudo pkill -f openshell-gateway # if a privileged host gateway process remains", - ); - } - printError( - ` docker volume ls -q --filter "name=openshell-cluster-${gatewayName}" | xargs -r docker volume rm`, - ); - printError(` ${onboardResumeRecoveryCommand()}`); - return exitProcess(1); - }; -} diff --git a/test/e2e/e2e-cloud-experimental/check-docs.sh b/test/e2e/e2e-cloud-experimental/check-docs.sh index c6b592fb83c..3f20bd38e8f 100755 --- a/test/e2e/e2e-cloud-experimental/check-docs.sh +++ b/test/e2e/e2e-cloud-experimental/check-docs.sh @@ -383,7 +383,7 @@ JSON # Scoping rule: inside fenced code blocks (where USAGE lines live like # `[--non-interactive]`), any `--foo` counts. Outside fences, only # backtick-bounded `\`--foo\`` mentions count, so prose references to - # other tools (e.g. `\`openshell gateway start --recreate\``) don't get + # other tools (e.g. `\`openshell sandbox create --from\``) don't get # mistaken for nemoclaw flag documentation. local _doc_flags _doc_flags="$( diff --git a/test/gateway-final-failure-cleanup.test.ts b/test/gateway-final-failure-cleanup.test.ts deleted file mode 100644 index 314b2d038e9..00000000000 --- a/test/gateway-final-failure-cleanup.test.ts +++ /dev/null @@ -1,115 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { describe, expect, it } from "vitest"; - -type FinalGatewayStartFailureOptions = { - retries: number; - collectDiagnostics?: () => string | null | undefined; - cleanupGateway?: () => void; - exitProcess?: (code: number) => never; - printError?: (message?: string) => void; -}; - -type OnboardGatewayFailureInternals = { - handleFinalGatewayStartFailure: (options: FinalGatewayStartFailureOptions) => never; -}; - -function isOnboardGatewayFailureInternals( - value: object | null, -): value is OnboardGatewayFailureInternals { - return ( - value !== null && typeof Reflect.get(value, "handleFinalGatewayStartFailure") === "function" - ); -} - -const loadedOnboardInternals = require("../src/lib/onboard"); -const onboardInternals = - typeof loadedOnboardInternals === "object" && loadedOnboardInternals !== null - ? loadedOnboardInternals - : null; -if (!isOnboardGatewayFailureInternals(onboardInternals)) { - throw new Error("Expected onboard internals to expose handleFinalGatewayStartFailure"); -} -const { handleFinalGatewayStartFailure } = onboardInternals; - -describe("final gateway startup failure cleanup", () => { - it("collects diagnostics before cleanup, then exits", () => { - const calls: string[] = []; - const errors: string[] = []; - - expect(() => - handleFinalGatewayStartFailure({ - retries: 2, - collectDiagnostics: () => { - calls.push("diagnostics"); - return "gateway log line\n"; - }, - cleanupGateway: () => { - calls.push("cleanup"); - }, - exitProcess: (code: number): never => { - calls.push(`exit:${code}`); - throw new Error(`exit ${code}`); - }, - printError: (message = "") => { - errors.push(message); - }, - }), - ).toThrow("exit 1"); - - expect(calls).toEqual(["diagnostics", "cleanup", "exit:1"]); - expect(errors).toContain(" Gateway failed to start after 3 attempts."); - expect(errors).toContain(" Gateway logs:"); - expect(errors).toContain(" gateway log line"); - expect(errors).toContain(" Cleanup attempted."); - expect(errors).toContain(" openshell gateway remove nemoclaw"); - // OpenShell rejects the pre-0.0.44 `gateway destroy` verb, so the command - // fails on every OpenShell version NemoClaw installs (#8139). - expect(errors.join("\n")).not.toContain("gateway destroy"); - if (process.platform === "linux") { - expect(errors).toContain( - " sudo pkill -f openshell-gateway # if a privileged host gateway process remains", - ); - } else { - expect(errors).not.toContain( - " sudo pkill -f openshell-gateway # if a privileged host gateway process remains", - ); - } - expect(errors).toContain( - ' docker volume ls -q --filter "name=openshell-cluster-nemoclaw" | xargs -r docker volume rm', - ); - expect(errors).toContain(" nemoclaw onboard --resume"); - }); - - it("still cleans up after the diagnostic command is attempted and fails", () => { - const calls: string[] = []; - const errors: string[] = []; - - expect(() => - handleFinalGatewayStartFailure({ - retries: 0, - collectDiagnostics: () => { - calls.push("diagnostics"); - throw new Error("doctor unavailable"); - }, - cleanupGateway: () => { - calls.push("cleanup"); - }, - exitProcess: (code: number): never => { - calls.push(`exit:${code}`); - throw new Error(`exit ${code}`); - }, - printError: (message = "") => { - errors.push(message); - }, - }), - ).toThrow("exit 1"); - - expect(calls).toEqual(["diagnostics", "cleanup", "exit:1"]); - expect(errors).toContain(" Gateway failed to start after 1 attempts."); - expect(errors).toContain(" Diagnostic command attempted before cleanup:"); - expect(errors).toContain(" openshell doctor logs --name nemoclaw"); - expect(errors).toContain(" If gateway cleanup did not complete, run:"); - }); -}); From dc17a828c10dd8063cadf1d63d6c525d04256d53 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 14 Aug 2026 10:25:13 -0700 Subject: [PATCH 4/5] docs(cli): align gateway ownership comment Signed-off-by: Prekshi Vyas --- src/lib/gateway-start-guidance.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/gateway-start-guidance.ts b/src/lib/gateway-start-guidance.ts index 500b47018a3..b259878f0c6 100644 --- a/src/lib/gateway-start-guidance.ts +++ b/src/lib/gateway-start-guidance.ts @@ -61,8 +61,8 @@ export function resolveGatewayLauncher( /** * Name the component that starts the gateway again. The OpenShell CLI has no * command that starts a gateway, so naming one sends the operator to a - * remediation that cannot run. The launcher the runtime provider records - * selects the branch. `nemoclaw` means NemoClaw starts the gateway process. + * remediation that cannot run. The resolved launcher and lifecycle authority + * select the branch. `nemoclaw` means NemoClaw starts the gateway process. * `openshell` means the deployment that created the gateway process still owns * starting it. Callers that know which gateway failed pass its name so the * printed selection command is copyable. Other callers print a template that From 2773e36b2cef5133a24d6fcbbf72160619d2d725 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 14 Aug 2026 11:10:16 -0700 Subject: [PATCH 5/5] test(inventory): isolate gateway guidance resolution Signed-off-by: Prekshi Vyas --- src/lib/inventory/index.test.ts | 2 ++ src/lib/inventory/index.ts | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/lib/inventory/index.test.ts b/src/lib/inventory/index.test.ts index 9660f9c882a..52bb0fa0756 100644 --- a/src/lib/inventory/index.test.ts +++ b/src/lib/inventory/index.test.ts @@ -1173,6 +1173,7 @@ describe("inventory commands", () => { state: "named_unreachable", reason: "host port held or container not running", }), + getGatewayStartGuidance: () => "Start the gateway with its lifecycle owner.", log: (message = "") => lines.push(message), }); @@ -1181,6 +1182,7 @@ describe("inventory commands", () => { l.includes("gateway: down [named_unreachable] (host port held or container not running)"), ), ).toBe(true); + expect(lines).toContain(" Start the gateway with its lifecycle owner."); expect(process.exitCode).toBe(1); } finally { process.exitCode = previousExitCode; diff --git a/src/lib/inventory/index.ts b/src/lib/inventory/index.ts index 224fc478bc5..da91901c8af 100644 --- a/src/lib/inventory/index.ts +++ b/src/lib/inventory/index.ts @@ -143,6 +143,8 @@ export interface ShowStatusCommandDeps { * detect the degraded state from `$?` (#3386). */ getGatewayHealth?: () => GatewayHealth; + /** Render lifecycle-aware recovery guidance after an unhealthy gateway probe. */ + getGatewayStartGuidance?: () => string; /** Last authority durably selected by onboarding, with secret-free identity fields. */ getGatewayAuthority?: () => GatewayOwnerDescription | null; checkMessagingBridgeHealth?: ( @@ -591,7 +593,7 @@ export function showStatusCommand(deps: ShowStatusCommandDeps): void { log(""); const detail = health.reason ? ` (${health.reason})` : ""; log(` gateway: down [${health.state}]${detail}`); - log(` ${gatewayStartGuidance()}`); + log(` ${(deps.getGatewayStartGuidance ?? gatewayStartGuidance)()}`); process.exitCode = 1; } }