Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions ci/source-architecture-budget.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,37 +11,37 @@
"src/lib/adapters/openshell/runtime.ts": 54,
"src/lib/adapters/openshell/timeouts.ts": 38,
"src/lib/agent/defs.ts": 33,
"src/lib/cli/branding.ts": 86,
"src/lib/cli/branding.ts": 87,
"src/lib/cli/nemoclaw-oclif-command.ts": 106,
"src/lib/cli/terminal-style.ts": 43,
"src/lib/core/json-types.ts": 37,
"src/lib/core/ports.ts": 89,
"src/lib/core/shell-quote.ts": 28,
"src/lib/core/url-utils.ts": 28,
"src/lib/core/url-utils.ts": 27,
"src/lib/core/wait.ts": 35,
"src/lib/credentials/store.ts": 46,
"src/lib/inference/config.ts": 30,
"src/lib/inference/web-search.ts": 21,
"src/lib/messaging/channels/index.ts": 25,
"src/lib/onboard/gateway-binding.ts": 52,
"src/lib/runner.ts": 88,
"src/lib/security/redact.ts": 53,
"src/lib/runner.ts": 87,
"src/lib/security/redact.ts": 52,
"src/lib/state/onboard-session.ts": 36,
"src/lib/state/registry.ts": 101,
"src/lib/state/state-root.ts": 21,
"src/lib/subprocess-env.ts": 24,
"src/lib/validation.ts": 25
"src/lib/validation.ts": 24
}
},
"fanOut": {
"defaultMax": 20,
"maxByFile": {
"src/lib/actions/inference-set.ts": 32,
"src/lib/actions/sandbox/connect.ts": 39,
"src/lib/actions/sandbox/connect.ts": 40,
"src/lib/actions/sandbox/destroy.ts": 29,
"src/lib/actions/sandbox/doctor.ts": 30,
"src/lib/actions/sandbox/status-snapshot.ts": 20,
"src/lib/actions/sandbox/policy-channel.ts": 29,
"src/lib/actions/sandbox/policy-channel.ts": 30,
"src/lib/actions/sandbox/process-recovery.ts": 21,
"src/lib/actions/sandbox/rebuild-pipeline.ts": 28,
"src/lib/actions/sandbox/snapshot.ts": 40,
Expand All @@ -56,7 +56,7 @@
},
"allowedCycles": [],
"maxRootFiles": {
"src/lib/onboard": 308,
"src/lib/onboard": 307,
"src/lib/actions": 19,
"src/lib/actions/sandbox": 183,
"src/lib/state": 38,
Expand Down
2 changes: 1 addition & 1 deletion docs/deployment/deploy-to-headless-server.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -455,7 +455,7 @@ $$nemoclaw headless-agent recover
```

`recover`, `start`, and `connect --probe-only` do not restart the shared host OpenShell gateway.
If they report a host gateway RPC error, follow the printed `openshell status`, named gateway start, or onboarding guidance.
If they report a host gateway RPC error, follow the printed host gateway recovery guidance.

</AgentOnly>
<AgentOnly variant="deepagents">
Expand Down
2 changes: 1 addition & 1 deletion docs/get-started/prerequisites.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ docker info

<Warning title="OpenShell Lifecycle">
For NemoClaw-managed environments, use `$$nemoclaw onboard` when you need to create or recreate the OpenShell gateway or sandbox.
Avoid `openshell self-update`, `npm update -g openshell`, `openshell gateway start --recreate`, or `openshell sandbox create` directly unless you intend to manage OpenShell separately and then rerun `$$nemoclaw onboard`.
Avoid `openshell self-update`, `npm update -g openshell`, or `openshell sandbox create` directly unless you intend to manage OpenShell separately and then rerun `$$nemoclaw onboard`.
</Warning>

<Note title="Docker Storage Driver">
Expand Down
5 changes: 4 additions & 1 deletion docs/manage-sandboxes/recover-rebuild-sandboxes.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,10 @@ Deployment ordering remains responsible for external changes that OpenShell and
These commands do not restart, replace, or reap the shared host OpenShell gateway process.
If the host gateway RPC returns an error while these sandbox-scoped commands run, they surface the error with explicit next-step guidance and exit rather than swapping the shared gateway out from under other sandboxes.

To repair the host gateway itself, use `openshell status`, `openshell gateway start --name <gateway>`, or run `$$nemoclaw onboard` again.
The OpenShell CLI has no command that starts a gateway, and `openshell status` only reports the gateway state.
If NemoClaw starts the gateway on your host, run `$$nemoclaw onboard` again to repair the host gateway itself.
If a deployment outside NemoClaw owns the gateway process, start the gateway with that deployment, then run `openshell gateway select <gateway>`.
NemoClaw prints the applicable recovery guidance when a sandbox-scoped command reports that the host gateway is down.
Other workflows, including onboarding, rebuild, and `doctor --fix`, can explicitly recover the named host gateway when their operation requires it.
`$$nemoclaw <name> gateway restart` instead restarts only the supported agent gateway inside the named sandbox.
</AgentOnly>
Expand Down
4 changes: 2 additions & 2 deletions docs/reference/cli-selection-guide.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -167,8 +167,8 @@ This section covers common decisions when using the NemoClaw CLI and the OpenShe
Use `$$nemoclaw onboard`.
It starts the OpenShell gateway when needed, registers providers, builds the selected agent sandbox image, applies NemoClaw policy choices, and creates the sandbox.

Avoid running `openshell gateway start --recreate` or `openshell sandbox create` directly for NemoClaw-managed sandboxes.
Those commands do not update NemoClaw's registry, session metadata, workspace-preservation flow, or agent-specific configuration.
The OpenShell CLI does not provide a command that starts a gateway.
Avoid running `openshell sandbox create` directly for NemoClaw-managed sandboxes because it does not update NemoClaw's registry, session metadata, workspace-preservation flow, or agent-specific configuration.

### Connect to the Sandbox

Expand Down
6 changes: 4 additions & 2 deletions docs/reference/commands.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -542,7 +542,7 @@ Only `--fresh` also discards the saved onboarding session; the refresh environme

<Warning>
For NemoClaw-managed environments, use `$$nemoclaw onboard` when you need to create or recreate the OpenShell gateway or sandbox.
Avoid `openshell self-update`, `npm update -g openshell`, `openshell gateway start --recreate`, or `openshell sandbox create` directly unless you intend to manage OpenShell separately and then rerun `$$nemoclaw onboard`.
Avoid `openshell self-update`, `npm update -g openshell`, or `openshell sandbox create` directly unless you intend to manage OpenShell separately and then rerun `$$nemoclaw onboard`.
</Warning>

Use `--fresh` to ignore any saved onboarding session and restart the wizard from scratch. This is useful after an interrupted `$$nemoclaw onboard` run when you want to discard saved state instead of continuing it with `--resume`.
Expand Down Expand Up @@ -3951,7 +3951,9 @@ $$nemoclaw status --json

When at least one sandbox is registered and the named NemoClaw gateway is unreachable, unhealthy, or attached to a different sandbox, the command prints a `gateway: down [state] (reason)` line between the sandbox list and the host-service list.
The command classifies the failing layer when possible: the named gateway port is not accepting connections, the named gateway is running but not Connected, the active OpenShell gateway points at a different name, or the named gateway is not configured at all.
It then suggests `$$nemoclaw onboard --resume` or equivalent managed-gateway recovery guidance.
It then prints the gateway recovery guidance for your host.
That guidance names `$$nemoclaw onboard` when NemoClaw starts the gateway process.
When another deployment owns that process, the guidance directs you to start it with the owning deployment and run `openshell gateway select <gateway>`.
It exits with code `1` so shell scripts and CI can detect the degraded state from `$?`.
For `--json`, the structured output includes `gatewayHealth`, and the exit code is set after the report is generated.
A clean machine with no registered sandboxes keeps the legacy `0` exit because no gateway is expected to be configured yet.
Expand Down
2 changes: 1 addition & 1 deletion docs/reference/troubleshooting.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -2979,7 +2979,7 @@ When shared gateway cleanup would be unsafe, follow the targeted destroy or gate

### `unresolvable CDI devices nvidia.com/gpu=all` during gateway start

Recent NVIDIA Container Toolkit installs configure the Docker daemon for Container Device Interface (CDI) device injection, which OpenShell's `gateway start --gpu` then auto-selects.
Recent NVIDIA Container Toolkit installs configure the Docker daemon for Container Device Interface (CDI) device injection, which a GPU-enabled gateway start then auto-selects.
If no `nvidia.com/gpu` CDI spec has been generated on the host yet, gateway start fails with `Docker responded with status code 500: CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all`.
Outside Station Express, the standard NemoClaw installer detects this gap before onboarding, first tries to enable the NVIDIA CDI refresh systemd units, and can fall back to generating the spec directly with `nvidia-ctk`.
Station Express never falls back to direct CDI generation.
Expand Down
3 changes: 2 additions & 1 deletion src/lib/actions/credentials/list.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { OPENSHELL_OPERATION_TIMEOUT_MS } from "../../adapters/openshell/timeout
import { CLI_NAME } from "../../cli/branding";
import { recoverGatewayOrExit } from "../../credentials/command-support";
import { parseGatewayProviderNames } from "../../credentials/provider-list";
import { gatewayStartGuidance } from "../../gateway-start-guidance";

export type CredentialsListResult = {
exitCode: number;
Expand All @@ -32,7 +33,7 @@ export async function runCredentialsListAction(): Promise<CredentialsListResult>
if (result.status !== 0) {
return fail([
" Could not query OpenShell gateway. Is it running?",
` Run 'openshell gateway start --name nemoclaw' or '${CLI_NAME} onboard' first.`,
` ${gatewayStartGuidance()}`,
]);
}

Expand Down
8 changes: 3 additions & 5 deletions src/lib/actions/sandbox/connect.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import { D, G, R, YW } from "../../cli/terminal-style";
import { spawnExitCode } from "../../core/process-exit";
import { shellQuote } from "../../core/shell-quote";
import { getNamedGatewayLifecycleState } from "../../gateway-runtime-action";
import { gatewayStartGuidance } from "../../gateway-start-guidance";
import {
formatInferenceRouteDriftForDisplay,
parseGatewayInference,
Expand Down Expand Up @@ -382,11 +383,8 @@ function failConnectReadinessGatewayUnavailable(sandboxName: string, detailOutpu
printGatewayLifecycleHint(detailOutput, sandboxName, console.error);
}
console.error(" Recovery:");
console.error(
` 1. Run: openshell gateway start --name ${getSandboxTargetGatewayName(sandboxName)}`,
);
console.error(` 2. If the gateway cannot be restarted, run: ${CLI_NAME} onboard`);
console.error(` 3. Retry: ${CLI_NAME} ${sandboxName} connect`);
console.error(` 1. ${gatewayStartGuidance(getSandboxTargetGatewayName(sandboxName))}`);
console.error(` 2. Retry: ${CLI_NAME} ${sandboxName} connect`);
process.exit(1);
}

Expand Down
5 changes: 2 additions & 3 deletions src/lib/actions/sandbox/doctor-system-checks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import path from "node:path";
import { buildValidatedCurlCommandArgs } from "../../adapters/http/curl-args";
import { stripAnsi } from "../../adapters/openshell/client";
import { CLI_NAME } from "../../cli/branding";
import { gatewayStartGuidance } from "../../gateway-start-guidance";
import { GATEWAY_PORT, OLLAMA_PORT } from "../../core/ports";
import {
CURRENT_RUNTIME_PROVIDER_BUNDLES,
Expand Down Expand Up @@ -39,9 +40,7 @@ function gatewayContainerCheck(
label: "Docker container",
status: running && healthy ? "ok" : "fail",
detail: `${containerName} ${running ? "running" : "stopped"} (${health}; ${image})`,
hint: running
? undefined
: `restart the gateway with \`openshell gateway start --name ${options.gatewayName ?? "nemoclaw"}\``,
hint: running ? undefined : gatewayStartGuidance(options.gatewayName ?? "nemoclaw"),
};
}

Expand Down
30 changes: 29 additions & 1 deletion src/lib/actions/sandbox/gateway-state-hints.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,8 @@ describe("printGatewayLifecycleHint multi-instance hints", () => {

const combined = lines.join("\n");
expect(combined).not.toContain("sandbox has no spec");
expect(combined).toContain("openshell gateway start");
expect(combined).toContain("no longer configured or its metadata/runtime has been lost");
expect(combined).toContain("Start the gateway again with `nemoclaw onboard`.");
});

it.each([
Expand Down Expand Up @@ -385,4 +386,31 @@ describe("printGatewayLifecycleHint multi-instance hints", () => {
errorSpy.mockRestore();
exitSpy.mockRestore();
});

it("names a command that exists when a sandbox-scoped command observes a stopped gateway", async () => {
captureOpenshellSpy.mockReturnValue({
status: 1,
output: "transport error\ntcp connect error\nConnection refused (os error 61)",
});
const lines: string[] = [];
const errorSpy = vi.spyOn(console, "error").mockImplementation((line = "") => {
lines.push(String(line));
});
const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => {
throw new Error(`process.exit(${code ?? 0})`);
}) as never);

await expect(
gatewayState.ensureLiveSandboxOrExit("instance-a", { gatewayRecovery: "observe" }),
).rejects.toThrow("process.exit(1)");

const output = lines.join("\n");
expect(output).toContain("This sandbox-scoped command will not restart the shared host gateway");
expect(output).toContain("Start the gateway again with `nemoclaw onboard`.");
expect(output).not.toContain("openshell gateway start");
expect(recoverNamedGatewayRuntimeSpy).not.toHaveBeenCalled();
expect(exitSpy).toHaveBeenCalledWith(1);
errorSpy.mockRestore();
exitSpy.mockRestore();
});
});
17 changes: 7 additions & 10 deletions src/lib/actions/sandbox/gateway-state.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import {
getNamedGatewayLifecycleState,
recoverNamedGatewayRuntime,
} from "../../gateway-runtime-action";
import { gatewayStartGuidance } from "../../gateway-start-guidance";
import { assertNoOpenShellGatewayEndpointOverride } from "../../openshell-gateway-endpoint-guard";
import { isTerminalSandboxPhase, parseSandboxPhase } from "../../state/gateway";
import { selectSandboxOwningGateway } from "./gateway-select";
Expand Down Expand Up @@ -479,9 +480,7 @@ export function printGatewayLifecycleHint(
writer(
` The selected ${CLI_DISPLAY_NAME} gateway is no longer configured or its metadata/runtime has been lost.`,
);
writer(
` Start the gateway again with \`openshell gateway start --name ${targetGatewayName}\` before expecting existing sandboxes to reconnect.`,
);
writer(` ${gatewayStartGuidance(targetGatewayName)}`);
writer(
" If the gateway has to be rebuilt from scratch, recreate the affected sandbox afterward.",
);
Expand All @@ -492,11 +491,11 @@ export function printGatewayLifecycleHint(
gatewayNamePattern(targetGatewayName).test(cleanOutput)
) {
writer(
" The selected NemoClaw gateway exists in metadata, but its API is refusing connections after restart.",
" The target OpenShell gateway exists in metadata, but its API is refusing connections after restart.",
);
writer(" This usually means the gateway runtime did not come back cleanly after the restart.");
writer(
` Retry \`openshell gateway start --name ${targetGatewayName}\`; if it stays in this state, rebuild the gateway before expecting existing sandboxes to reconnect.`,
` ${gatewayStartGuidance(targetGatewayName)} If the gateway stays in this state, rebuild it before expecting existing sandboxes to reconnect.`,
Comment thread
coderabbitai[bot] marked this conversation as resolved.
);
return;
}
Expand Down Expand Up @@ -783,7 +782,7 @@ export async function ensureLiveSandboxOrExit(
console.error(lookup.output);
}
console.error(
` Retry \`openshell gateway start --name ${getSandboxTargetGatewayName(sandboxName)}\` and verify \`openshell status\` is healthy before reconnecting.`,
` ${gatewayStartGuidance(getSandboxTargetGatewayName(sandboxName))} Check that \`openshell status\` reports the gateway healthy before reconnecting.`,
);
console.error(
" If the gateway never becomes healthy, rebuild the gateway and then recreate the affected sandbox.",
Expand All @@ -799,7 +798,7 @@ export async function ensureLiveSandboxOrExit(
}
printGatewayLifecycleHint(lookup.output, sandboxName);
console.error(
` This sandbox-scoped command will not restart the shared host gateway. Run \`openshell status\` and \`openshell gateway start --name ${getSandboxTargetGatewayName(sandboxName)}\` before retrying.`,
` This sandbox-scoped command will not restart the shared host gateway. ${gatewayStartGuidance(getSandboxTargetGatewayName(sandboxName))} Then retry this command.`,
);
process.exit(1);
}
Expand All @@ -810,9 +809,7 @@ export async function ensureLiveSandboxOrExit(
if (lookup.output) {
console.error(lookup.output);
}
console.error(
` Start the gateway again with \`openshell gateway start --name ${getSandboxTargetGatewayName(sandboxName)}\` before retrying.`,
);
console.error(` ${gatewayStartGuidance(getSandboxTargetGatewayName(sandboxName))}`);
console.error(
" If the gateway had to be rebuilt from scratch, recreate the affected sandbox afterward.",
);
Expand Down
9 changes: 4 additions & 5 deletions src/lib/actions/sandbox/policy-channel.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import {
parsePolicyAddOptions,
} from "../../domain/policy-channel";
import { recoverNamedGatewayRuntime } from "../../gateway-runtime-action";
import { gatewayStartGuidance } from "../../gateway-start-guidance";
import {
type ChannelManifest,
createBuiltInChannelManifestRegistry,
Expand Down Expand Up @@ -886,8 +887,8 @@ async function applyChannelAddToGatewayAndRegistry(
console.error(
` Could not reach the ${CLI_DISPLAY_NAME} OpenShell gateway. Tokens were staged`,
);
console.error(" in env for this run only — re-run after starting the gateway, or run");
console.error(` 'openshell gateway start --name ${gatewayName}' manually.`);
console.error(" in env for this run only. Rerun after starting the gateway.");
console.error(` ${gatewayStartGuidance(gatewayName)}`);
process.exit(1);
}
try {
Expand Down Expand Up @@ -946,9 +947,7 @@ async function applyChannelRemoveToGatewayAndRegistry(
console.error(
` Could not reach the ${CLI_DISPLAY_NAME} OpenShell gateway to delete the bridge.`,
);
console.error(
` Re-run after starting the gateway, or run 'openshell gateway start --name ${gatewayName}'.`,
);
console.error(` ${gatewayStartGuidance(gatewayName)} Then rerun this command.`);
if (!bestEffort) process.exit(1);
gatewayReachable = false;
residual.push("gateway-providers");
Expand Down
2 changes: 1 addition & 1 deletion src/lib/actions/sandbox/status-flow.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -548,7 +548,7 @@ describe("showSandboxStatus flow", () => {
expect(output).not.toContain("Inference: healthy");
expect(output).toContain("Inference: not verified (gateway/sandbox state not verified)");
expect(output).toContain("gateway is still refusing connections after restart");
expect(output).toContain("Retry `openshell gateway start --name nemoclaw`");
expect(output).toContain("Start the gateway again with `nemoclaw onboard`.");
expect(output).toContain("If the gateway never becomes healthy");
expect(harness.collectSandboxStatusSnapshotSpy).toHaveBeenCalledWith("alpha", {
preflight: {
Expand Down
7 changes: 3 additions & 4 deletions src/lib/actions/sandbox/status-lookup-rendering.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

import { CLI_DISPLAY_NAME, CLI_NAME } from "../../cli/branding";
import { D, R } from "../../cli/terminal-style";
import { gatewayStartGuidance } from "../../gateway-start-guidance";
import { isTerminalSandboxPhase } from "../../state/gateway";
import { getSandboxDockerRuntime } from "./docker-health";
import { isDockerRuntimeDown, printDockerRuntimeDownGuidance } from "./gateway-failure-classifier";
Expand Down Expand Up @@ -170,7 +171,7 @@ async function printGatewayUnreachableAfterRestartLookupStatus({
console.log(lookup.output);
}
console.log(
` Retry \`openshell gateway start --name ${getSandboxTargetGatewayName(sandboxName)}\` and verify \`openshell status\` is healthy before reconnecting.`,
` ${gatewayStartGuidance(getSandboxTargetGatewayName(sandboxName))} Check that \`openshell status\` reports the gateway healthy before reconnecting.`,
);
console.log(
" If the gateway never becomes healthy, rebuild the gateway and then recreate the affected sandbox.",
Expand All @@ -191,9 +192,7 @@ async function printGatewayMissingAfterRestartLookupStatus({
if (lookup.output) {
console.log(lookup.output);
}
console.log(
` Start the gateway again with \`openshell gateway start --name ${getSandboxTargetGatewayName(sandboxName)}\` before retrying.`,
);
console.log(` ${gatewayStartGuidance(getSandboxTargetGatewayName(sandboxName))}`);
console.log(
" If the gateway had to be rebuilt from scratch, recreate the affected sandbox afterward.",
);
Expand Down
Loading
Loading