Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions src/lib/openshell-timeouts.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { describe, expect, it } from "vitest";

import {
OPENSHELL_DOWNLOAD_TIMEOUT_MS,
OPENSHELL_HEAVY_TIMEOUT_MS,
OPENSHELL_OPERATION_TIMEOUT_MS,
OPENSHELL_PROBE_TIMEOUT_MS,
} from "./openshell-timeouts";

describe("openshell-timeouts", () => {
it("exports positive integer constants", () => {
const constants = [
OPENSHELL_PROBE_TIMEOUT_MS,
OPENSHELL_OPERATION_TIMEOUT_MS,
OPENSHELL_HEAVY_TIMEOUT_MS,
OPENSHELL_DOWNLOAD_TIMEOUT_MS,
];

for (const value of constants) {
expect(value).toBeTypeOf("number");
expect(value).toBeGreaterThan(0);
expect(Number.isInteger(value)).toBe(true);
}
});

it("maintains expected ordering: PROBE < OPERATION <= DOWNLOAD < HEAVY", () => {
expect(OPENSHELL_PROBE_TIMEOUT_MS).toBeLessThan(OPENSHELL_OPERATION_TIMEOUT_MS);
expect(OPENSHELL_OPERATION_TIMEOUT_MS).toBeLessThanOrEqual(OPENSHELL_DOWNLOAD_TIMEOUT_MS);
expect(OPENSHELL_DOWNLOAD_TIMEOUT_MS).toBeLessThan(OPENSHELL_HEAVY_TIMEOUT_MS);
});
Comment thread
coderabbitai[bot] marked this conversation as resolved.

it("uses the same probe constant name as PR #2454 for forward compatibility", () => {
// PR #2454 introduces OPENSHELL_PROBE_TIMEOUT_MS = 15_000 locally.
// This ensures the shared module stays aligned so #2454 can import it after rebase.
expect(OPENSHELL_PROBE_TIMEOUT_MS).toBe(15_000);
});
});
28 changes: 28 additions & 0 deletions src/lib/openshell-timeouts.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

/**
* Named timeout constants for openshell child-process execution.
*
* Every openshell CLI call should use one of these categories rather than
* raw millisecond literals. This ensures consistent behaviour across all
* call sites and makes it easy to tune timeouts from a single location.
*
* Categories:
* PROBE — read-only queries that should return instantly (list, status, info, ssh-config)
* OPERATION — mutating commands (provider CRUD, forward start/stop, gateway select)
* HEAVY — destructive or long-running (sandbox delete, gateway destroy, build)
* DOWNLOAD — file transfers over the sandbox SSH tunnel (config download)
*/

/** Quick probe — sandbox list, status, gateway info, forward list, ssh-config */
export const OPENSHELL_PROBE_TIMEOUT_MS = 15_000;

/** Mutating operations — provider create/delete, gateway select, forward start/stop */
export const OPENSHELL_OPERATION_TIMEOUT_MS = 30_000;

/** Heavy operations — sandbox delete, gateway destroy, full build */
export const OPENSHELL_HEAVY_TIMEOUT_MS = 60_000;

/** Sandbox download (config file fetch over SSH) */
export const OPENSHELL_DOWNLOAD_TIMEOUT_MS = 30_000;
80 changes: 65 additions & 15 deletions src/nemoclaw.ts
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,10 @@ import {
knownChannelNames,
persistChannelTokens,
} from "./lib/sandbox-channels";
import {
OPENSHELL_OPERATION_TIMEOUT_MS,
OPENSHELL_PROBE_TIMEOUT_MS,
} from "./lib/openshell-timeouts";
const onboardProviders = require("./lib/onboard-providers");

// ── Global commands (derived from command registry) ──────────────
Expand Down Expand Up @@ -182,7 +186,10 @@ function cleanupGatewayAfterLastSandbox() {
}

function hasNoLiveSandboxes() {
const liveList = captureOpenshell(["sandbox", "list"], { ignoreError: true });
const liveList = captureOpenshell(["sandbox", "list"], {
ignoreError: true,
timeout: OPENSHELL_PROBE_TIMEOUT_MS,
});
if (liveList.status !== 0) {
return false;
}
Expand Down Expand Up @@ -218,6 +225,7 @@ function getInstalledOpenshellVersionOrNull() {
function executeSandboxCommand(sandboxName: string, command: string): SandboxCommandResult | null {
const sshConfigResult = captureOpenshell(["sandbox", "ssh-config", sandboxName], {
ignoreError: true,
timeout: OPENSHELL_PROBE_TIMEOUT_MS,
});
if (sshConfigResult.status !== 0) return null;

Expand Down Expand Up @@ -507,7 +515,10 @@ async function recoverRegistryFromLiveGateway(
}

let recoveredFromGateway = 0;
const liveList = captureOpenshell(["sandbox", "list"], { ignoreError: true });
const liveList = captureOpenshell(["sandbox", "list"], {
ignoreError: true,
timeout: OPENSHELL_PROBE_TIMEOUT_MS,
});
const liveNames = Array.from<string>(parseLiveSandboxNames(liveList.output));
for (const name of liveNames) {
const metadata = metadataByName.get(name) || undefined;
Expand Down Expand Up @@ -569,8 +580,10 @@ function getActiveGatewayName(output = ""): string | null {
}

function getNamedGatewayLifecycleState() {
const status = captureOpenshell(["status"]);
const gatewayInfo = captureOpenshell(["gateway", "info", "-g", "nemoclaw"]);
const status = captureOpenshell(["status"], { timeout: OPENSHELL_PROBE_TIMEOUT_MS });
const gatewayInfo = captureOpenshell(["gateway", "info", "-g", "nemoclaw"], {
timeout: OPENSHELL_PROBE_TIMEOUT_MS,
});
const cleanStatus = stripAnsi(status.output);
const activeGateway = getActiveGatewayName(status.output);
const connected = /^\s*Status:\s*Connected\b/im.test(cleanStatus);
Expand Down Expand Up @@ -625,7 +638,10 @@ async function recoverNamedGatewayRuntime() {
return { recovered: true, before, after: before, attempted: false };
}

runOpenshell(["gateway", "select", "nemoclaw"], { ignoreError: true });
runOpenshell(["gateway", "select", "nemoclaw"], {
ignoreError: true,
timeout: OPENSHELL_OPERATION_TIMEOUT_MS,
});
let after = getNamedGatewayLifecycleState();
if (after.state === "healthy_named") {
process.env.OPENSHELL_GATEWAY = "nemoclaw";
Expand All @@ -643,7 +659,10 @@ async function recoverNamedGatewayRuntime() {
// Fall through to the lifecycle re-check below so we preserve the
// existing recovery result shape and emit the correct classification.
}
runOpenshell(["gateway", "select", "nemoclaw"], { ignoreError: true });
runOpenshell(["gateway", "select", "nemoclaw"], {
ignoreError: true,
timeout: OPENSHELL_OPERATION_TIMEOUT_MS,
});
after = getNamedGatewayLifecycleState();
if (after.state === "healthy_named") {
process.env.OPENSHELL_GATEWAY = "nemoclaw";
Expand All @@ -656,7 +675,9 @@ async function recoverNamedGatewayRuntime() {

/** Query sandbox presence and return its output with the live enforced policy. */
function getSandboxGatewayState(sandboxName: string) {
const result = captureOpenshell(["sandbox", "get", sandboxName]);
const result = captureOpenshell(["sandbox", "get", sandboxName], {
timeout: OPENSHELL_PROBE_TIMEOUT_MS,
});
let output = result.output;
if (result.status === 0) {
// `openshell sandbox get` returns the immutable baseline policy from sandbox
Expand All @@ -666,6 +687,7 @@ function getSandboxGatewayState(sandboxName: string) {
// Sandbox info above it. (#1132)
const livePolicy = captureOpenshell(["policy", "get", "--full", sandboxName], {
ignoreError: true,
timeout: OPENSHELL_PROBE_TIMEOUT_MS,
});
if (livePolicy.status === 0 && livePolicy.output.trim()) {
const rawLines = String(output).split("\n");
Expand Down Expand Up @@ -727,7 +749,10 @@ function reconcileMissingAgainstNamedGateway(
) {
const lifecycle = getNamedGatewayLifecycleState();
if (lifecycle.state === "connected_other") {
runOpenshell(["gateway", "select", "nemoclaw"], { ignoreError: true });
runOpenshell(["gateway", "select", "nemoclaw"], {
ignoreError: true,
timeout: OPENSHELL_OPERATION_TIMEOUT_MS,
});
const retry = getSandboxGatewayState(sandboxName);
if (retry.state === "present") {
return { ...retry, recoveredGateway: true, recoveryVia: "select" };
Expand Down Expand Up @@ -1153,7 +1178,10 @@ function debug(args: string[]) {
);
return undefined;
}
const liveList = captureOpenshell(["sandbox", "list"], { ignoreError: true });
const liveList = captureOpenshell(["sandbox", "list"], {
ignoreError: true,
timeout: OPENSHELL_PROBE_TIMEOUT_MS,
});
if (liveList.status === 0 && !parseLiveSandboxNames(liveList.output).has(defaultSandbox)) {
console.error(
`${_RD}Warning:${R} default sandbox '${defaultSandbox}' exists in the local registry but not in OpenShell.`,
Expand Down Expand Up @@ -1399,15 +1427,21 @@ function makeConflictProbe() {
let gatewayAlive: boolean | null = null;
const isGatewayAlive = (): boolean => {
if (gatewayAlive === null) {
const result = captureOpenshell(["sandbox", "list"], { ignoreError: true });
const result = captureOpenshell(["sandbox", "list"], {
ignoreError: true,
timeout: OPENSHELL_PROBE_TIMEOUT_MS,
});
gatewayAlive = result.status === 0;
}
return gatewayAlive;
};
return {
providerExists: (name: string) => {
if (!isGatewayAlive()) return "error";
const result = captureOpenshell(["provider", "get", name], { ignoreError: true });
const result = captureOpenshell(["provider", "get", name], {
ignoreError: true,
timeout: OPENSHELL_PROBE_TIMEOUT_MS,
});
return result.status === 0 ? "present" : "absent";
},
};
Expand Down Expand Up @@ -1457,7 +1491,12 @@ function showStatus() {
showStatusCommand({
listSandboxes: () => registry.listSandboxes(),
getLiveInference: () =>
parseGatewayInference(captureOpenshell(["inference", "get"], { ignoreError: true }).output),
parseGatewayInference(
captureOpenshell(["inference", "get"], {
ignoreError: true,
timeout: OPENSHELL_PROBE_TIMEOUT_MS,
}).output,
),
showServiceStatus,
checkMessagingBridgeHealth,
backfillAndFindOverlaps,
Expand All @@ -1483,7 +1522,12 @@ async function listSandboxes(): Promise<void> {
await listSandboxesCommand({
recoverRegistryEntries: () => recoverRegistryEntries(),
getLiveInference: () =>
parseGatewayInference(captureOpenshell(["inference", "get"], { ignoreError: true }).output),
parseGatewayInference(
captureOpenshell(["inference", "get"], {
ignoreError: true,
timeout: OPENSHELL_PROBE_TIMEOUT_MS,
}).output,
),
loadLastSession: () => onboardSession.loadSession(),
getActiveSessionCount: sessionDeps
? (name: string) => {
Expand Down Expand Up @@ -1559,7 +1603,10 @@ async function sandboxConnect(
const sb = registry.getSandbox(sandboxName);
if (sb && sb.provider && sb.model) {
const live = parseGatewayInference(
captureOpenshell(["inference", "get"], { ignoreError: true }).output,
captureOpenshell(["inference", "get"], {
ignoreError: true,
timeout: OPENSHELL_PROBE_TIMEOUT_MS,
}).output,
);
if (!live || live.provider !== sb.provider || live.model !== sb.model) {
console.log(
Expand Down Expand Up @@ -1699,7 +1746,10 @@ async function sandboxConnect(
async function sandboxStatus(sandboxName: string) {
const sb = registry.getSandbox(sandboxName);
const live = parseGatewayInference(
captureOpenshell(["inference", "get"], { ignoreError: true }).output,
captureOpenshell(["inference", "get"], {
ignoreError: true,
timeout: OPENSHELL_PROBE_TIMEOUT_MS,
}).output,
);
const currentModel = (live && live.model) || (sb && sb.model) || "unknown";
const currentProvider = (live && live.provider) || (sb && sb.provider) || "unknown";
Expand Down
Loading