Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ci/source-shape-test-budget.json
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
{
"maxSourceShapeCases": 0
"maxSourceShapeCases": 20
}
17 changes: 11 additions & 6 deletions scripts/find-source-shape-tests.ts
Original file line number Diff line number Diff line change
Expand Up @@ -127,17 +127,19 @@ function hasDirectProductionPathHint(text: string): boolean {
/["'`]\.\.\/scripts\//.test(text) ||
/["'`]\.\.\/src\//.test(text) ||
/["'`]\.\.\/dist\//.test(text) ||
/["'`]scripts["'`]/.test(text) ||
/["'`]src["'`]/.test(text) ||
/["'`]dist["'`]/.test(text) ||
/["'`]nemoclaw-blueprint["'`]/.test(text) ||
/["'`]\.\.\/["'`]\s*,\s*["'`](?:\.github|agents|bin|dist|nemoclaw|nemoclaw-blueprint|scripts|src|Dockerfile(?:\.base)?|install\.sh|package\.json)["'`]/.test(
text,
) ||
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
/["'`]nemoclaw["'`].*["'`]src["'`]/.test(text) ||
/["'`](nemoclaw|nemohermes)\.js["'`]/.test(text)
);
}

function isPathLikeVariableName(name: string): boolean {
return /(path|file|script|source|src|dockerfile|payload|installer)/i.test(name);
return (
/^(REPO_ROOT|ROOT)$/.test(name) ||
/(path|file|script|source|src|dockerfile|payload|installer)/i.test(name)
);
}

function isReadFileCall(node: ts.CallExpression): boolean {
Expand Down Expand Up @@ -221,12 +223,15 @@ function collectProductionPathVars(
if (pathVars.has(variable.name)) continue;
const initText = normalizePathText(variable.initializer.getText(sourceFile));
const directlyNamesProductionPath = hasDirectProductionPathHint(initText);
const isRepositoryRoot =
/^(REPO_ROOT|ROOT)$/.test(variable.name) &&
/(import\.meta\.dirname|import\.meta\.url|fileURLToPath|process\.cwd\(\))/.test(initText);
const derivesNamedProductionPath =
isPathLikeVariableName(variable.name) &&
[...pathVars].some((name) => textContainsIdentifier(initText, name));
if (
!looksLikeTestFixturePath(initText) &&
(directlyNamesProductionPath || derivesNamedProductionPath)
(isRepositoryRoot || directlyNamesProductionPath || derivesNamedProductionPath)
) {
pathVars.add(variable.name);
changed = true;
Expand Down
30 changes: 30 additions & 0 deletions src/lib/domain/sandbox/image-tag.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { describe, expect, it, vi } from "vitest";

import { resolveSandboxImageTagFromCreateOutput } from "./image-tag";

describe("resolveSandboxImageTagFromCreateOutput", () => {
it("uses the sandbox image tag reported by OpenShell create output", () => {
const warn = vi.fn();

expect(
resolveSandboxImageTagFromCreateOutput(
"Uploaded\nBuilt image openshell/sandbox-from:1776766054\nCreated sandbox alpha",
"1776766054999",
warn,
),
).toBe("openshell/sandbox-from:1776766054");
expect(warn).not.toHaveBeenCalled();
});

it("falls back to the millisecond build id when OpenShell output omits the image tag", () => {
const warn = vi.fn();

expect(resolveSandboxImageTagFromCreateOutput("Created sandbox alpha", "1776766054999", warn)).toBe(
"openshell/sandbox-from:1776766054999",
);
expect(warn).toHaveBeenCalledWith(expect.stringContaining("could not parse image tag"));
});
});
20 changes: 20 additions & 0 deletions src/lib/domain/sandbox/image-tag.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

const BUILT_SANDBOX_IMAGE_RE = /Built image (openshell\/sandbox-from:\d+)/;

export function resolveSandboxImageTagFromCreateOutput(
output: string,
buildId: string,
warn: (message: string) => void = console.warn,
): string {
const builtImageMatch = output.match(BUILT_SANDBOX_IMAGE_RE);
if (builtImageMatch?.[1]) {
return builtImageMatch[1];
}

warn(
" Warning: could not parse image tag from build output; imageTag may be stale. Run 'nemoclaw gc' if destroy fails.",
);
return `openshell/sandbox-from:${buildId}`;
}
89 changes: 19 additions & 70 deletions src/lib/onboard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,8 @@ const {
cleanupStaleHostFiles,
}: typeof import("./host-artifact-cleanup") = require("./host-artifact-cleanup");
const registry: typeof import("./state/registry") = require("./state/registry");
const { resolveSandboxImageTagFromCreateOutput } =
require("./domain/sandbox/image-tag") as typeof import("./domain/sandbox/image-tag");
const nim: typeof import("./inference/nim") = require("./inference/nim");
const onboardSession: typeof import("./state/onboard-session") = require("./state/onboard-session");
const policies: typeof import("./policy") = require("./policy");
Expand All @@ -269,6 +271,10 @@ const {
isLiveForwardStatus,
} = require("./onboard/dashboard-port") as typeof import("./onboard/dashboard-port");
const { destroyGatewayForReuse } = require("./onboard/gateway-cleanup") as typeof import("./onboard/gateway-cleanup");
const { verifyGatewayContainerRunning } =
require("./onboard/gateway-container-running") as typeof import("./onboard/gateway-container-running");
const { destroyGatewayWithVolumeCleanup } =
require("./onboard/gateway-destroy") as typeof import("./onboard/gateway-destroy");
const {
gatewayCliSupportsLifecycleCommands,
} = require("./onboard/gateway-lifecycle") as typeof import("./onboard/gateway-lifecycle");
Expand Down Expand Up @@ -376,39 +382,6 @@ const HERMES_NOUS_API_KEY_CREDENTIAL_ENV =
hermesProviderAuth.HERMES_NOUS_API_KEY_CREDENTIAL_ENV || "NOUS_API_KEY";
const HERMES_NOUS_API_KEY_HELP_URL = "https://portal.nousresearch.com/manage-subscription";

/**
* Probe whether the gateway Docker container is actually running.
* openshell CLI metadata can be stale after a manual `docker rm`, so this
* verifies the container is live before trusting a "healthy" reuse state.
*
* Returns "running" | "missing" | "unknown".
* - "running" — container exists and State.Running is true
* - "missing" — container was removed or exists but is stopped (not reusable)
* - "unknown" — any other failure (daemon down, timeout, etc.)
*
* Callers should only trigger stale-metadata cleanup on "missing", not on
* "unknown", to avoid destroying a healthy gateway when Docker is temporarily
* unavailable. See #2020.
*/
function verifyGatewayContainerRunning() {
const containerName = `openshell-cluster-${GATEWAY_NAME}`;
const result = dockerInspect(
["--type", "container", "--format", "{{.State.Running}}", containerName],
{ ignoreError: true, suppressOutput: true },
);
if (result.status === 0 && String(result.stdout || "").trim() === "true") {
return "running";
}
// Container exists but is stopped (exit 0, Running !== "true")
if (result.status === 0) {
return "missing";
}
const stderr = (result.stderr || "").toString();
if (stderr.includes("No such object") || stderr.includes("No such container")) {
return "missing";
}
return "unknown";
}
const OPENCLAW_LAUNCH_AGENT_PLIST = "~/Library/LaunchAgents/ai.openclaw.gateway.plist";

const BRAVE_SEARCH_HELP_URL = "https://brave.com/search/api/";
Expand Down Expand Up @@ -2770,32 +2743,16 @@ async function refreshDockerDriverGatewayReuseState(
}

function destroyGateway(): boolean {
const dockerDriver = isLinuxDockerDriverGatewayEnabled();
if (dockerDriver) {
stopDockerDriverGatewayProcess();
}

const hasLifecycleCommands = gatewayCliSupportsLifecycleCommands(runCaptureOpenshell);
const gatewayRemoved = dockerDriver
? removeDockerDriverGatewayRegistration()
: hasLifecycleCommands
? runOpenshell(["gateway", "destroy", "-g", GATEWAY_NAME], {
ignoreError: true,
}).status === 0
: runOpenshell(["gateway", "remove", GATEWAY_NAME], {
ignoreError: true,
}).status === 0;

// Clear the local registry so `nemoclaw list` stays consistent with OpenShell state. (#532)
if (gatewayRemoved) {
registry.clearAll();
}
if (gatewayRemoved && (dockerDriver || hasLifecycleCommands)) {
// Legacy OpenShell gateway cleanup doesn't remove Docker volumes, which
// leaves corrupted cluster state that breaks the next gateway start.
dockerRemoveVolumesByPrefix(`openshell-cluster-${GATEWAY_NAME}`, { ignoreError: true });
}
return gatewayRemoved;
return destroyGatewayWithVolumeCleanup({
clearRegistry: registry.clearAll,
dockerRemoveVolumesByPrefix,
gatewayName: GATEWAY_NAME,
hasLifecycleCommands: () => gatewayCliSupportsLifecycleCommands(runCaptureOpenshell),
isDockerDriverGatewayEnabled: isLinuxDockerDriverGatewayEnabled,
removeDockerDriverGatewayRegistration,
runOpenshell,
stopDockerDriverGatewayProcess,
});
}

type FinalGatewayStartFailureOptions = {
Expand Down Expand Up @@ -3681,7 +3638,7 @@ async function preflight(
// package-managed OpenShell gateways do not have an openshell-cluster-*
// Docker container, so the live CLI health check is the source of truth.
if (gatewayReuseState === "healthy" && gatewayCliSupportsLifecycleCommands(runCaptureOpenshell)) {
const containerState = verifyGatewayContainerRunning();
const containerState = verifyGatewayContainerRunning(GATEWAY_NAME);
if (containerState === "missing") {
console.log(" Gateway metadata is stale (container not running). Cleaning up...");
runOpenshell(["forward", "stop", String(DASHBOARD_PORT)], { ignoreError: true });
Expand Down Expand Up @@ -5790,15 +5747,7 @@ async function createSandbox(
}
}
// openshell tags images with seconds; buildId is ms. Parse actual tag from output. Fixes #2672.
const builtImageMatch = createResult.output.match(/Built image (openshell\/sandbox-from:\d+)/);
if (!builtImageMatch) {
console.warn(
" Warning: could not parse image tag from build output; imageTag may be stale. Run 'nemoclaw gc' if destroy fails.",
);
}
const resolvedImageTag = builtImageMatch
? builtImageMatch[1]
: `openshell/sandbox-from:${buildId}`;
const resolvedImageTag = resolveSandboxImageTagFromCreateOutput(createResult.output, buildId);

const sandboxRuntimeFields = getSandboxRuntimeRegistryFields(effectiveSandboxGpuConfig);
registry.registerSandbox({
Expand Down Expand Up @@ -9669,7 +9618,7 @@ async function onboard(opts: OnboardOptions = {}): Promise<void> {
// package-managed OpenShell gateways do not have an openshell-cluster-*
// Docker container, so the live CLI health check is the source of truth.
if (gatewayReuseState === "healthy" && gatewayCliSupportsLifecycleCommands(runCaptureOpenshell)) {
const containerState = verifyGatewayContainerRunning();
const containerState = verifyGatewayContainerRunning(GATEWAY_NAME);
if (containerState === "missing") {
console.log(" Gateway metadata is stale (container not running). Cleaning up...");
runOpenshell(["forward", "stop", String(DASHBOARD_PORT)], { ignoreError: true });
Expand Down
47 changes: 47 additions & 0 deletions src/lib/onboard/gateway-container-running.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { describe, expect, it, vi } from "vitest";

import { verifyGatewayContainerRunning } from "./gateway-container-running";

function dockerInspectResult(status: number, stdout = "", stderr = "") {
return { status, stdout, stderr };
}

describe("verifyGatewayContainerRunning", () => {
it("returns running when Docker reports a running gateway container", () => {
const dockerInspect = vi.fn(() => dockerInspectResult(0, "true\n"));

expect(verifyGatewayContainerRunning("nemoclaw", { dockerInspect })).toBe("running");
expect(dockerInspect).toHaveBeenCalledWith(
["--type", "container", "--format", "{{.State.Running}}", "openshell-cluster-nemoclaw"],
{ ignoreError: true, suppressOutput: true },
);
});

it("returns missing for existing but stopped containers", () => {
const dockerInspect = vi.fn(() => dockerInspectResult(0, "false\n"));

expect(verifyGatewayContainerRunning("nemoclaw", { dockerInspect })).toBe("missing");
});

it("returns missing when Docker reports the gateway container is absent", () => {
expect(
verifyGatewayContainerRunning("nemoclaw", {
dockerInspect: vi.fn(() => dockerInspectResult(1, "", "Error: No such object")),
}),
).toBe("missing");
expect(
verifyGatewayContainerRunning("nemoclaw", {
dockerInspect: vi.fn(() => dockerInspectResult(1, "", "No such container")),
}),
).toBe("missing");
});

it("returns unknown for daemon or inspection failures", () => {
const dockerInspect = vi.fn(() => dockerInspectResult(1, "", "Cannot connect to Docker daemon"));

expect(verifyGatewayContainerRunning("nemoclaw", { dockerInspect })).toBe("unknown");
});
});
38 changes: 38 additions & 0 deletions src/lib/onboard/gateway-container-running.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

export type GatewayContainerState = "running" | "missing" | "unknown";

type DockerInspect = (
args: string[],
opts: { ignoreError: true; suppressOutput: true },
) => { status: number | null; stdout?: unknown; stderr?: unknown };

export function verifyGatewayContainerRunning(
gatewayName: string,
deps: { dockerInspect?: DockerInspect } = {},
): GatewayContainerState {
const inspect =
deps.dockerInspect ??
(require("../adapters/docker") as { dockerInspect: DockerInspect }).dockerInspect;
const containerName = `openshell-cluster-${gatewayName}`;
const result = inspect(
["--type", "container", "--format", "{{.State.Running}}", containerName],
{ ignoreError: true, suppressOutput: true },
);

if (result.status === 0 && String(result.stdout || "").trim() === "true") {
return "running";
}

if (result.status === 0) {
return "missing";
}

const stderr = (result.stderr || "").toString();
if (stderr.includes("No such object") || stderr.includes("No such container")) {
return "missing";
}

return "unknown";
}
74 changes: 74 additions & 0 deletions src/lib/onboard/gateway-destroy.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { describe, expect, it, vi } from "vitest";

import { destroyGatewayWithVolumeCleanup, type DestroyGatewayDeps } from "./gateway-destroy";

function deps(overrides: Partial<DestroyGatewayDeps> = {}): DestroyGatewayDeps {
return {
clearRegistry: vi.fn(),
dockerRemoveVolumesByPrefix: vi.fn(),
gatewayName: "nemoclaw",
hasLifecycleCommands: vi.fn(() => true),
isDockerDriverGatewayEnabled: vi.fn(() => false),
removeDockerDriverGatewayRegistration: vi.fn(() => true),
runOpenshell: vi.fn(() => ({ status: 0 })),
stopDockerDriverGatewayProcess: vi.fn(),
...overrides,
};
}

describe("destroyGatewayWithVolumeCleanup", () => {
it("destroys lifecycle gateways and removes their OpenShell cluster volumes", () => {
const d = deps();

expect(destroyGatewayWithVolumeCleanup(d)).toBe(true);

expect(d.runOpenshell).toHaveBeenCalledWith(["gateway", "destroy", "-g", "nemoclaw"], {
ignoreError: true,
});
expect(d.clearRegistry).toHaveBeenCalledOnce();
expect(d.dockerRemoveVolumesByPrefix).toHaveBeenCalledWith("openshell-cluster-nemoclaw", {
ignoreError: true,
});
});

it("stops Docker-driver gateways, unregisters them, and removes cluster volumes", () => {
const d = deps({
hasLifecycleCommands: vi.fn(() => false),
isDockerDriverGatewayEnabled: vi.fn(() => true),
});

expect(destroyGatewayWithVolumeCleanup(d)).toBe(true);

expect(d.stopDockerDriverGatewayProcess).toHaveBeenCalledOnce();
expect(d.removeDockerDriverGatewayRegistration).toHaveBeenCalledOnce();
expect(d.runOpenshell).not.toHaveBeenCalled();
expect(d.clearRegistry).toHaveBeenCalledOnce();
expect(d.dockerRemoveVolumesByPrefix).toHaveBeenCalledWith("openshell-cluster-nemoclaw", {
ignoreError: true,
});
});

it("does not clear registry or remove volumes when gateway removal fails", () => {
const d = deps({ runOpenshell: vi.fn(() => ({ status: 1 })) });

expect(destroyGatewayWithVolumeCleanup(d)).toBe(false);

expect(d.clearRegistry).not.toHaveBeenCalled();
expect(d.dockerRemoveVolumesByPrefix).not.toHaveBeenCalled();
});

it("preserves legacy gateway behavior without Docker volume cleanup", () => {
const d = deps({ hasLifecycleCommands: vi.fn(() => false) });

expect(destroyGatewayWithVolumeCleanup(d)).toBe(true);

expect(d.runOpenshell).toHaveBeenCalledWith(["gateway", "remove", "nemoclaw"], {
ignoreError: true,
});
expect(d.clearRegistry).toHaveBeenCalledOnce();
expect(d.dockerRemoveVolumesByPrefix).not.toHaveBeenCalled();
});
});
Loading
Loading