Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 63 additions & 0 deletions src/lib/actions/sandbox/exec-gateway-target.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { afterEach, describe, expect, it, vi } from "vitest";

import { execSandbox, type SandboxExecCleanupDeps } from "./exec";

const cleanupSkipped: SandboxExecCleanupDeps = {
getSandbox: () => null,
inspectMutableConfigPerms: (() => {
throw new Error("cleanup should be skipped");
}) as unknown as SandboxExecCleanupDeps["inspectMutableConfigPerms"],
repairMutableConfigPerms: (() => {
throw new Error("cleanup should be skipped");
}) as unknown as SandboxExecCleanupDeps["repairMutableConfigPerms"],
};

describe("execSandbox gateway targeting", () => {
afterEach(() => {
vi.restoreAllMocks();
});

it("selects the sandbox's owning gateway before dispatching the exec", async () => {
const order: string[] = [];
const selectGateway = vi.fn((name: string) => {
order.push(`select:${name}`);
});
const run = vi.fn(async () => {
order.push("run");
return { status: 0 };
});
vi.spyOn(process, "exit").mockImplementation(((code?: number) => {
throw new Error(`__exit_${code ?? 0}__`);
}) as never);
vi.spyOn(console, "error").mockImplementation(() => undefined);

await execSandbox(
"beta",
["hostname"],
{},
{
resolveBinary: () => "openshell",
selectGateway,
run,
cleanupDeps: cleanupSkipped,
policyHint: {
now: () => 0,
env: {},
probeLogs: () => "",
enableAudit: () => {},
sleep: async () => {},
attempts: 1,
writeStderr: () => {},
},
},
).catch(() => {});

expect(selectGateway).toHaveBeenCalledWith("beta");
expect(run).toHaveBeenCalled();
expect(order.indexOf("select:beta")).toBeGreaterThanOrEqual(0);
expect(order.indexOf("select:beta")).toBeLessThan(order.indexOf("run"));
Comment thread
coderabbitai[bot] marked this conversation as resolved.
});
});
9 changes: 9 additions & 0 deletions src/lib/actions/sandbox/exec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -329,6 +329,12 @@ function defaultResolveBinary(): string {
return getOpenshellBinary();
}

function defaultSelectGateway(sandboxName: string): void {
(require("./gateway-select") as typeof import("./gateway-select")).selectSandboxOwningGateway(
sandboxName,
);
}

// Test seams for execSandbox. All default to the production behavior; tests
// inject them so the dispatch path stays hermetic without spawning a real
// process or hitting the process-exiting OpenShell binary lookup.
Expand All @@ -340,6 +346,8 @@ export type ExecSandboxDeps = {
run?: SandboxExecRunner;
policyHint?: ExecPolicyHintDeps;
cleanupDeps?: SandboxExecCleanupDeps;
/** Select the sandbox's owning gateway before the exec talks to OpenShell. */
selectGateway?: (sandboxName: string) => void;
};

export async function execSandbox(
Expand All @@ -361,6 +369,7 @@ export async function execSandbox(
process.exit(2);
}
const binary = (deps.resolveBinary ?? defaultResolveBinary)();
(deps.selectGateway ?? defaultSelectGateway)(sandboxName);
if (options.workdir) {
validateWorkdirOrFail(binary, sandboxName, options.workdir, deps.probeWorkdir);
}
Expand Down
42 changes: 42 additions & 0 deletions src/lib/actions/sandbox/gateway-select.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { afterEach, describe, expect, it, vi } from "vitest";

import * as registry from "../../state/registry";
import { selectSandboxOwningGateway } from "./gateway-select";

describe("selectSandboxOwningGateway", () => {
afterEach(() => {
vi.restoreAllMocks();
});

it("selects the owning non-default gateway for a registered sandbox", () => {
vi.spyOn(registry, "getSandbox").mockReturnValue({ gatewayPort: 8091 } as never);
const run = vi.fn(() => ({ status: 0 }) as never);

const selected = selectSandboxOwningGateway("beta", run);

expect(selected).toBe("nemoclaw-8091");
expect(run).toHaveBeenCalledWith(
["gateway", "select", "nemoclaw-8091"],
expect.objectContaining({ ignoreError: true }),
);
});

it("keeps the bare default gateway name for a default-port sandbox", () => {
vi.spyOn(registry, "getSandbox").mockReturnValue({ gatewayPort: 8080 } as never);
const run = vi.fn(() => ({ status: 0 }) as never);

expect(selectSandboxOwningGateway("alpha", run)).toBe("nemoclaw");
expect(run).toHaveBeenCalledWith(["gateway", "select", "nemoclaw"], expect.anything());
});

it("does not touch the active gateway for an unregistered sandbox", () => {
vi.spyOn(registry, "getSandbox").mockReturnValue(null);
const run = vi.fn(() => ({ status: 0 }) as never);

expect(selectSandboxOwningGateway("ghost", run)).toBeNull();
expect(run).not.toHaveBeenCalled();
});
});
21 changes: 21 additions & 0 deletions src/lib/actions/sandbox/gateway-select.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { runOpenshell } from "../../adapters/openshell/runtime";
import { OPENSHELL_OPERATION_TIMEOUT_MS } from "../../adapters/openshell/timeouts";
import { getKnownSandboxTargetGatewayName } from "./gateway-target";

export type GatewaySelectRunner = typeof runOpenshell;

export function selectSandboxOwningGateway(
sandboxName: string,
run: GatewaySelectRunner = runOpenshell,
): string | null {
const targetGatewayName = getKnownSandboxTargetGatewayName(sandboxName);
if (!targetGatewayName) return null;
run(["gateway", "select", targetGatewayName], {
ignoreError: true,
timeout: OPENSHELL_OPERATION_TIMEOUT_MS,
});
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Outdated
return targetGatewayName;
}
76 changes: 76 additions & 0 deletions src/lib/actions/sandbox/gateway-state-owning-gateway.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { afterEach, describe, expect, it, vi } from "vitest";

import * as gatewayRuntime from "../../gateway-runtime-action";
import * as registry from "../../state/registry";
import * as gatewaySelect from "./gateway-select";
import { getReconciledSandboxGatewayState } from "./gateway-state";

describe("getReconciledSandboxGatewayState owning-gateway guard", () => {
afterEach(() => {
vi.restoreAllMocks();
});

it("reselects the owning gateway and re-queries when a present comes from another active gateway", async () => {
vi.spyOn(registry, "getSandbox").mockReturnValue({ gatewayPort: 8091 } as never);
vi.spyOn(gatewayRuntime, "getNamedGatewayLifecycleState").mockReturnValue({
state: "connected_other",
status: "",
activeGateway: "nemoclaw",
} as never);
const selectSpy = vi
.spyOn(gatewaySelect, "selectSandboxOwningGateway")
.mockReturnValue("nemoclaw-8091");

const getState = vi
.fn()
.mockResolvedValueOnce({ state: "present", output: "Phase: Provisioning" })
.mockResolvedValueOnce({ state: "present", output: "Phase: Ready" });

const result = await getReconciledSandboxGatewayState("beta", { getState });

expect(getState).toHaveBeenCalledTimes(2);
expect(selectSpy).toHaveBeenCalledWith("beta");
expect(result).toMatchObject({
state: "present",
output: "Phase: Ready",
recoveredGateway: true,
recoveryVia: "select",
});
});

it("trusts a present from the owning gateway without reselecting", async () => {
vi.spyOn(registry, "getSandbox").mockReturnValue({ gatewayPort: 8091 } as never);
vi.spyOn(gatewayRuntime, "getNamedGatewayLifecycleState").mockReturnValue({
state: "healthy_named",
status: "",
} as never);
const selectSpy = vi.spyOn(gatewaySelect, "selectSandboxOwningGateway");

const getState = vi.fn().mockResolvedValue({ state: "present", output: "Phase: Ready" });

const result = await getReconciledSandboxGatewayState("beta", { getState });

expect(getState).toHaveBeenCalledTimes(1);
expect(selectSpy).not.toHaveBeenCalled();
expect(result).toMatchObject({ state: "present", output: "Phase: Ready" });
expect(result.recoveredGateway).toBeUndefined();
});

it("leaves an unregistered sandbox's present result untouched", async () => {
vi.spyOn(registry, "getSandbox").mockReturnValue(null);
const lifecycleSpy = vi.spyOn(gatewayRuntime, "getNamedGatewayLifecycleState");
const selectSpy = vi.spyOn(gatewaySelect, "selectSandboxOwningGateway");

const getState = vi.fn().mockResolvedValue({ state: "present", output: "Phase: Ready" });

const result = await getReconciledSandboxGatewayState("ghost", { getState });

expect(getState).toHaveBeenCalledTimes(1);
expect(lifecycleSpy).not.toHaveBeenCalled();
expect(selectSpy).not.toHaveBeenCalled();
expect(result).toMatchObject({ state: "present" });
});
});
39 changes: 37 additions & 2 deletions src/lib/actions/sandbox/gateway-state.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,12 @@ import {
recoverNamedGatewayRuntime,
} from "../../gateway-runtime-action";
import { isTerminalSandboxPhase, parseSandboxPhase } from "../../state/gateway";
import { gatewayNamePattern, getSandboxTargetGatewayName } from "./gateway-target";
import { selectSandboxOwningGateway } from "./gateway-select";
import {
gatewayNamePattern,
getKnownSandboxTargetGatewayName,
getSandboxTargetGatewayName,
} from "./gateway-target";

const { pruneKnownHostsEntries } = require("../../onboard/known-hosts") as {
pruneKnownHostsEntries: (contents: string) => string;
Expand Down Expand Up @@ -423,14 +428,44 @@ export function printGatewayLifecycleHint(
}
}

/**
* A `present` lookup is only trustworthy when it came from the sandbox's own
* gateway. On a multi-instance host the active OpenShell gateway can be a
* sibling that reports a stale or provisioning entry for the same sandbox name,
* so a plain `present` masks the real state on the owning gateway. When the
* sandbox's gateway is registered but a different gateway is currently active,
* select the owning gateway and re-query so the returned state reflects the
* gateway the sandbox was onboarded against rather than the last-used one.
* Returns `null` when the initial `present` can be trusted as-is (owning
* gateway already active, or the sandbox is not registry-known).
*/
async function reselectOwningGatewayForPresent(
sandboxName: string,
getState: SandboxGatewayStateLookup,
): Promise<SandboxGatewayState | null> {
const targetGatewayName = getKnownSandboxTargetGatewayName(sandboxName);
if (!targetGatewayName) return null;
const lifecycle = getNamedGatewayLifecycleState(targetGatewayName);
if (lifecycle.state !== "connected_other") return null;
selectSandboxOwningGateway(sandboxName);
const retry = await getState(sandboxName);
if (retry.state === "present") {
return { ...retry, recoveredGateway: true, recoveryVia: "select" };
}
if (retry.state === "missing") {
return reconcileMissingAgainstNamedGateway(sandboxName, retry);
}
return retry;
}

export async function getReconciledSandboxGatewayState(
sandboxName: string,
opts: { getState?: SandboxGatewayStateLookup } = {},
): Promise<SandboxGatewayState> {
const getState = opts.getState ?? getSandboxGatewayState;
const lookup = await getState(sandboxName);
if (lookup.state === "present") {
return lookup;
return (await reselectOwningGatewayForPresent(sandboxName, getState)) ?? lookup;
}
if (lookup.state === "missing") {
return reconcileMissingAgainstNamedGateway(sandboxName, lookup);
Expand Down
8 changes: 6 additions & 2 deletions src/lib/actions/sandbox/gateway-target.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,13 @@ import { GATEWAY_PORT } from "../../core/ports";
import { resolveGatewayName, resolveSandboxGatewayName } from "../../onboard/gateway-binding";
import * as registry from "../../state/registry";

export function getSandboxTargetGatewayName(sandboxName = ""): string {
export function getKnownSandboxTargetGatewayName(sandboxName = ""): string | null {
const sb = sandboxName ? registry.getSandbox(sandboxName) : null;
return sb ? resolveSandboxGatewayName(sb) : resolveGatewayName(GATEWAY_PORT);
return sb ? resolveSandboxGatewayName(sb) : null;
}

export function getSandboxTargetGatewayName(sandboxName = ""): string {
return getKnownSandboxTargetGatewayName(sandboxName) ?? resolveGatewayName(GATEWAY_PORT);
}

export function gatewayNamePattern(gatewayName: string): RegExp {
Expand Down
Loading