Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions src/lib/actions/sandbox/connect-lifecycle-lock.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,44 @@ describe("connectSandbox lifecycle lock", () => {
value: originalStdoutIsTty,
});
delete process.env.NEMOCLAW_TEST_NO_SLEEP;
delete process.env.NEMOCLAW_GATEWAY_PORT;
delete require.cache[requireDist.resolve(connectModulePath)];
});

it("uses the host-scoped lifecycle lock for interactive Hermes connect", async () => {
process.env.NEMOCLAW_GATEWAY_PORT = "18080";
const harness = createConnectHarness({
agentName: "hermes",
sessionAgent: { name: "hermes" },
registryEntry: {
openshellDriver: "docker",
gatewayName: "nemoclaw",
lifecycleGeneration: "generation-1",
},
portableReceiptDisposition: { kind: "hermes", phase: "active" },
portableRecoveryResult: { kind: "already-running" },
});
const gatewayState = requireDist(
"../../src/lib/actions/sandbox/gateway-state.js",
) as typeof import("./gateway-state");

await expect(harness.connectSandbox("alpha")).rejects.toThrow("process.exit(0)");

expect(gatewayState.withConnectSandboxLifecycleLock).toHaveBeenCalledTimes(2);
expect(gatewayState.withConnectSandboxLifecycleLock).toHaveBeenNthCalledWith(
1,
"alpha",
expect.any(Function),
{ stateDir: "/home/test/.nemoclaw/state" },
);
expect(gatewayState.withConnectSandboxLifecycleLock).toHaveBeenNthCalledWith(
2,
"alpha",
expect.any(Function),
{ stateDir: "/home/test/.nemoclaw/state" },
);
});

it("releases the lifecycle lock before waiting on the interactive shell (#9737)", async () => {
const harness = createConnectHarness();
const gatewayState = requireDist(
Expand Down
16 changes: 14 additions & 2 deletions src/lib/actions/sandbox/connect.ts
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ import {
defaultPortableDemoStateDir,
type HermesPortableActiveLifecycleAuthority,
getNamedGatewayLifecycleState,
hermesPortableLifecycleLockOptions,
printGatewayLifecycleHint,
qualifyHermesPortableAcceptedReadinessAuthority,
qualifyPortableAgentLifecycleAuthority,
Expand Down Expand Up @@ -1957,6 +1958,17 @@ export async function waitForSandboxReadyOrExit(
* the express-vLLM model preflight, the owning-gateway pin, and the Docker
* outage fast-fail. Runs before any probe or interactive work.
*/
function withPortableConnectSandboxLifecycleLock<T>(
sandboxName: string,
operation: () => Promise<T> | T,
): Promise<T> {
return withConnectSandboxLifecycleLock(
sandboxName,
operation,
hermesPortableLifecycleLockOptions(sandboxName, process.env),
);
}

async function runConnectEntryPreflight(
sandboxName: string,
{
Expand Down Expand Up @@ -1984,7 +1996,7 @@ async function runConnectEntryPreflight(
probeTiming ? probeTiming.measure(stage, operation) : operation();
const measureAsync = <T>(stage: "gateway", operation: () => Promise<T>): Promise<T> =>
probeTiming ? probeTiming.measureAsync(stage, operation) : operation();
await withConnectSandboxLifecycleLock(sandboxName, async () => {
await withPortableConnectSandboxLifecycleLock(sandboxName, async () => {
let hermesPortable = false;
let requalify = () => undefined;
try {
Expand Down Expand Up @@ -2340,7 +2352,7 @@ export async function connectSandbox(
};
if (probeTiming) process.once("exit", finishOnExit);
try {
const started = await withConnectSandboxLifecycleLock(sandboxName, async () => {
const started = await withPortableConnectSandboxLifecycleLock(sandboxName, async () => {
const prepared = await prepareConnectSandboxWithinLifecycleFence(
sandboxName,
options,
Expand Down
2 changes: 2 additions & 0 deletions src/lib/actions/sandbox/gateway-state.ts
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ import {
buildHermesPortableCommandEnvironment,
buildHermesPortableCommandAuthority,
defaultPortableDemoStateDir,
hermesPortableLifecycleLockOptions,
inspectPortableAgentReceiptDisposition,
qualifyHermesPortableAcceptedReadinessAuthority,
qualifyPortableAgentLifecycleAuthority,
Expand Down Expand Up @@ -134,6 +135,7 @@ export {
buildHermesPortableCommandAuthority,
buildHermesPortableCommandEnvironment,
defaultPortableDemoStateDir,
hermesPortableLifecycleLockOptions,
inspectPortableAgentReceiptDisposition,
qualifyHermesPortableAcceptedReadinessAuthority,
qualifyPortableAgentLifecycleAuthority,
Expand Down
9 changes: 6 additions & 3 deletions src/lib/actions/sandbox/start.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import {
READINESS_INFERENCE_INVOCATION_TIMEOUT_MS,
type SandboxInferenceInvocationResult,
} from "./inference-invocation-probe";
import { withSandboxLifecycleLock } from "./gateway-state";
import { hermesPortableLifecycleLockOptions, withSandboxLifecycleLock } from "./gateway-state";
import { getPersistedSandboxTargetGatewayName } from "./gateway-target";
import {
resolveSandboxLifecycleProvider,
Expand Down Expand Up @@ -161,8 +161,11 @@ export async function startSandbox(
sandboxName: string,
deps: SandboxStartDeps = {},
): Promise<SandboxLifecycleResult> {
return (deps.withLifecycleLock ?? withSandboxLifecycleLock)(sandboxName, () =>
startSandboxWithinLifecycleFence(sandboxName, deps),
const environment = deps.environment ?? process.env;
return (deps.withLifecycleLock ?? withSandboxLifecycleLock)(
sandboxName,
() => startSandboxWithinLifecycleFence(sandboxName, deps),
hermesPortableLifecycleLockOptions(sandboxName, environment),
);
}

Expand Down
8 changes: 6 additions & 2 deletions src/lib/actions/sandbox/stop.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import { stopSandboxChannels } from "../../tunnel/sandbox-gateway-stop";
import { teardownSandboxDashboardForward } from "./forward-recovery";
import {
captureSandboxOwnershipPhases,
hermesPortableLifecycleLockOptions,
resolvePersistedSandboxOwnershipGateway,
withSandboxLifecycleLockSync,
} from "./gateway-state";
Expand Down Expand Up @@ -263,8 +264,11 @@ export function stopSandbox(
sandboxName: string,
deps: SandboxStopDeps = {},
): SandboxLifecycleResult {
return (deps.withLifecycleLockSync ?? withSandboxLifecycleLockSync)(sandboxName, () =>
stopSandboxWithinLifecycleFence(sandboxName, deps),
const environment = deps.environment ?? process.env;
return (deps.withLifecycleLockSync ?? withSandboxLifecycleLockSync)(
sandboxName,
() => stopSandboxWithinLifecycleFence(sandboxName, deps),
hermesPortableLifecycleLockOptions(sandboxName, environment),
);
}

Expand Down
36 changes: 36 additions & 0 deletions src/lib/cli/nemoclaw-oclif-command.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,28 @@ class PortableStartCommand extends NemoClawCommand {
}
}

class PortableStopCommand extends NemoClawCommand {
static id = "sandbox:stop";
static args = { sandboxName: Args.string({ required: true }) };
static flags = {};
static observed = { host: false, lifecycle: false, portableLifecycle: false };

public async run(): Promise<void> {
const { args } = await this.parse(PortableStopCommand);
const sandboxName = args.sandboxName!;
PortableStopCommand.observed = {
host: fs.existsSync(
portableHostAuthority.portableHostFencePath(process.env.HOME || os.homedir()),
),
lifecycle: isMcpLifecycleLockHeld(sandboxName),
portableLifecycle: isMcpLifecycleLockHeld(
sandboxName,
path.join(portableHostAuthority.defaultPortableStateDir(process.env), "state"),
),
};
}
}

function useHermesPortableAuthority(): void {
vi.spyOn(receiptAuthority, "hasHermesPortableReceiptCandidate").mockReturnValue(true);
vi.spyOn(
Expand Down Expand Up @@ -214,6 +236,7 @@ describe("NemoClawCommand", () => {
GlobalUseMutationCommand.ran = false;
ProbeOnlyConnectCommand.operation = () => undefined;
PortableStartCommand.observed = { host: false, lifecycle: false, portableLifecycle: false };
PortableStopCommand.observed = { host: false, lifecycle: false, portableLifecycle: false };
});

it("records status-like command results without throwing", () => {
Expand Down Expand Up @@ -364,6 +387,19 @@ describe("NemoClawCommand", () => {
});
});

it("uses the Portable lifecycle lock for stop without broadening the host fence", async () => {
vi.stubEnv("NEMOCLAW_GATEWAY_PORT", "18080");
useHermesPortableAuthority();

await PortableStopCommand.run(["alpha"], process.cwd());

expect(PortableStopCommand.observed).toEqual({
host: false,
lifecycle: false,
portableLifecycle: true,
});
});

it("does not create the Portable host fence when a lifecycle command has no Hermes receipt candidate", async () => {
vi.stubEnv("HOME", stateDir);
vi.stubEnv("NEMOCLAW_TEST_BASE_HOME", stateDir);
Expand Down
17 changes: 7 additions & 10 deletions src/lib/cli/nemoclaw-oclif-command.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,14 @@
// SPDX-License-Identifier: Apache-2.0

import { Command, Flags, type Interfaces } from "@oclif/core";
import path from "node:path";
import {
assertHermesPortableCommandSupported,
assertHermesPortableCommandUnavailable,
classifyHermesPortableCommand,
hermesPortableLifecycleLockOptions,
HERMES_PORTABLE_UNSUPPORTED_COMMAND_MESSAGE,
HERMES_PORTABLE_UNSUPPORTED_DOCTOR_FIX_MESSAGE,
} from "../onboard/experimental/portable-agent-lifecycle";
import { hasHermesPortableReceiptCandidate } from "../onboard/experimental/hermes-portable-receipt";
import { defaultPortableDemoStateDir } from "../onboard/experimental/portable-runtime-receipt-readiness";
import { redactForLog } from "../security/redact";
import {
Expand Down Expand Up @@ -143,17 +142,15 @@ export abstract class NemoClawCommand extends Command {
}
return super._run<T>();
};
const portableLifecycleLockOptions = hermesPortableLifecycleLockOptions(
sandboxName,
process.env,
);
const usesHermesPortableHostAuthority =
(commandId === "sandbox:start" || this.isProbeOnlyConnect(commandId)) &&
hasHermesPortableReceiptCandidate(sandboxName, defaultPortableDemoStateDir(process.env));
portableLifecycleLockOptions !== undefined;
const runWithLifecycleFence = async () => {
return await withMcpLifecycleLock(
sandboxName,
runLocked,
usesHermesPortableHostAuthority
? { stateDir: path.join(defaultPortableDemoStateDir(process.env), "state") }
: {},
);
return await withMcpLifecycleLock(sandboxName, runLocked, portableLifecycleLockOptions);
};
if (usesHermesPortableHostAuthority) {
return await withCurrentPortableHostFence(runWithLifecycleFence);
Expand Down
16 changes: 8 additions & 8 deletions src/lib/onboard/experimental/hermes-portable-lifecycle.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,7 @@ function lifecycleDeps(
readonly sandboxPhase?: (running: boolean) => string;
readonly sandboxIdentity?: (running: boolean) => string | undefined;
readonly failPostStartInspectOnce?: boolean;
readonly startStatus?: number;
} = {},
) {
let running = initiallyRunning,
Expand Down Expand Up @@ -293,7 +294,7 @@ function lifecycleDeps(
"sandbox:start": () => {
running = true;
postStartInspectFailurePending = options.failPostStartInspectOnce === true;
return { status: 0, stdout: "", stderr: "" };
return { status: options.startStatus ?? 0, stdout: "", stderr: "start failed" };
},
"sandbox:stop": () => {
running = false;
Expand Down Expand Up @@ -1063,23 +1064,22 @@ describe("Hermes portable lifecycle", () => {
expect(openshellMutationCalls(captureOpenShell, "start")).toHaveLength(1);
expect(openshellMutationCalls(captureOpenShell, "stop")).toHaveLength(1);
});

it("reconciles and rolls back a start whose post-start inspection fails (#9203)", () => {
it.each([
["post-start inspection fails", { failPostStartInspectOnce: true }, "exact inspect failed"],
["OpenShell reports failure", { startStatus: 1 }, "OpenShell start failed with status 1"],
])("reconciles and rolls back when %s (#9203)", (_case, options, failure) => {
const receipt = activeReceipt();
const { deps, captureOpenShell } = lifecycleDeps(receipt, false, {
failPostStartInspectOnce: true,
});
const { deps, captureOpenShell } = lifecycleDeps(receipt, false, options);
expect(() =>
withMcpLifecycleLockSync(
SANDBOX,
() => recoverHermesPortableSandboxLifecycle(SANDBOX, lifecycleContext(), deps),
{ stateDir: path.join(stateDir, "state") },
),
).toThrow("exact inspect failed with status 1");
).toThrow(failure);
expect(openshellMutationCalls(captureOpenShell, "start")).toHaveLength(1);
expect(openshellMutationCalls(captureOpenShell, "stop")).toHaveLength(1);
});

it("does not stop an already-running container after a health failure (#9203)", () => {
const receipt = activeReceipt();
const { deps, podman, captureOpenShell, launchOpenShell } = lifecycleDeps(receipt);
Expand Down
10 changes: 9 additions & 1 deletion src/lib/onboard/experimental/hermes-portable-lifecycle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1430,7 +1430,15 @@ export function recoverHermesPortableSandboxLifecycle(
OPENSHELL_LIFECYCLE_MUTATION_TIMEOUT_MS,
),
);
startedByRecovery = startResult.status === 0 && !startResult.error;
if (startResult.status !== 0 || startResult.error) {
throw (
startResult.error ??
new Error(
`Hermes portable OpenShell start failed with status ${String(startResult.status)}`,
)
);
}
startedByRecovery = true;
primaryFailureClass = "post-start-authority";
if (qualified.hasTransactionAuthority) {
timing.increment("transactionCurrentness");
Expand Down
12 changes: 12 additions & 0 deletions src/lib/onboard/experimental/portable-agent-lifecycle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,18 @@ import { defaultPortableDemoStateDir } from "./portable-runtime-receipt-readines

export { defaultPortableDemoStateDir };

/** Resolve the shared lifecycle-lock root for a retained Hermes portable sandbox. */
export function hermesPortableLifecycleLockOptions(
sandboxName: string,
env: NodeJS.ProcessEnv = process.env,
hasReceiptCandidate: typeof hasHermesPortableReceiptCandidate = hasHermesPortableReceiptCandidate,
): { readonly stateDir: string } | undefined {
if (!hasReceiptCandidate(sandboxName, defaultPortableDemoStateDir(env))) {
return undefined;
}
return { stateDir: path.join(defaultPortableDemoStateDir(env), "state") };
}

export type PortableAgentLifecycleDeps = PortableDemoLifecycleDeps & HermesPortableLifecycleDeps;
export type PortableAgentLifecycleStopResult = PortableDemoLifecycleStopResult & {
readonly portableAgent?: "hermes";
Expand Down
18 changes: 8 additions & 10 deletions src/lib/onboard/runtime-provider/docker.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import path from "node:path";

import { captureHostCommand } from "../../actions/sandbox/doctor-host-command";
import { dockerCapture, dockerRun } from "../../adapters/docker/run";
import {
Expand All @@ -28,12 +26,12 @@ import {
} from "../experimental/docker-network-authority";
import {
hasPortableAgentSandboxLifecycleReceipt,
hermesPortableLifecycleLockOptions,
recoverPortableAgentSandboxLifecycle,
requalifyPortableAgentSandboxAuthority,
stopPortableAgentSandboxLifecycle,
} from "../experimental/portable-agent-lifecycle";
import { withMcpLifecycleLockSync } from "../../state/mcp-lifecycle-lock-acquisition";
import { defaultPortableStateDir } from "../../state/portable-uninstall-retirement";
import { queryOpenShellDockerSandboxRuntimeSnapshot } from "../openshell-docker-sandbox-containers";
import { validateSandboxGpuPreflight } from "../sandbox-gpu-preflight";
import {
Expand Down Expand Up @@ -309,13 +307,13 @@ function dockerLifecycleLockOptions(
input: RuntimeProviderLifecycleInput,
deps: DockerRuntimeProviderDependencies,
): { readonly stateDir: string } | undefined {
if (
input.sandbox.agent !== "hermes" ||
!deps.hasPortableLifecycleReceipt(input.sandboxName, input.environment)
) {
return undefined;
}
return { stateDir: path.join(defaultPortableStateDir(input.environment), "state") };
return hermesPortableLifecycleLockOptions(
input.sandboxName,
input.environment,
() =>
input.sandbox.agent === "hermes" &&
deps.hasPortableLifecycleReceipt(input.sandboxName, input.environment),
);
}

function startDockerSandboxUnlocked(
Expand Down
5 changes: 5 additions & 0 deletions test/support/connect-flow-test-harness.ts
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,11 @@ export function createConnectHarness(options: ConnectHarnessOptions = {}): Conne
() => undefined,
);
const requestedPortableDisposition = options.portableReceiptDisposition ?? { kind: "absent" };
vi.spyOn(gatewayState, "hermesPortableLifecycleLockOptions").mockReturnValue(
requestedPortableDisposition.kind === "hermes"
? { stateDir: "/home/test/.nemoclaw/state" }
: undefined,
);
const portableDisposition =
requestedPortableDisposition.kind === "hermes"
? {
Expand Down
Loading