diff --git a/src/lib/actions/sandbox/connect-lifecycle-lock.test.ts b/src/lib/actions/sandbox/connect-lifecycle-lock.test.ts index c3c1d65828b..74390fa5b2e 100644 --- a/src/lib/actions/sandbox/connect-lifecycle-lock.test.ts +++ b/src/lib/actions/sandbox/connect-lifecycle-lock.test.ts @@ -26,9 +26,44 @@ describe("connectSandbox lifecycle lock", () => { value: originalStdoutIsTty, }); delete process.env.NEMOCLAW_TEST_NO_SLEEP; + delete process.env.NEMOCLAW_GATEWAY_PORT; delete require.cache[requireDist.resolve(connectModulePath)]; }); + it("uses the host-scoped lifecycle lock for interactive Hermes connect", async () => { + process.env.NEMOCLAW_GATEWAY_PORT = "18080"; + const harness = createConnectHarness({ + agentName: "hermes", + sessionAgent: { name: "hermes" }, + registryEntry: { + openshellDriver: "docker", + gatewayName: "nemoclaw", + lifecycleGeneration: "generation-1", + }, + portableReceiptDisposition: { kind: "hermes", phase: "active" }, + portableRecoveryResult: { kind: "already-running" }, + }); + const gatewayState = requireDist( + "../../src/lib/actions/sandbox/gateway-state.js", + ) as typeof import("./gateway-state"); + + await expect(harness.connectSandbox("alpha")).rejects.toThrow("process.exit(0)"); + + expect(gatewayState.withConnectSandboxLifecycleLock).toHaveBeenCalledTimes(2); + expect(gatewayState.withConnectSandboxLifecycleLock).toHaveBeenNthCalledWith( + 1, + "alpha", + expect.any(Function), + { stateDir: "/home/test/.nemoclaw/state" }, + ); + expect(gatewayState.withConnectSandboxLifecycleLock).toHaveBeenNthCalledWith( + 2, + "alpha", + expect.any(Function), + { stateDir: "/home/test/.nemoclaw/state" }, + ); + }); + it("releases the lifecycle lock before waiting on the interactive shell (#9737)", async () => { const harness = createConnectHarness(); const gatewayState = requireDist( diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index cf2bd0c428c..f7eed4faa3e 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -91,6 +91,7 @@ import { defaultPortableDemoStateDir, type HermesPortableActiveLifecycleAuthority, getNamedGatewayLifecycleState, + hermesPortableLifecycleLockOptions, printGatewayLifecycleHint, qualifyHermesPortableAcceptedReadinessAuthority, qualifyPortableAgentLifecycleAuthority, @@ -1957,6 +1958,17 @@ export async function waitForSandboxReadyOrExit( * the express-vLLM model preflight, the owning-gateway pin, and the Docker * outage fast-fail. Runs before any probe or interactive work. */ +function withPortableConnectSandboxLifecycleLock( + sandboxName: string, + operation: () => Promise | T, +): Promise { + return withConnectSandboxLifecycleLock( + sandboxName, + operation, + hermesPortableLifecycleLockOptions(sandboxName, process.env), + ); +} + async function runConnectEntryPreflight( sandboxName: string, { @@ -1984,7 +1996,7 @@ async function runConnectEntryPreflight( probeTiming ? probeTiming.measure(stage, operation) : operation(); const measureAsync = (stage: "gateway", operation: () => Promise): Promise => probeTiming ? probeTiming.measureAsync(stage, operation) : operation(); - await withConnectSandboxLifecycleLock(sandboxName, async () => { + await withPortableConnectSandboxLifecycleLock(sandboxName, async () => { let hermesPortable = false; let requalify = () => undefined; try { @@ -2340,7 +2352,7 @@ export async function connectSandbox( }; if (probeTiming) process.once("exit", finishOnExit); try { - const started = await withConnectSandboxLifecycleLock(sandboxName, async () => { + const started = await withPortableConnectSandboxLifecycleLock(sandboxName, async () => { const prepared = await prepareConnectSandboxWithinLifecycleFence( sandboxName, options, diff --git a/src/lib/actions/sandbox/gateway-state.ts b/src/lib/actions/sandbox/gateway-state.ts index 9f2cd082344..650a012398e 100644 --- a/src/lib/actions/sandbox/gateway-state.ts +++ b/src/lib/actions/sandbox/gateway-state.ts @@ -73,6 +73,7 @@ import { buildHermesPortableCommandEnvironment, buildHermesPortableCommandAuthority, defaultPortableDemoStateDir, + hermesPortableLifecycleLockOptions, inspectPortableAgentReceiptDisposition, qualifyHermesPortableAcceptedReadinessAuthority, qualifyPortableAgentLifecycleAuthority, @@ -134,6 +135,7 @@ export { buildHermesPortableCommandAuthority, buildHermesPortableCommandEnvironment, defaultPortableDemoStateDir, + hermesPortableLifecycleLockOptions, inspectPortableAgentReceiptDisposition, qualifyHermesPortableAcceptedReadinessAuthority, qualifyPortableAgentLifecycleAuthority, diff --git a/src/lib/actions/sandbox/start.ts b/src/lib/actions/sandbox/start.ts index bd60d59c2c2..912ad8539b8 100644 --- a/src/lib/actions/sandbox/start.ts +++ b/src/lib/actions/sandbox/start.ts @@ -14,7 +14,7 @@ import { READINESS_INFERENCE_INVOCATION_TIMEOUT_MS, type SandboxInferenceInvocationResult, } from "./inference-invocation-probe"; -import { withSandboxLifecycleLock } from "./gateway-state"; +import { hermesPortableLifecycleLockOptions, withSandboxLifecycleLock } from "./gateway-state"; import { getPersistedSandboxTargetGatewayName } from "./gateway-target"; import { resolveSandboxLifecycleProvider, @@ -161,8 +161,11 @@ export async function startSandbox( sandboxName: string, deps: SandboxStartDeps = {}, ): Promise { - return (deps.withLifecycleLock ?? withSandboxLifecycleLock)(sandboxName, () => - startSandboxWithinLifecycleFence(sandboxName, deps), + const environment = deps.environment ?? process.env; + return (deps.withLifecycleLock ?? withSandboxLifecycleLock)( + sandboxName, + () => startSandboxWithinLifecycleFence(sandboxName, deps), + hermesPortableLifecycleLockOptions(sandboxName, environment), ); } diff --git a/src/lib/actions/sandbox/stop.ts b/src/lib/actions/sandbox/stop.ts index b45932489d5..f2a7ec11aa7 100644 --- a/src/lib/actions/sandbox/stop.ts +++ b/src/lib/actions/sandbox/stop.ts @@ -19,6 +19,7 @@ import { stopSandboxChannels } from "../../tunnel/sandbox-gateway-stop"; import { teardownSandboxDashboardForward } from "./forward-recovery"; import { captureSandboxOwnershipPhases, + hermesPortableLifecycleLockOptions, resolvePersistedSandboxOwnershipGateway, withSandboxLifecycleLockSync, } from "./gateway-state"; @@ -263,8 +264,11 @@ export function stopSandbox( sandboxName: string, deps: SandboxStopDeps = {}, ): SandboxLifecycleResult { - return (deps.withLifecycleLockSync ?? withSandboxLifecycleLockSync)(sandboxName, () => - stopSandboxWithinLifecycleFence(sandboxName, deps), + const environment = deps.environment ?? process.env; + return (deps.withLifecycleLockSync ?? withSandboxLifecycleLockSync)( + sandboxName, + () => stopSandboxWithinLifecycleFence(sandboxName, deps), + hermesPortableLifecycleLockOptions(sandboxName, environment), ); } diff --git a/src/lib/cli/nemoclaw-oclif-command.test.ts b/src/lib/cli/nemoclaw-oclif-command.test.ts index 26332d8b8fd..091ace99ef3 100644 --- a/src/lib/cli/nemoclaw-oclif-command.test.ts +++ b/src/lib/cli/nemoclaw-oclif-command.test.ts @@ -168,6 +168,28 @@ class PortableStartCommand extends NemoClawCommand { } } +class PortableStopCommand extends NemoClawCommand { + static id = "sandbox:stop"; + static args = { sandboxName: Args.string({ required: true }) }; + static flags = {}; + static observed = { host: false, lifecycle: false, portableLifecycle: false }; + + public async run(): Promise { + const { args } = await this.parse(PortableStopCommand); + const sandboxName = args.sandboxName!; + PortableStopCommand.observed = { + host: fs.existsSync( + portableHostAuthority.portableHostFencePath(process.env.HOME || os.homedir()), + ), + lifecycle: isMcpLifecycleLockHeld(sandboxName), + portableLifecycle: isMcpLifecycleLockHeld( + sandboxName, + path.join(portableHostAuthority.defaultPortableStateDir(process.env), "state"), + ), + }; + } +} + function useHermesPortableAuthority(): void { vi.spyOn(receiptAuthority, "hasHermesPortableReceiptCandidate").mockReturnValue(true); vi.spyOn( @@ -214,6 +236,7 @@ describe("NemoClawCommand", () => { GlobalUseMutationCommand.ran = false; ProbeOnlyConnectCommand.operation = () => undefined; PortableStartCommand.observed = { host: false, lifecycle: false, portableLifecycle: false }; + PortableStopCommand.observed = { host: false, lifecycle: false, portableLifecycle: false }; }); it("records status-like command results without throwing", () => { @@ -364,6 +387,19 @@ describe("NemoClawCommand", () => { }); }); + it("uses the Portable lifecycle lock for stop without broadening the host fence", async () => { + vi.stubEnv("NEMOCLAW_GATEWAY_PORT", "18080"); + useHermesPortableAuthority(); + + await PortableStopCommand.run(["alpha"], process.cwd()); + + expect(PortableStopCommand.observed).toEqual({ + host: false, + lifecycle: false, + portableLifecycle: true, + }); + }); + it("does not create the Portable host fence when a lifecycle command has no Hermes receipt candidate", async () => { vi.stubEnv("HOME", stateDir); vi.stubEnv("NEMOCLAW_TEST_BASE_HOME", stateDir); diff --git a/src/lib/cli/nemoclaw-oclif-command.ts b/src/lib/cli/nemoclaw-oclif-command.ts index 5eb6c3aa55c..b0bdf51def0 100644 --- a/src/lib/cli/nemoclaw-oclif-command.ts +++ b/src/lib/cli/nemoclaw-oclif-command.ts @@ -2,15 +2,14 @@ // SPDX-License-Identifier: Apache-2.0 import { Command, Flags, type Interfaces } from "@oclif/core"; -import path from "node:path"; import { assertHermesPortableCommandSupported, assertHermesPortableCommandUnavailable, classifyHermesPortableCommand, + hermesPortableLifecycleLockOptions, HERMES_PORTABLE_UNSUPPORTED_COMMAND_MESSAGE, HERMES_PORTABLE_UNSUPPORTED_DOCTOR_FIX_MESSAGE, } from "../onboard/experimental/portable-agent-lifecycle"; -import { hasHermesPortableReceiptCandidate } from "../onboard/experimental/hermes-portable-receipt"; import { defaultPortableDemoStateDir } from "../onboard/experimental/portable-runtime-receipt-readiness"; import { redactForLog } from "../security/redact"; import { @@ -143,17 +142,15 @@ export abstract class NemoClawCommand extends Command { } return super._run(); }; + const portableLifecycleLockOptions = hermesPortableLifecycleLockOptions( + sandboxName, + process.env, + ); const usesHermesPortableHostAuthority = (commandId === "sandbox:start" || this.isProbeOnlyConnect(commandId)) && - hasHermesPortableReceiptCandidate(sandboxName, defaultPortableDemoStateDir(process.env)); + portableLifecycleLockOptions !== undefined; const runWithLifecycleFence = async () => { - return await withMcpLifecycleLock( - sandboxName, - runLocked, - usesHermesPortableHostAuthority - ? { stateDir: path.join(defaultPortableDemoStateDir(process.env), "state") } - : {}, - ); + return await withMcpLifecycleLock(sandboxName, runLocked, portableLifecycleLockOptions); }; if (usesHermesPortableHostAuthority) { return await withCurrentPortableHostFence(runWithLifecycleFence); diff --git a/src/lib/onboard/experimental/hermes-portable-lifecycle.test.ts b/src/lib/onboard/experimental/hermes-portable-lifecycle.test.ts index f426ac30f5a..fdc52b4d7a7 100644 --- a/src/lib/onboard/experimental/hermes-portable-lifecycle.test.ts +++ b/src/lib/onboard/experimental/hermes-portable-lifecycle.test.ts @@ -235,6 +235,7 @@ function lifecycleDeps( readonly sandboxPhase?: (running: boolean) => string; readonly sandboxIdentity?: (running: boolean) => string | undefined; readonly failPostStartInspectOnce?: boolean; + readonly startStatus?: number; } = {}, ) { let running = initiallyRunning, @@ -293,7 +294,7 @@ function lifecycleDeps( "sandbox:start": () => { running = true; postStartInspectFailurePending = options.failPostStartInspectOnce === true; - return { status: 0, stdout: "", stderr: "" }; + return { status: options.startStatus ?? 0, stdout: "", stderr: "start failed" }; }, "sandbox:stop": () => { running = false; @@ -1063,23 +1064,22 @@ describe("Hermes portable lifecycle", () => { expect(openshellMutationCalls(captureOpenShell, "start")).toHaveLength(1); expect(openshellMutationCalls(captureOpenShell, "stop")).toHaveLength(1); }); - - it("reconciles and rolls back a start whose post-start inspection fails (#9203)", () => { + it.each([ + ["post-start inspection fails", { failPostStartInspectOnce: true }, "exact inspect failed"], + ["OpenShell reports failure", { startStatus: 1 }, "OpenShell start failed with status 1"], + ])("reconciles and rolls back when %s (#9203)", (_case, options, failure) => { const receipt = activeReceipt(); - const { deps, captureOpenShell } = lifecycleDeps(receipt, false, { - failPostStartInspectOnce: true, - }); + const { deps, captureOpenShell } = lifecycleDeps(receipt, false, options); expect(() => withMcpLifecycleLockSync( SANDBOX, () => recoverHermesPortableSandboxLifecycle(SANDBOX, lifecycleContext(), deps), { stateDir: path.join(stateDir, "state") }, ), - ).toThrow("exact inspect failed with status 1"); + ).toThrow(failure); expect(openshellMutationCalls(captureOpenShell, "start")).toHaveLength(1); expect(openshellMutationCalls(captureOpenShell, "stop")).toHaveLength(1); }); - it("does not stop an already-running container after a health failure (#9203)", () => { const receipt = activeReceipt(); const { deps, podman, captureOpenShell, launchOpenShell } = lifecycleDeps(receipt); diff --git a/src/lib/onboard/experimental/hermes-portable-lifecycle.ts b/src/lib/onboard/experimental/hermes-portable-lifecycle.ts index 1144a0b1c1a..5f42f286010 100644 --- a/src/lib/onboard/experimental/hermes-portable-lifecycle.ts +++ b/src/lib/onboard/experimental/hermes-portable-lifecycle.ts @@ -1430,7 +1430,15 @@ export function recoverHermesPortableSandboxLifecycle( OPENSHELL_LIFECYCLE_MUTATION_TIMEOUT_MS, ), ); - startedByRecovery = startResult.status === 0 && !startResult.error; + if (startResult.status !== 0 || startResult.error) { + throw ( + startResult.error ?? + new Error( + `Hermes portable OpenShell start failed with status ${String(startResult.status)}`, + ) + ); + } + startedByRecovery = true; primaryFailureClass = "post-start-authority"; if (qualified.hasTransactionAuthority) { timing.increment("transactionCurrentness"); diff --git a/src/lib/onboard/experimental/portable-agent-lifecycle.ts b/src/lib/onboard/experimental/portable-agent-lifecycle.ts index 7151a8655a8..b201de5d84d 100644 --- a/src/lib/onboard/experimental/portable-agent-lifecycle.ts +++ b/src/lib/onboard/experimental/portable-agent-lifecycle.ts @@ -39,6 +39,18 @@ import { defaultPortableDemoStateDir } from "./portable-runtime-receipt-readines export { defaultPortableDemoStateDir }; +/** Resolve the shared lifecycle-lock root for a retained Hermes portable sandbox. */ +export function hermesPortableLifecycleLockOptions( + sandboxName: string, + env: NodeJS.ProcessEnv = process.env, + hasReceiptCandidate: typeof hasHermesPortableReceiptCandidate = hasHermesPortableReceiptCandidate, +): { readonly stateDir: string } | undefined { + if (!hasReceiptCandidate(sandboxName, defaultPortableDemoStateDir(env))) { + return undefined; + } + return { stateDir: path.join(defaultPortableDemoStateDir(env), "state") }; +} + export type PortableAgentLifecycleDeps = PortableDemoLifecycleDeps & HermesPortableLifecycleDeps; export type PortableAgentLifecycleStopResult = PortableDemoLifecycleStopResult & { readonly portableAgent?: "hermes"; diff --git a/src/lib/onboard/runtime-provider/docker.ts b/src/lib/onboard/runtime-provider/docker.ts index 86ec73a846c..68859edf054 100644 --- a/src/lib/onboard/runtime-provider/docker.ts +++ b/src/lib/onboard/runtime-provider/docker.ts @@ -1,8 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import path from "node:path"; - import { captureHostCommand } from "../../actions/sandbox/doctor-host-command"; import { dockerCapture, dockerRun } from "../../adapters/docker/run"; import { @@ -28,12 +26,12 @@ import { } from "../experimental/docker-network-authority"; import { hasPortableAgentSandboxLifecycleReceipt, + hermesPortableLifecycleLockOptions, recoverPortableAgentSandboxLifecycle, requalifyPortableAgentSandboxAuthority, stopPortableAgentSandboxLifecycle, } from "../experimental/portable-agent-lifecycle"; import { withMcpLifecycleLockSync } from "../../state/mcp-lifecycle-lock-acquisition"; -import { defaultPortableStateDir } from "../../state/portable-uninstall-retirement"; import { queryOpenShellDockerSandboxRuntimeSnapshot } from "../openshell-docker-sandbox-containers"; import { validateSandboxGpuPreflight } from "../sandbox-gpu-preflight"; import { @@ -309,13 +307,13 @@ function dockerLifecycleLockOptions( input: RuntimeProviderLifecycleInput, deps: DockerRuntimeProviderDependencies, ): { readonly stateDir: string } | undefined { - if ( - input.sandbox.agent !== "hermes" || - !deps.hasPortableLifecycleReceipt(input.sandboxName, input.environment) - ) { - return undefined; - } - return { stateDir: path.join(defaultPortableStateDir(input.environment), "state") }; + return hermesPortableLifecycleLockOptions( + input.sandboxName, + input.environment, + () => + input.sandbox.agent === "hermes" && + deps.hasPortableLifecycleReceipt(input.sandboxName, input.environment), + ); } function startDockerSandboxUnlocked( diff --git a/test/support/connect-flow-test-harness.ts b/test/support/connect-flow-test-harness.ts index 6fb9ff72478..73285002573 100644 --- a/test/support/connect-flow-test-harness.ts +++ b/test/support/connect-flow-test-harness.ts @@ -289,6 +289,11 @@ export function createConnectHarness(options: ConnectHarnessOptions = {}): Conne () => undefined, ); const requestedPortableDisposition = options.portableReceiptDisposition ?? { kind: "absent" }; + vi.spyOn(gatewayState, "hermesPortableLifecycleLockOptions").mockReturnValue( + requestedPortableDisposition.kind === "hermes" + ? { stateDir: "/home/test/.nemoclaw/state" } + : undefined, + ); const portableDisposition = requestedPortableDisposition.kind === "hermes" ? {