diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 44b4f4fc233..06f31af27b0 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -28,7 +28,7 @@ "src/lib/onboard/gateway-binding.ts": 53, "src/lib/runner.ts": 83, "src/lib/security/redact.ts": 52, - "src/lib/state/mcp-lifecycle-lock.ts": 21, + "src/lib/state/mcp-lifecycle-lock.ts": 19, "src/lib/state/onboard-session.ts": 35, "src/lib/state/registry.ts": 96, "src/lib/state/state-root.ts": 21, diff --git a/src/lib/actions/sandbox/connect-hermes-accepted-readiness.test.ts b/src/lib/actions/sandbox/connect-hermes-accepted-readiness.test.ts index aabe5bdf8c2..f78db2996ed 100644 --- a/src/lib/actions/sandbox/connect-hermes-accepted-readiness.test.ts +++ b/src/lib/actions/sandbox/connect-hermes-accepted-readiness.test.ts @@ -1,31 +1,46 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + import { afterEach, beforeEach, describe, expect, it, type MockInstance, vi } from "vitest"; +import { createHermesPortableUninstallFixture } from "../../../../test/helpers/hermes-portable-uninstall-fixture"; import { createConnectHarness } from "../../../../test/support/connect-flow-test-harness"; import type { OpenShellSandboxBufferedCommandRequest } from "../../adapters/openshell/sandbox-command"; +import { hermesPortableReceiptDirectory } from "../../onboard/experimental/hermes-portable-receipt"; +import type { SandboxEntry } from "../../state/registry"; import { HermesPortableForwardRecoveryError } from "./probe/hermes-portable-forward-recovery"; const originalStdoutIsTty = process.stdout.isTTY; -function acceptedHermesHarness(provider: string | null, model: string | null) { - const entry = { - name: "alpha", - agent: "hermes", - provider, - model, - policies: [], - openshellDriver: "docker", - gatewayName: "nemoclaw", - lifecycleGeneration: "generation-1", - lifecycleLiveIdentityFingerprint: "f".repeat(64), - } as never; +function acceptedHermesHarness( + provider: string | null, + model: string | null, + options: { entry?: SandboxEntry; useRealPortableReceipt?: boolean } = {}, +) { + const entry = + options.entry ?? + ({ + name: "alpha", + agent: "hermes", + provider, + model, + policies: [], + openshellDriver: "docker", + gatewayName: "nemoclaw", + lifecycleGeneration: "generation-1", + lifecycleLiveIdentityFingerprint: "f".repeat(64), + } as never); const harness = createConnectHarness({ agentName: "hermes", sessionAgent: { name: "hermes" }, registryEntry: entry, - portableReceiptDisposition: { kind: "hermes", phase: "active" }, + ...(options.useRealPortableReceipt + ? { useRealPortableReceipt: true } + : { portableReceiptDisposition: { kind: "hermes" as const, phase: "active" as const } }), readinessDecision: { kind: "accepted", category: "accepted", @@ -44,13 +59,14 @@ function acceptedHermesHarness(provider: string | null, model: string | null) { function configureHealthyForward(harness: ReturnType): void { const captureResolved = harness.captureResolvedOpenshellSpy.getMockImplementation()!; + const sandboxName = harness.registryEntries[0]!.name; harness.spawnSyncSpy.mockReturnValue({ status: 0, signal: null } as never); harness.captureResolvedOpenshellSpy.mockImplementation(((args: unknown, options: unknown) => { const argv = Array.isArray(args) ? args.map(String) : []; return argv[0] === "forward" && argv[1] === "list" ? { status: 0, - output: "SANDBOX BIND PORT PID STATUS\nalpha 127.0.0.1 18789 12345 running", + output: `SANDBOX BIND PORT PID STATUS\n${sandboxName} 127.0.0.1 18789 12345 running`, } : captureResolved(args, options); }) as never); @@ -144,6 +160,43 @@ describe("Hermes accepted launch-readiness probe", () => { ); }); + it("accepts only host-state Portable authority on a non-default gateway port", async () => { + const homeDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-connect-portable-state-")); + vi.stubEnv("HOME", homeDir); + vi.stubEnv("NEMOCLAW_GATEWAY_PORT", "18080"); + const fixture = await createHermesPortableUninstallFixture(homeDir); + const gatewayStateDir = path.join(homeDir, ".nemoclaw", "gateways", "18080"); + const gatewayReceiptDirectory = hermesPortableReceiptDirectory( + fixture.targetRow.name, + gatewayStateDir, + ); + fs.mkdirSync(path.dirname(gatewayReceiptDirectory), { recursive: true, mode: 0o700 }); + fs.cpSync( + hermesPortableReceiptDirectory(fixture.targetRow.name, fixture.stateDir), + gatewayReceiptDirectory, + { recursive: true }, + ); + + try { + const entry = { ...fixture.targetRow, gatewayPort: 18_080 }; + const harness = acceptedHermesHarness(entry.provider ?? null, entry.model ?? null, { + entry, + useRealPortableReceipt: true, + }); + configureHealthyForward(harness); + + await expect( + harness.connectSandbox(entry.name, { probeOnly: true }), + ).resolves.toBeUndefined(); + + expect(harness.qualifyHermesPortableAcceptedReadinessAuthoritySpy).toHaveBeenCalled(); + expect(harness.checkAndRecoverSpy).not.toHaveBeenCalled(); + } finally { + fixture.restore(); + fs.rmSync(homeDir, { recursive: true, force: true }); + } + }); + it("publishes missing readiness for one running exact runtime without recovery", async () => { vi.stubEnv("PATH", "/hostile/ambient/bin"); const harness = missingHermesHarness(); diff --git a/src/lib/actions/sandbox/doctor-system-checks.ts b/src/lib/actions/sandbox/doctor-system-checks.ts index ed434c4983d..7de372ec6bc 100644 --- a/src/lib/actions/sandbox/doctor-system-checks.ts +++ b/src/lib/actions/sandbox/doctor-system-checks.ts @@ -16,7 +16,7 @@ import { resolveRuntimeProviderBundle, } from "../../onboard/runtime-provider/access"; import { qualifyPortableAgentLifecycleAuthority } from "../../onboard/experimental/portable-agent-lifecycle"; -import { withMcpLifecycleLock } from "../../state/mcp-lifecycle-lock-acquisition"; +import { withSandboxLifecycleLock } from "./lifecycle/lock"; import type { SandboxEntry } from "../../state/registry"; import { readCloudflaredState } from "../../tunnel/services"; import { @@ -26,7 +26,7 @@ import { import { captureHostCommand } from "./doctor-host-command"; import type { DoctorCheck } from "./doctor-report"; -export const withSandboxDoctorLifecycleLock = withMcpLifecycleLock; +export const withSandboxDoctorLifecycleLock = withSandboxLifecycleLock; export function gatewayDoctorStartHint(gatewayName: string): string { return `${gatewayStartGuidance(gatewayName)} Then retry this command.`; diff --git a/src/lib/actions/sandbox/launch-readiness.ts b/src/lib/actions/sandbox/launch-readiness.ts index 4d43d5bceb9..29f3271f823 100644 --- a/src/lib/actions/sandbox/launch-readiness.ts +++ b/src/lib/actions/sandbox/launch-readiness.ts @@ -46,7 +46,7 @@ import { publishLaunchReadinessLease, readLaunchReadinessLease, } from "../../state/launch-readiness-lease"; -import { withMcpLifecycleLock as withSandboxMutationLock } from "../../state/mcp-lifecycle-lock-acquisition"; +import { withSandboxLifecycleLock as withSandboxMutationLock } from "./lifecycle/lock"; import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry"; import { normalizeSandboxMcpState } from "../../state/registry"; import * as registry from "../../state/registry"; diff --git a/src/lib/actions/sandbox/lifecycle/lock.test.ts b/src/lib/actions/sandbox/lifecycle/lock.test.ts new file mode 100644 index 00000000000..c6e73cc7185 --- /dev/null +++ b/src/lib/actions/sandbox/lifecycle/lock.test.ts @@ -0,0 +1,214 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import * as receiptAuthority from "../../../onboard/experimental/hermes-portable-receipt"; +import { + portableLifecycleLockOptions, + resolveHermesPortableLifecycleLockOptions, +} from "../../../onboard/experimental/portable-lifecycle-lock"; +import { isMcpLifecycleLockHeld } from "../../../state/mcp-lifecycle-lock-acquisition"; +import { + portableHostFencePath, + withCurrentPortableHostFence, +} from "../../../state/portable-uninstall-retirement"; +import { withSandboxLifecycleLock, withSandboxLifecycleLockSync } from "./lock"; + +describe("Portable-aware sandbox lifecycle lock", () => { + let homeDir: string; + + beforeEach(() => { + homeDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-portable-lifecycle-lock-")); + vi.stubEnv("HOME", homeDir); + vi.stubEnv("NEMOCLAW_GATEWAY_PORT", "18080"); + vi.spyOn(receiptAuthority, "hasHermesPortableReceiptCandidate").mockReturnValue(true); + }); + + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + fs.rmSync(homeDir, { recursive: true, force: true }); + }); + + it("reuses one host-scoped lock through asynchronous and synchronous lifecycle layers", async () => { + const portableStateDir = portableLifecycleLockOptions(process.env).stateDir; + const gatewayStateDir = path.join(homeDir, ".nemoclaw", "gateways", "18080", "state"); + + await withSandboxLifecycleLock("alpha", async () => { + expect(fs.existsSync(portableHostFencePath(homeDir))).toBe(true); + expect(isMcpLifecycleLockHeld("alpha", portableStateDir)).toBe(true); + expect(isMcpLifecycleLockHeld("alpha", gatewayStateDir)).toBe(false); + + withSandboxLifecycleLockSync("alpha", () => { + expect(isMcpLifecycleLockHeld("alpha", portableStateDir)).toBe(true); + expect(isMcpLifecycleLockHeld("alpha", gatewayStateDir)).toBe(false); + }); + + await withSandboxLifecycleLock("alpha", () => { + expect(isMcpLifecycleLockHeld("alpha", portableStateDir)).toBe(true); + expect(isMcpLifecycleLockHeld("alpha", gatewayStateDir)).toBe(false); + }); + }); + + expect(fs.existsSync(portableHostFencePath(homeDir))).toBe(false); + }); + + it("selects host receipt state only for a Hermes Portable candidate", () => { + const env = { HOME: homeDir, NEMOCLAW_GATEWAY_PORT: "18080" }; + + expect(resolveHermesPortableLifecycleLockOptions("alpha", env, () => true)).toEqual({ + stateDir: path.join(homeDir, ".nemoclaw", "state"), + }); + expect(resolveHermesPortableLifecycleLockOptions("alpha", env, () => false)).toBeUndefined(); + }); + + it("rejects every direct synchronous operation without the host fence", () => { + expect(() => withSandboxLifecycleLockSync("alpha", () => undefined)).toThrow( + "Portable host authority mutation requires the current HOME fence", + ); + }); + + it("classifies absent-to-Portable transitions after acquiring the host fence", async () => { + vi.restoreAllMocks(); + vi.resetModules(); + const [freshReceiptAuthority, freshPortableLock, freshAcquisition, freshHostFence, freshLock] = + await Promise.all([ + import("../../../onboard/experimental/hermes-portable-receipt"), + import("../../../onboard/experimental/portable-lifecycle-lock"), + import("../../../state/mcp-lifecycle-lock-acquisition"), + import("../../../state/portable-uninstall-retirement"), + import("./lock"), + ]); + let candidate = false; + vi.spyOn(freshReceiptAuthority, "hasHermesPortableReceiptCandidate").mockImplementation( + () => candidate, + ); + let releaseTransition!: () => void; + const transitionBlocked = new Promise((resolve) => { + releaseTransition = resolve; + }); + let transitionStarted!: () => void; + const transitionEntered = new Promise((resolve) => { + transitionStarted = resolve; + }); + const transition = freshHostFence.withCurrentPortableHostFence(async () => { + transitionStarted(); + await transitionBlocked; + candidate = true; + }); + let startOperation!: () => void; + const operationStart = new Promise((resolve) => { + startOperation = resolve; + }); + const observed: string[] = []; + const operation = operationStart.then(() => + freshLock.withSandboxLifecycleLock("alpha", () => { + observed.push( + freshAcquisition.isMcpLifecycleLockHeld( + "alpha", + freshPortableLock.portableLifecycleLockOptions(process.env).stateDir, + ) + ? "portable" + : "gateway", + ); + }), + ); + await transitionEntered; + startOperation(); + await new Promise((resolve) => setImmediate(resolve)); + expect(observed).toEqual([]); + + releaseTransition(); + await Promise.all([transition, operation]); + expect(observed).toEqual(["portable"]); + }); + + it("classifies Portable-to-absent transitions after acquiring the host fence", async () => { + vi.restoreAllMocks(); + vi.resetModules(); + const [freshReceiptAuthority, freshPortableLock, freshAcquisition, freshHostFence, freshLock] = + await Promise.all([ + import("../../../onboard/experimental/hermes-portable-receipt"), + import("../../../onboard/experimental/portable-lifecycle-lock"), + import("../../../state/mcp-lifecycle-lock-acquisition"), + import("../../../state/portable-uninstall-retirement"), + import("./lock"), + ]); + let candidate = true; + vi.spyOn(freshReceiptAuthority, "hasHermesPortableReceiptCandidate").mockImplementation( + () => candidate, + ); + let releaseTransition!: () => void; + const transitionBlocked = new Promise((resolve) => { + releaseTransition = resolve; + }); + let transitionStarted!: () => void; + const transitionEntered = new Promise((resolve) => { + transitionStarted = resolve; + }); + const transition = freshHostFence.withCurrentPortableHostFence(async () => { + transitionStarted(); + await transitionBlocked; + candidate = false; + }); + const portableStateDir = freshPortableLock.portableLifecycleLockOptions(process.env).stateDir; + const gatewayStateDir = path.join(homeDir, ".nemoclaw", "gateways", "18080", "state"); + let startOperation!: () => void; + const operationStart = new Promise((resolve) => { + startOperation = resolve; + }); + const observed: string[] = []; + const operation = operationStart.then(() => + freshLock.withSandboxLifecycleLock("alpha", () => { + observed.push( + freshAcquisition.isMcpLifecycleLockHeld("alpha", gatewayStateDir) && + !freshAcquisition.isMcpLifecycleLockHeld("alpha", portableStateDir) + ? "gateway" + : "portable", + ); + }), + ); + await transitionEntered; + startOperation(); + await new Promise((resolve) => setImmediate(resolve)); + expect(observed).toEqual([]); + + releaseTransition(); + await Promise.all([transition, operation]); + expect(observed).toEqual(["gateway"]); + }); + + it("serializes competing Portable lifecycle operations on the same authority", async () => { + const events: string[] = []; + let releaseFirst!: () => void; + const firstWaiting = new Promise((resolve) => { + releaseFirst = resolve; + }); + let firstEntered!: () => void; + const firstStarted = new Promise((resolve) => { + firstEntered = resolve; + }); + + const first = withSandboxLifecycleLock("alpha", async () => { + events.push("first-entered"); + firstEntered(); + await firstWaiting; + events.push("first-released"); + }); + await firstStarted; + const second = withSandboxLifecycleLock("alpha", () => { + events.push("second-entered"); + }); + await new Promise((resolve) => setImmediate(resolve)); + + expect(events).toEqual(["first-entered"]); + releaseFirst(); + await Promise.all([first, second]); + expect(events).toEqual(["first-entered", "first-released", "second-entered"]); + }); +}); diff --git a/src/lib/actions/sandbox/lifecycle/lock.ts b/src/lib/actions/sandbox/lifecycle/lock.ts index 602f7eb519f..d17dfdf17d5 100644 --- a/src/lib/actions/sandbox/lifecycle/lock.ts +++ b/src/lib/actions/sandbox/lifecycle/lock.ts @@ -1,11 +1,53 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import os from "node:os"; + import { + type McpLifecycleLockOptions, withMcpLifecycleLock, withMcpLifecycleLockSync, } from "../../../state/mcp-lifecycle-lock-acquisition"; +import { + assertCurrentPortableHostFenceHeld, + withCurrentPortableHostFence, +} from "../../../state/portable-uninstall-retirement"; +import { resolveHermesPortableLifecycleLockOptions } from "../../../onboard/experimental/portable-lifecycle-lock"; + +function resolveLifecycleLockOptions( + sandboxName: string, + options: McpLifecycleLockOptions, +): McpLifecycleLockOptions { + if (options.stateDir !== undefined) return options; + const portable = resolveHermesPortableLifecycleLockOptions(sandboxName); + return portable ? { ...options, ...portable } : options; +} + +/** Select the sandbox lock domain while holding the host fence, then serialize the operation. */ +export async function withSandboxLifecycleLock( + sandboxName: string, + operation: () => Promise | T, + options: McpLifecycleLockOptions = {}, +): Promise { + return await withCurrentPortableHostFence(async () => { + const resolved = resolveLifecycleLockOptions(sandboxName, options); + return Object.keys(resolved).length === 0 + ? await withMcpLifecycleLock(sandboxName, operation) + : await withMcpLifecycleLock(sandboxName, operation, resolved); + }); +} + +/** Synchronous lifecycle operations are valid only beneath the asynchronous host fence. */ +export function withSandboxLifecycleLockSync( + sandboxName: string, + operation: () => T, + options: McpLifecycleLockOptions = {}, +): T { + assertCurrentPortableHostFenceHeld(process.env.HOME || os.homedir()); + const resolved = resolveLifecycleLockOptions(sandboxName, options); + return Object.keys(resolved).length === 0 + ? withMcpLifecycleLockSync(sandboxName, operation) + : withMcpLifecycleLockSync(sandboxName, operation, resolved); +} -export const withSandboxLifecycleLock = withMcpLifecycleLock; -export const withSandboxLifecycleLockSync = withMcpLifecycleLockSync; -export const withConnectSandboxLifecycleLock = withMcpLifecycleLock; +export const withConnectSandboxLifecycleLock = withSandboxLifecycleLock; diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index ad614de02e0..18fa4d891de 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -41,7 +41,7 @@ import { resolvePrivilegedSandboxTarget, withPrivilegedSandboxExecutionLease, } from "../../sandbox/privileged-exec"; -import { withMcpLifecycleLock } from "../../state/mcp-lifecycle-lock-acquisition"; +import { withSandboxLifecycleLock } from "./lifecycle/lock"; import * as registry from "../../state/registry"; import { buildSubprocessEnv } from "../../subprocess-env"; import { @@ -1096,7 +1096,7 @@ export async function restartSandboxGateway( const defaultSupervisorAction = runtimeSelection ? refuseHostLocalSupervisorForSelectedRuntime : executeGatewaySupervisorAction; - return withMcpLifecycleLock(sandboxName, () => + return withSandboxLifecycleLock(sandboxName, () => restartSandboxGatewayWithDeps(sandboxName, { quiet, deps: { @@ -2191,7 +2191,7 @@ export async function checkAndRecoverSandboxProcesses( runtimeSelection?: OpenShellRuntimeSelection; } = {}, ) { - return withMcpLifecycleLock(sandboxName, () => + return withSandboxLifecycleLock(sandboxName, () => checkAndRecoverSandboxProcessesWithoutHostLock(sandboxName, options), ); } diff --git a/src/lib/actions/sandbox/runtime/hermes-cron-restore-recovery.test.ts b/src/lib/actions/sandbox/runtime/hermes-cron-restore-recovery.test.ts index 329588ae706..d155d5b1078 100644 --- a/src/lib/actions/sandbox/runtime/hermes-cron-restore-recovery.test.ts +++ b/src/lib/actions/sandbox/runtime/hermes-cron-restore-recovery.test.ts @@ -9,7 +9,7 @@ const mocks = vi.hoisted(() => ({ inspectPortableAgentReceiptDisposition: vi.fn(), prepareHermesCronRestoreRecovery: vi.fn(), recoverHermesCronRestore: vi.fn(), - withMcpLifecycleLock: vi.fn( + withSandboxLifecycleLock: vi.fn( async (_sandboxName: string, operation: () => Promise, _options: unknown) => operation(), ), })); @@ -19,8 +19,8 @@ vi.mock("../../../agent/runtime", async (importOriginal) => ({ getSessionAgent: mocks.getSessionAgent, })); -vi.mock("../../../state/mcp-lifecycle-lock", () => ({ - withMcpLifecycleLock: mocks.withMcpLifecycleLock, +vi.mock("../lifecycle/lock", () => ({ + withSandboxLifecycleLock: mocks.withSandboxLifecycleLock, })); vi.mock("../../../onboard/experimental/portable-agent-lifecycle", () => ({ @@ -64,7 +64,7 @@ describe("sandbox recovery with a Hermes cron restore gate", () => { await recoverSandboxWithHermesCronRestore("alpha"); - expect(mocks.withMcpLifecycleLock).toHaveBeenCalledWith("alpha", expect.any(Function), { + expect(mocks.withSandboxLifecycleLock).toHaveBeenCalledWith("alpha", expect.any(Function), { timeoutMs: 30_000, }); expect(events).toEqual(["prepare", "connect", "recover"]); diff --git a/src/lib/actions/sandbox/runtime/hermes-cron-restore-recovery.ts b/src/lib/actions/sandbox/runtime/hermes-cron-restore-recovery.ts index 5c3b10f43ff..bfc432d121a 100644 --- a/src/lib/actions/sandbox/runtime/hermes-cron-restore-recovery.ts +++ b/src/lib/actions/sandbox/runtime/hermes-cron-restore-recovery.ts @@ -3,7 +3,7 @@ import * as agentRuntime from "../../../agent/runtime"; import { inspectPortableAgentReceiptDisposition } from "../../../onboard/experimental/portable-agent-lifecycle"; -import { withMcpLifecycleLock } from "../../../state/mcp-lifecycle-lock"; +import { withSandboxLifecycleLock } from "../lifecycle/lock"; import { connectSandbox } from "../connect"; import { prepareHermesCronRestoreRecovery, @@ -14,7 +14,7 @@ const RECOVERY_LOCK_TIMEOUT_MS = 30_000; /** Re-establish a Hermes gate before gateway repair, then validate and release it. */ export async function recoverSandboxWithHermesCronRestore(sandboxName: string): Promise { - await withMcpLifecycleLock( + await withSandboxLifecycleLock( sandboxName, async () => { const portable = inspectPortableAgentReceiptDisposition(sandboxName); diff --git a/src/lib/actions/sandbox/status.ts b/src/lib/actions/sandbox/status.ts index 3504e77479d..8736352a251 100644 --- a/src/lib/actions/sandbox/status.ts +++ b/src/lib/actions/sandbox/status.ts @@ -6,7 +6,7 @@ import { CLI_NAME } from "../../cli/branding"; import { deferSandboxLifecycleExit, isSandboxLifecycleDeferredExit } from "../../core/process-exit"; import { inspectManagedLlamaCppStatus } from "../../inference/llama-cpp/managed-status"; import { getGatewayPresets } from "../../policy"; -import { withMcpLifecycleLock } from "../../state/mcp-lifecycle-lock-acquisition"; +import { withSandboxLifecycleLock } from "./lifecycle/lock"; import * as registry from "../../state/registry"; import { getSandboxDockerRuntime } from "./docker-health"; import { @@ -124,7 +124,7 @@ export async function getSandboxStatusReport( sandboxName: string, deps: Parameters[1] = {}, ): Promise { - return withMcpLifecycleLock(sandboxName, async () => { + return withSandboxLifecycleLock(sandboxName, async () => { const hermesPortable = inspectHermesPortableStatus(sandboxName); if (hermesPortable) { return hermesPortableStatusReport( @@ -157,7 +157,7 @@ function maybeEnsureHermesToolGatewayBroker(sb: registry.SandboxEntry | null): v export async function showSandboxStatus(sandboxName: string): Promise { let deferredExitCode: number | null = null; try { - await withMcpLifecycleLock(sandboxName, async () => { + await withSandboxLifecycleLock(sandboxName, async () => { const hermesPortable = inspectHermesPortableStatus(sandboxName); if (hermesPortable) { console.log(` Sandbox: ${sandboxName}`); diff --git a/src/lib/cli/nemoclaw-oclif-command.test.ts b/src/lib/cli/nemoclaw-oclif-command.test.ts index 091ace99ef3..a98400f2121 100644 --- a/src/lib/cli/nemoclaw-oclif-command.test.ts +++ b/src/lib/cli/nemoclaw-oclif-command.test.ts @@ -154,17 +154,19 @@ class PortableStartCommand extends NemoClawCommand { public async run(): Promise { const { args } = await this.parse(PortableStartCommand); - const sandboxName = args.sandboxName!; - PortableStartCommand.observed = { - host: fs.existsSync( - portableHostAuthority.portableHostFencePath(process.env.HOME || os.homedir()), - ), - lifecycle: isMcpLifecycleLockHeld(sandboxName), - portableLifecycle: isMcpLifecycleLockHeld( - sandboxName, - path.join(portableHostAuthority.defaultPortableStateDir(process.env), "state"), - ), - }; + PortableStartCommand.observed = observeLifecycleAuthority(args.sandboxName!); + } +} + +class PortableLaunchCommand extends NemoClawCommand { + static id = "launch"; + static args = { sandboxName: Args.string({ required: true }) }; + static flags = {}; + static observed = { host: false, lifecycle: false, portableLifecycle: false }; + + public async run(): Promise { + const { args } = await this.parse(PortableLaunchCommand); + PortableLaunchCommand.observed = observeLifecycleAuthority(args.sandboxName!); } } @@ -176,20 +178,23 @@ class PortableStopCommand extends NemoClawCommand { public async run(): Promise { const { args } = await this.parse(PortableStopCommand); - const sandboxName = args.sandboxName!; - PortableStopCommand.observed = { - host: fs.existsSync( - portableHostAuthority.portableHostFencePath(process.env.HOME || os.homedir()), - ), - lifecycle: isMcpLifecycleLockHeld(sandboxName), - portableLifecycle: isMcpLifecycleLockHeld( - sandboxName, - path.join(portableHostAuthority.defaultPortableStateDir(process.env), "state"), - ), - }; + PortableStopCommand.observed = observeLifecycleAuthority(args.sandboxName!); } } +function observeLifecycleAuthority(sandboxName: string) { + return { + host: fs.existsSync( + portableHostAuthority.portableHostFencePath(process.env.HOME || os.homedir()), + ), + lifecycle: isMcpLifecycleLockHeld(sandboxName), + portableLifecycle: isMcpLifecycleLockHeld( + sandboxName, + path.join(portableHostAuthority.defaultPortableStateDir(process.env), "state"), + ), + }; +} + function useHermesPortableAuthority(): void { vi.spyOn(receiptAuthority, "hasHermesPortableReceiptCandidate").mockReturnValue(true); vi.spyOn( @@ -236,6 +241,7 @@ describe("NemoClawCommand", () => { GlobalUseMutationCommand.ran = false; ProbeOnlyConnectCommand.operation = () => undefined; PortableStartCommand.observed = { host: false, lifecycle: false, portableLifecycle: false }; + PortableLaunchCommand.observed = { host: false, lifecycle: false, portableLifecycle: false }; PortableStopCommand.observed = { host: false, lifecycle: false, portableLifecycle: false }; }); @@ -387,20 +393,22 @@ describe("NemoClawCommand", () => { }); }); - it("uses the Portable lifecycle lock for stop without broadening the host fence", async () => { + it("uses the same Portable host and lifecycle fences for launch and stop", async () => { vi.stubEnv("NEMOCLAW_GATEWAY_PORT", "18080"); useHermesPortableAuthority(); + await PortableLaunchCommand.run(["alpha"], process.cwd()); await PortableStopCommand.run(["alpha"], process.cwd()); - expect(PortableStopCommand.observed).toEqual({ - host: false, + expect(PortableLaunchCommand.observed).toEqual({ + host: true, lifecycle: false, portableLifecycle: true, }); + expect(PortableStopCommand.observed).toEqual(PortableLaunchCommand.observed); }); - it("does not create the Portable host fence when a lifecycle command has no Hermes receipt candidate", async () => { + it("selects the gateway lifecycle lock under the host fence when there is no Hermes receipt", async () => { vi.stubEnv("HOME", stateDir); vi.stubEnv("NEMOCLAW_TEST_BASE_HOME", stateDir); vi.spyOn(receiptAuthority, "hasHermesPortableReceiptCandidate").mockReturnValue(false); @@ -409,12 +417,12 @@ describe("NemoClawCommand", () => { await PortableStartCommand.run(["alpha"], process.cwd()); expect(ProbeOnlyConnectCommand.observed).toEqual({ - host: false, + host: true, lifecycle: true, portableLifecycle: false, }); expect(PortableStartCommand.observed).toEqual({ - host: false, + host: true, lifecycle: true, portableLifecycle: false, }); diff --git a/src/lib/cli/nemoclaw-oclif-command.ts b/src/lib/cli/nemoclaw-oclif-command.ts index b0bdf51def0..3915d289319 100644 --- a/src/lib/cli/nemoclaw-oclif-command.ts +++ b/src/lib/cli/nemoclaw-oclif-command.ts @@ -6,7 +6,6 @@ import { assertHermesPortableCommandSupported, assertHermesPortableCommandUnavailable, classifyHermesPortableCommand, - hermesPortableLifecycleLockOptions, HERMES_PORTABLE_UNSUPPORTED_COMMAND_MESSAGE, HERMES_PORTABLE_UNSUPPORTED_DOCTOR_FIX_MESSAGE, } from "../onboard/experimental/portable-agent-lifecycle"; @@ -16,7 +15,7 @@ import { assertNoHermesPortableHostAuthority, withCurrentPortableHostFence, } from "../state/portable-uninstall-retirement"; -import { withMcpLifecycleLock } from "../state/mcp-lifecycle-lock"; +import { withSandboxLifecycleLock } from "../actions/sandbox/lifecycle/lock"; import { enforceRemovedImmutabilityMigrationBoundary, reportRemovedImmutabilityUpgrade, @@ -31,7 +30,7 @@ export type CommandExitResult = { export { HERMES_PORTABLE_UNSUPPORTED_COMMAND_MESSAGE }; export { assertHermesPortableCommandUnavailable }; -export const withSandboxCommandLifecycleLock = withMcpLifecycleLock; +export const withSandboxCommandLifecycleLock = withSandboxLifecycleLock; export { HERMES_PORTABLE_UNSUPPORTED_DOCTOR_FIX_MESSAGE }; const REMOVED_IMMUTABILITY_REMEDIATION_COMMANDS = new Set([ @@ -142,26 +141,7 @@ export abstract class NemoClawCommand extends Command { } return super._run(); }; - const portableLifecycleLockOptions = hermesPortableLifecycleLockOptions( - sandboxName, - process.env, - ); - const usesHermesPortableHostAuthority = - (commandId === "sandbox:start" || this.isProbeOnlyConnect(commandId)) && - portableLifecycleLockOptions !== undefined; - const runWithLifecycleFence = async () => { - return await withMcpLifecycleLock(sandboxName, runLocked, portableLifecycleLockOptions); - }; - if (usesHermesPortableHostAuthority) { - return await withCurrentPortableHostFence(runWithLifecycleFence); - } - return await runWithLifecycleFence(); - } - - private isProbeOnlyConnect(commandId: string | undefined): boolean { - return ( - commandId === "sandbox:connect" && this.lifecycleParserOutput?.flags["probe-only"] === true - ); + return await withSandboxLifecycleLock(sandboxName, runLocked); } private isInteractiveConnect(commandId: string | undefined): boolean { diff --git a/src/lib/onboard/experimental/hermes-portable-onboarding.ts b/src/lib/onboard/experimental/hermes-portable-onboarding.ts index e189da4222f..e788a301399 100644 --- a/src/lib/onboard/experimental/hermes-portable-onboarding.ts +++ b/src/lib/onboard/experimental/hermes-portable-onboarding.ts @@ -44,6 +44,7 @@ import { } from "../../openshell-gateway-endpoint-guard"; import { isPortableExperimentalProfile } from "./portable-profile"; import { defaultPortableDemoStateDir } from "./portable-runtime-receipt-readiness"; +import { portableLifecycleLockOptions } from "./portable-lifecycle-lock"; export { defaultPortableDemoStateDir as defaultHermesPortableStateDir }; type McpLifecycleLock = ( @@ -57,9 +58,8 @@ export function bindHermesPortableOnboardingLifecycleLock( withMcpLifecycleLock: McpLifecycleLock, env: NodeJS.ProcessEnv = process.env, ): (sandboxName: string, operation: () => Promise) => Promise { - const stateDir = path.join(defaultPortableDemoStateDir(env), "state"); return async (sandboxName: string, operation: () => Promise): Promise => - await withMcpLifecycleLock(sandboxName, operation, { stateDir }); + await withMcpLifecycleLock(sandboxName, operation, portableLifecycleLockOptions(env)); } import { assertCurrentHermesPortableContainer, diff --git a/src/lib/onboard/experimental/portable-agent-lifecycle.ts b/src/lib/onboard/experimental/portable-agent-lifecycle.ts index b201de5d84d..2ab2a11a338 100644 --- a/src/lib/onboard/experimental/portable-agent-lifecycle.ts +++ b/src/lib/onboard/experimental/portable-agent-lifecycle.ts @@ -35,6 +35,7 @@ import { type PortableDemoLifecycleRecoveryResult, type PortableDemoLifecycleStopResult, } from "./portable-demo-lifecycle"; +import { resolveHermesPortableLifecycleLockOptions } from "./portable-lifecycle-lock"; import { defaultPortableDemoStateDir } from "./portable-runtime-receipt-readiness"; export { defaultPortableDemoStateDir }; @@ -45,10 +46,9 @@ export function hermesPortableLifecycleLockOptions( env: NodeJS.ProcessEnv = process.env, hasReceiptCandidate: typeof hasHermesPortableReceiptCandidate = hasHermesPortableReceiptCandidate, ): { readonly stateDir: string } | undefined { - if (!hasReceiptCandidate(sandboxName, defaultPortableDemoStateDir(env))) { - return undefined; - } - return { stateDir: path.join(defaultPortableDemoStateDir(env), "state") }; + return resolveHermesPortableLifecycleLockOptions(sandboxName, env, (name, environment) => + hasReceiptCandidate(name, defaultPortableDemoStateDir(environment)), + ); } export type PortableAgentLifecycleDeps = PortableDemoLifecycleDeps & HermesPortableLifecycleDeps; diff --git a/src/lib/onboard/experimental/portable-lifecycle-lock.ts b/src/lib/onboard/experimental/portable-lifecycle-lock.ts new file mode 100644 index 00000000000..37cb084bf42 --- /dev/null +++ b/src/lib/onboard/experimental/portable-lifecycle-lock.ts @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import path from "node:path"; + +import type { McpLifecycleLockOptions } from "../../state/mcp-lifecycle-lock"; +import { defaultPortableStateDir } from "../../state/portable-uninstall-retirement"; +import { hasHermesPortableReceiptCandidate } from "./hermes-portable-receipt"; + +export type HermesPortableReceiptCandidate = ( + sandboxName: string, + env: NodeJS.ProcessEnv, +) => boolean; + +/** Return the one host-scoped lifecycle-lock location owned by Portable receipts. */ +export function portableLifecycleLockOptions( + env: NodeJS.ProcessEnv = process.env, +): McpLifecycleLockOptions & { readonly stateDir: string } { + return { stateDir: path.join(defaultPortableStateDir(env), "state") }; +} + +/** Select Portable locking only when the sandbox has Hermes receipt authority. */ +export function resolveHermesPortableLifecycleLockOptions( + sandboxName: string, + env: NodeJS.ProcessEnv = process.env, + hasReceiptCandidate: HermesPortableReceiptCandidate = (name, environment) => + hasHermesPortableReceiptCandidate(name, defaultPortableStateDir(environment)), +): (McpLifecycleLockOptions & { readonly stateDir: string }) | undefined { + return hasReceiptCandidate(sandboxName, env) ? portableLifecycleLockOptions(env) : undefined; +} diff --git a/src/lib/onboard/runtime-provider/docker.ts b/src/lib/onboard/runtime-provider/docker.ts index 68859edf054..a06e0157dc5 100644 --- a/src/lib/onboard/runtime-provider/docker.ts +++ b/src/lib/onboard/runtime-provider/docker.ts @@ -26,12 +26,12 @@ import { } from "../experimental/docker-network-authority"; import { hasPortableAgentSandboxLifecycleReceipt, - hermesPortableLifecycleLockOptions, recoverPortableAgentSandboxLifecycle, requalifyPortableAgentSandboxAuthority, stopPortableAgentSandboxLifecycle, } from "../experimental/portable-agent-lifecycle"; import { withMcpLifecycleLockSync } from "../../state/mcp-lifecycle-lock-acquisition"; +import { resolveHermesPortableLifecycleLockOptions } from "../experimental/portable-lifecycle-lock"; import { queryOpenShellDockerSandboxRuntimeSnapshot } from "../openshell-docker-sandbox-containers"; import { validateSandboxGpuPreflight } from "../sandbox-gpu-preflight"; import { @@ -307,12 +307,11 @@ function dockerLifecycleLockOptions( input: RuntimeProviderLifecycleInput, deps: DockerRuntimeProviderDependencies, ): { readonly stateDir: string } | undefined { - return hermesPortableLifecycleLockOptions( + if (input.sandbox.agent !== "hermes") return undefined; + return resolveHermesPortableLifecycleLockOptions( input.sandboxName, input.environment, - () => - input.sandbox.agent === "hermes" && - deps.hasPortableLifecycleReceipt(input.sandboxName, input.environment), + deps.hasPortableLifecycleReceipt, ); } diff --git a/test/onboarding/onboard-hermes-portable-provider-publication.test.ts b/test/onboarding/onboard-hermes-portable-provider-publication.test.ts index f6f9a527794..1503814554e 100644 --- a/test/onboarding/onboard-hermes-portable-provider-publication.test.ts +++ b/test/onboarding/onboard-hermes-portable-provider-publication.test.ts @@ -27,7 +27,6 @@ type ProviderBoundaryMode = | "superseded"; type ProviderBoundaryResult = { - binderCalls: number; events: string[]; firstError: string | null; gpuCreateCalls: number; @@ -70,7 +69,6 @@ const sandboxProviderCleanupId = require.resolve(${modulePath("onboard/sandbox-p const normalize = (command) => Array.isArray(command) ? command.map(String) : [String(command)]; const events = []; const providerCalls = []; -let binderCalls = 0; let gpuCreateCalls = 0; let portableLockInvocations = 0; let portableTransactions = 0; @@ -199,23 +197,16 @@ require.cache[agentOnboardId].exports = { }; const sandboxGpuCreateFlow = require(sandboxGpuCreateFlowId); -const portableLifecycleLock = async (name, operation) => { - assert.equal(name, sandboxName); - portableLockInvocations += 1; - return await operation(); -}; require.cache[sandboxGpuCreateFlowId].exports = { ...sandboxGpuCreateFlow, - bindHermesPortableOnboardingLifecycleLock: (withLifecycleLock) => { - assert.equal(withLifecycleLock, lifecycleLock.withMcpLifecycleLock); - binderCalls += 1; - return portableLifecycleLock; - }, runHermesPortableOnboardingFromOnboard: async (input) => { portableTransactions += 1; events.push("portable:transaction"); - assert.equal(input.withLifecycleLock, portableLifecycleLock); - await input.withLifecycleLock(sandboxName, async () => {}); + await input.withLifecycleLock(sandboxName, async () => { + portableLockInvocations += 1; + const portableStateDir = process.env.HOME + "/.nemoclaw/state"; + assert.equal(lifecycleLock.isMcpLifecycleLockHeld(sandboxName, portableStateDir), true); + }); if (${JSON.stringify(mode)} === "superseded") return { created: false }; const attemptArgv = [...input.createArgv]; const separator = attemptArgv.indexOf("--"); @@ -325,7 +316,6 @@ const { resolveSandboxGpuConfig } = require(${modulePath("onboard/sandbox-gpu-mo const result = await createSandbox(...createArgs); console.log( JSON.stringify({ - binderCalls, events, firstError, gpuCreateCalls, @@ -371,7 +361,6 @@ describe("sandbox-create provider publication branches", () => { assert.equal(payload.result, "my-assistant"); assert.deepEqual(payload.providerCalls, expectedProviderCalls); - assert.equal(payload.binderCalls, 1); assert.equal(payload.portableLockInvocations, 1); assert.equal(payload.portableTransactions, 1); assert.ok( diff --git a/test/support/connect-flow-test-harness.ts b/test/support/connect-flow-test-harness.ts index 5b7c720ce49..f5fe31c5c95 100644 --- a/test/support/connect-flow-test-harness.ts +++ b/test/support/connect-flow-test-harness.ts @@ -139,6 +139,7 @@ export type ConnectHarnessOptions = { gatewayName?: string; lifecycleGeneration?: string; }; + useRealPortableReceipt?: boolean; dockerRuntime?: { health?: string; paused?: boolean; @@ -312,32 +313,45 @@ export function createConnectHarness(options: ConnectHarnessOptions = {}): Conne liveIdentityFingerprint: "f".repeat(64), } : requestedPortableDisposition; - const inspectPortableReceiptDispositionSpy = vi - .spyOn(portableAgentLifecycle, "inspectPortableAgentReceiptDisposition") - .mockReturnValue(portableDisposition); + const inspectPortableReceiptDispositionSpy = vi.spyOn( + portableAgentLifecycle, + "inspectPortableAgentReceiptDisposition", + ); + if (!options.useRealPortableReceipt) { + inspectPortableReceiptDispositionSpy.mockReturnValue(portableDisposition); + } let registryEntries: SandboxEntry[] = []; const qualifyPortableAgentLifecycleAuthority = portableAgentLifecycle.qualifyPortableAgentLifecycleAuthority; const requireHermesPortableActiveLifecycleAuthority = portableAgentLifecycle.requireHermesPortableActiveLifecycleAuthority; const portableAuthorityDeps = () => ({ - inspectReceiptDisposition: (sandboxName: string) => - portableAgentLifecycle.inspectPortableAgentReceiptDisposition(sandboxName), + ...(options.useRealPortableReceipt + ? {} + : { + inspectReceiptDisposition: (sandboxName: string) => + portableAgentLifecycle.inspectPortableAgentReceiptDisposition(sandboxName), + }), readRegistry: (sandboxName: string) => registryEntries.find((candidate) => candidate.name === sandboxName) ?? null, }); vi.spyOn(gatewayState, "qualifyPortableAgentLifecycleAuthority").mockImplementation((( sandboxName: string, - ) => qualifyPortableAgentLifecycleAuthority(sandboxName, portableAuthorityDeps())) as never); + deps: object, + ) => + qualifyPortableAgentLifecycleAuthority(sandboxName, { + ...deps, + ...portableAuthorityDeps(), + })) as never); vi.spyOn(gatewayState, "requireHermesPortableActiveLifecycleAuthority").mockImplementation((( sandboxName: string, expected: unknown, + deps: object, ) => - requireHermesPortableActiveLifecycleAuthority( - sandboxName, - expected, - portableAuthorityDeps(), - )) as never); + requireHermesPortableActiveLifecycleAuthority(sandboxName, expected, { + ...deps, + ...portableAuthorityDeps(), + })) as never); const recoverHermesPortableOllamaInferenceSpy = vi .spyOn(hermesInferenceRecovery, "recoverHermesPortableInferenceForConnectProbe") .mockImplementation((async (input: {