Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ci/source-architecture-budget.json
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
"src/lib/onboard/gateway-binding.ts": 53,
"src/lib/runner.ts": 83,
"src/lib/security/redact.ts": 52,
"src/lib/state/mcp-lifecycle-lock.ts": 21,
"src/lib/state/mcp-lifecycle-lock.ts": 19,
"src/lib/state/onboard-session.ts": 35,
"src/lib/state/registry.ts": 96,
"src/lib/state/state-root.ts": 21,
Expand Down
81 changes: 67 additions & 14 deletions src/lib/actions/sandbox/connect-hermes-accepted-readiness.test.ts
Original file line number Diff line number Diff line change
@@ -1,31 +1,46 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import fs from "node:fs";
import os from "node:os";
import path from "node:path";

import { afterEach, beforeEach, describe, expect, it, type MockInstance, vi } from "vitest";

import { createHermesPortableUninstallFixture } from "../../../../test/helpers/hermes-portable-uninstall-fixture";
import { createConnectHarness } from "../../../../test/support/connect-flow-test-harness";
import type { OpenShellSandboxBufferedCommandRequest } from "../../adapters/openshell/sandbox-command";
import { hermesPortableReceiptDirectory } from "../../onboard/experimental/hermes-portable-receipt";
import type { SandboxEntry } from "../../state/registry";
import { HermesPortableForwardRecoveryError } from "./probe/hermes-portable-forward-recovery";

const originalStdoutIsTty = process.stdout.isTTY;

function acceptedHermesHarness(provider: string | null, model: string | null) {
const entry = {
name: "alpha",
agent: "hermes",
provider,
model,
policies: [],
openshellDriver: "docker",
gatewayName: "nemoclaw",
lifecycleGeneration: "generation-1",
lifecycleLiveIdentityFingerprint: "f".repeat(64),
} as never;
function acceptedHermesHarness(
provider: string | null,
model: string | null,
options: { entry?: SandboxEntry; useRealPortableReceipt?: boolean } = {},
) {
const entry =
options.entry ??
({
name: "alpha",
agent: "hermes",
provider,
model,
policies: [],
openshellDriver: "docker",
gatewayName: "nemoclaw",
lifecycleGeneration: "generation-1",
lifecycleLiveIdentityFingerprint: "f".repeat(64),
} as never);
const harness = createConnectHarness({
agentName: "hermes",
sessionAgent: { name: "hermes" },
registryEntry: entry,
portableReceiptDisposition: { kind: "hermes", phase: "active" },
...(options.useRealPortableReceipt
? { useRealPortableReceipt: true }
: { portableReceiptDisposition: { kind: "hermes" as const, phase: "active" as const } }),
readinessDecision: {
kind: "accepted",
category: "accepted",
Expand All @@ -44,13 +59,14 @@ function acceptedHermesHarness(provider: string | null, model: string | null) {

function configureHealthyForward(harness: ReturnType<typeof acceptedHermesHarness>): void {
const captureResolved = harness.captureResolvedOpenshellSpy.getMockImplementation()!;
const sandboxName = harness.registryEntries[0]!.name;
harness.spawnSyncSpy.mockReturnValue({ status: 0, signal: null } as never);
harness.captureResolvedOpenshellSpy.mockImplementation(((args: unknown, options: unknown) => {
const argv = Array.isArray(args) ? args.map(String) : [];
return argv[0] === "forward" && argv[1] === "list"
? {
status: 0,
output: "SANDBOX BIND PORT PID STATUS\nalpha 127.0.0.1 18789 12345 running",
output: `SANDBOX BIND PORT PID STATUS\n${sandboxName} 127.0.0.1 18789 12345 running`,
}
: captureResolved(args, options);
}) as never);
Expand Down Expand Up @@ -144,6 +160,43 @@ describe("Hermes accepted launch-readiness probe", () => {
);
});

it("accepts only host-state Portable authority on a non-default gateway port", async () => {
const homeDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-connect-portable-state-"));
vi.stubEnv("HOME", homeDir);
vi.stubEnv("NEMOCLAW_GATEWAY_PORT", "18080");
const fixture = await createHermesPortableUninstallFixture(homeDir);
const gatewayStateDir = path.join(homeDir, ".nemoclaw", "gateways", "18080");
const gatewayReceiptDirectory = hermesPortableReceiptDirectory(
fixture.targetRow.name,
gatewayStateDir,
);
fs.mkdirSync(path.dirname(gatewayReceiptDirectory), { recursive: true, mode: 0o700 });
fs.cpSync(
Comment thread
prekshivyas marked this conversation as resolved.
hermesPortableReceiptDirectory(fixture.targetRow.name, fixture.stateDir),
gatewayReceiptDirectory,
{ recursive: true },
);

try {
const entry = { ...fixture.targetRow, gatewayPort: 18_080 };
const harness = acceptedHermesHarness(entry.provider ?? null, entry.model ?? null, {
entry,
useRealPortableReceipt: true,
});
configureHealthyForward(harness);

await expect(
harness.connectSandbox(entry.name, { probeOnly: true }),
).resolves.toBeUndefined();

expect(harness.qualifyHermesPortableAcceptedReadinessAuthoritySpy).toHaveBeenCalled();
expect(harness.checkAndRecoverSpy).not.toHaveBeenCalled();
} finally {
fixture.restore();
fs.rmSync(homeDir, { recursive: true, force: true });
}
});

it("publishes missing readiness for one running exact runtime without recovery", async () => {
vi.stubEnv("PATH", "/hostile/ambient/bin");
const harness = missingHermesHarness();
Expand Down
4 changes: 2 additions & 2 deletions src/lib/actions/sandbox/doctor-system-checks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import {
resolveRuntimeProviderBundle,
} from "../../onboard/runtime-provider/access";
import { qualifyPortableAgentLifecycleAuthority } from "../../onboard/experimental/portable-agent-lifecycle";
import { withMcpLifecycleLock } from "../../state/mcp-lifecycle-lock-acquisition";
import { withSandboxLifecycleLock } from "./lifecycle/lock";
import type { SandboxEntry } from "../../state/registry";
import { readCloudflaredState } from "../../tunnel/services";
import {
Expand All @@ -26,7 +26,7 @@ import {
import { captureHostCommand } from "./doctor-host-command";
import type { DoctorCheck } from "./doctor-report";

export const withSandboxDoctorLifecycleLock = withMcpLifecycleLock;
export const withSandboxDoctorLifecycleLock = withSandboxLifecycleLock;

export function gatewayDoctorStartHint(gatewayName: string): string {
return `${gatewayStartGuidance(gatewayName)} Then retry this command.`;
Expand Down
2 changes: 1 addition & 1 deletion src/lib/actions/sandbox/launch-readiness.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ import {
publishLaunchReadinessLease,
readLaunchReadinessLease,
} from "../../state/launch-readiness-lease";
import { withMcpLifecycleLock as withSandboxMutationLock } from "../../state/mcp-lifecycle-lock-acquisition";
import { withSandboxLifecycleLock as withSandboxMutationLock } from "./lifecycle/lock";
import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry";
import { normalizeSandboxMcpState } from "../../state/registry";
import * as registry from "../../state/registry";
Expand Down
214 changes: 214 additions & 0 deletions src/lib/actions/sandbox/lifecycle/lock.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,214 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import fs from "node:fs";
import os from "node:os";
import path from "node:path";

import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";

import * as receiptAuthority from "../../../onboard/experimental/hermes-portable-receipt";
import {
portableLifecycleLockOptions,
resolveHermesPortableLifecycleLockOptions,
} from "../../../onboard/experimental/portable-lifecycle-lock";
import { isMcpLifecycleLockHeld } from "../../../state/mcp-lifecycle-lock-acquisition";
import {
portableHostFencePath,
withCurrentPortableHostFence,
} from "../../../state/portable-uninstall-retirement";
import { withSandboxLifecycleLock, withSandboxLifecycleLockSync } from "./lock";

describe("Portable-aware sandbox lifecycle lock", () => {
let homeDir: string;

beforeEach(() => {
homeDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-portable-lifecycle-lock-"));
vi.stubEnv("HOME", homeDir);
vi.stubEnv("NEMOCLAW_GATEWAY_PORT", "18080");
vi.spyOn(receiptAuthority, "hasHermesPortableReceiptCandidate").mockReturnValue(true);
});

afterEach(() => {
vi.restoreAllMocks();
vi.unstubAllEnvs();
fs.rmSync(homeDir, { recursive: true, force: true });
});

it("reuses one host-scoped lock through asynchronous and synchronous lifecycle layers", async () => {
const portableStateDir = portableLifecycleLockOptions(process.env).stateDir;
const gatewayStateDir = path.join(homeDir, ".nemoclaw", "gateways", "18080", "state");

await withSandboxLifecycleLock("alpha", async () => {
expect(fs.existsSync(portableHostFencePath(homeDir))).toBe(true);
expect(isMcpLifecycleLockHeld("alpha", portableStateDir)).toBe(true);
expect(isMcpLifecycleLockHeld("alpha", gatewayStateDir)).toBe(false);

withSandboxLifecycleLockSync("alpha", () => {
expect(isMcpLifecycleLockHeld("alpha", portableStateDir)).toBe(true);
expect(isMcpLifecycleLockHeld("alpha", gatewayStateDir)).toBe(false);
});

await withSandboxLifecycleLock("alpha", () => {
expect(isMcpLifecycleLockHeld("alpha", portableStateDir)).toBe(true);
expect(isMcpLifecycleLockHeld("alpha", gatewayStateDir)).toBe(false);
});
});

expect(fs.existsSync(portableHostFencePath(homeDir))).toBe(false);
});

it("selects host receipt state only for a Hermes Portable candidate", () => {
const env = { HOME: homeDir, NEMOCLAW_GATEWAY_PORT: "18080" };

expect(resolveHermesPortableLifecycleLockOptions("alpha", env, () => true)).toEqual({
stateDir: path.join(homeDir, ".nemoclaw", "state"),
});
expect(resolveHermesPortableLifecycleLockOptions("alpha", env, () => false)).toBeUndefined();
});

it("rejects every direct synchronous operation without the host fence", () => {
expect(() => withSandboxLifecycleLockSync("alpha", () => undefined)).toThrow(
"Portable host authority mutation requires the current HOME fence",
);
});

it("classifies absent-to-Portable transitions after acquiring the host fence", async () => {
vi.restoreAllMocks();
vi.resetModules();
const [freshReceiptAuthority, freshPortableLock, freshAcquisition, freshHostFence, freshLock] =
await Promise.all([
import("../../../onboard/experimental/hermes-portable-receipt"),
import("../../../onboard/experimental/portable-lifecycle-lock"),
import("../../../state/mcp-lifecycle-lock-acquisition"),
import("../../../state/portable-uninstall-retirement"),
import("./lock"),
]);
let candidate = false;
vi.spyOn(freshReceiptAuthority, "hasHermesPortableReceiptCandidate").mockImplementation(
() => candidate,
);
let releaseTransition!: () => void;
const transitionBlocked = new Promise<void>((resolve) => {
releaseTransition = resolve;
});
let transitionStarted!: () => void;
const transitionEntered = new Promise<void>((resolve) => {
transitionStarted = resolve;
});
const transition = freshHostFence.withCurrentPortableHostFence(async () => {
transitionStarted();
await transitionBlocked;
candidate = true;
});
let startOperation!: () => void;
const operationStart = new Promise<void>((resolve) => {
startOperation = resolve;
});
const observed: string[] = [];
const operation = operationStart.then(() =>
freshLock.withSandboxLifecycleLock("alpha", () => {
observed.push(
freshAcquisition.isMcpLifecycleLockHeld(
"alpha",
freshPortableLock.portableLifecycleLockOptions(process.env).stateDir,
)
? "portable"
: "gateway",
);
}),
);
await transitionEntered;
startOperation();
await new Promise<void>((resolve) => setImmediate(resolve));
expect(observed).toEqual([]);

releaseTransition();
await Promise.all([transition, operation]);
expect(observed).toEqual(["portable"]);
});

it("classifies Portable-to-absent transitions after acquiring the host fence", async () => {
vi.restoreAllMocks();
vi.resetModules();
const [freshReceiptAuthority, freshPortableLock, freshAcquisition, freshHostFence, freshLock] =
await Promise.all([
import("../../../onboard/experimental/hermes-portable-receipt"),
import("../../../onboard/experimental/portable-lifecycle-lock"),
import("../../../state/mcp-lifecycle-lock-acquisition"),
import("../../../state/portable-uninstall-retirement"),
import("./lock"),
]);
let candidate = true;
vi.spyOn(freshReceiptAuthority, "hasHermesPortableReceiptCandidate").mockImplementation(
() => candidate,
);
let releaseTransition!: () => void;
const transitionBlocked = new Promise<void>((resolve) => {
releaseTransition = resolve;
});
let transitionStarted!: () => void;
const transitionEntered = new Promise<void>((resolve) => {
transitionStarted = resolve;
});
const transition = freshHostFence.withCurrentPortableHostFence(async () => {
transitionStarted();
await transitionBlocked;
candidate = false;
});
const portableStateDir = freshPortableLock.portableLifecycleLockOptions(process.env).stateDir;
const gatewayStateDir = path.join(homeDir, ".nemoclaw", "gateways", "18080", "state");
let startOperation!: () => void;
const operationStart = new Promise<void>((resolve) => {
startOperation = resolve;
});
const observed: string[] = [];
const operation = operationStart.then(() =>
freshLock.withSandboxLifecycleLock("alpha", () => {
observed.push(
freshAcquisition.isMcpLifecycleLockHeld("alpha", gatewayStateDir) &&
!freshAcquisition.isMcpLifecycleLockHeld("alpha", portableStateDir)
? "gateway"
: "portable",
);
}),
);
await transitionEntered;
startOperation();
await new Promise<void>((resolve) => setImmediate(resolve));
expect(observed).toEqual([]);

releaseTransition();
await Promise.all([transition, operation]);
expect(observed).toEqual(["gateway"]);
});

it("serializes competing Portable lifecycle operations on the same authority", async () => {
const events: string[] = [];
let releaseFirst!: () => void;
const firstWaiting = new Promise<void>((resolve) => {
releaseFirst = resolve;
});
let firstEntered!: () => void;
const firstStarted = new Promise<void>((resolve) => {
firstEntered = resolve;
});

const first = withSandboxLifecycleLock("alpha", async () => {
events.push("first-entered");
firstEntered();
await firstWaiting;
events.push("first-released");
});
await firstStarted;
const second = withSandboxLifecycleLock("alpha", () => {
events.push("second-entered");
});
await new Promise<void>((resolve) => setImmediate(resolve));

expect(events).toEqual(["first-entered"]);
releaseFirst();
await Promise.all([first, second]);
expect(events).toEqual(["first-entered", "first-released", "second-entered"]);
});
});
Loading
Loading