Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/commands/sandbox/stop.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,6 @@ export default class SandboxStopCommand extends NemoClawCommand {

public async run(): Promise<void> {
const { args } = await this.parse(SandboxStopCommand);
this.applyExitResult(stopSandbox(args.sandboxName));
this.applyExitResult(await stopSandbox(args.sandboxName));
}
}
54 changes: 28 additions & 26 deletions src/lib/actions/sandbox/connect.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1962,17 +1962,19 @@ async function runConnectEntryPreflight(
withinLifecycleFence?: (route: {
readonly hermesPortable: boolean;
readonly hermesPortableCommandAuthority?: HermesPortableReadinessCommandAuthority;
readonly requalify: () => void;
readonly requalify: () => Promise<void>;
}) => Promise<void>;
},
): Promise<void> {
const measure = <T>(stage: "authority" | "lifecycle" | "gateway", operation: () => T): T =>
probeTiming ? probeTiming.measure(stage, operation) : operation();
const measureAsync = <T>(stage: "gateway", operation: () => Promise<T>): Promise<T> =>
probeTiming ? probeTiming.measureAsync(stage, operation) : operation();
const measureAsync = <T>(
stage: "gateway" | "lifecycle",
operation: () => Promise<T>,
): Promise<T> => (probeTiming ? probeTiming.measureAsync(stage, operation) : operation());
await withPortableConnectSandboxLifecycleLock(sandboxName, async () => {
let hermesPortable = false;
let requalify = () => undefined;
let requalify = async (): Promise<void> => undefined;
try {
measure("authority", assertNoOpenShellGatewayEndpointOverride);
const authority = measure("authority", () =>
Expand Down Expand Up @@ -2014,9 +2016,9 @@ async function runConnectEntryPreflight(
retainedHermesLifecycleRecovery.assertCurrent();
return { kind: retainedHermesLifecycleRecovery.kind } as const;
})
: measure("lifecycle", () =>
: await measureAsync("lifecycle", async () =>
hermesPortableCommandAuthority
? recoverPortableDemoSandboxLifecycleForConnect(
? await recoverPortableDemoSandboxLifecycleForConnect(
sandboxName,
registered,
gatewayName,
Expand All @@ -2032,7 +2034,7 @@ async function runConnectEntryPreflight(
: undefined,
)
: hermesPortable && probeTiming
? recoverPortableDemoSandboxLifecycleForConnect(
? await recoverPortableDemoSandboxLifecycleForConnect(
sandboxName,
registered,
gatewayName,
Expand All @@ -2041,7 +2043,7 @@ async function runConnectEntryPreflight(
{ onComplete: writeHermesPortableCurrentnessTiming },
{ onComplete: writeHermesPortableInspectionTiming },
)
: recoverPortableDemoSandboxLifecycleForConnect(
: await recoverPortableDemoSandboxLifecycleForConnect(
sandboxName,
registered,
gatewayName,
Expand Down Expand Up @@ -2070,7 +2072,7 @@ async function runConnectEntryPreflight(
),
);
}
requalify = () => {
requalify = async () => {
if (!hermesAuthority) {
if (
measure(
Expand Down Expand Up @@ -2098,9 +2100,9 @@ async function runConnectEntryPreflight(
if (!probeOnly) {
const activeAuthority = hermesAuthority;
const currentGateway = resolveSandboxGatewayName(activeAuthority.entry);
const recovery = measure("lifecycle", () =>
const recovery = await measureAsync("lifecycle", async () =>
hermesPortableCommandAuthority
? recoverPortableDemoSandboxLifecycleForConnect(
? await recoverPortableDemoSandboxLifecycleForConnect(
sandboxName,
activeAuthority.entry,
currentGateway,
Expand All @@ -2112,7 +2114,7 @@ async function runConnectEntryPreflight(
probeTiming ? { onComplete: writeHermesPortableInspectionTiming } : undefined,
)
: hermesPortable && probeTiming
? recoverPortableDemoSandboxLifecycleForConnect(
? await recoverPortableDemoSandboxLifecycleForConnect(
sandboxName,
activeAuthority.entry,
currentGateway,
Expand All @@ -2121,7 +2123,7 @@ async function runConnectEntryPreflight(
{ onComplete: writeHermesPortableCurrentnessTiming },
{ onComplete: writeHermesPortableInspectionTiming },
)
: recoverPortableDemoSandboxLifecycleForConnect(
: await recoverPortableDemoSandboxLifecycleForConnect(
sandboxName,
activeAuthority.entry,
currentGateway,
Expand Down Expand Up @@ -2177,7 +2179,7 @@ async function runConnectEntryPreflight(
...(hermesPortableCommandAuthority ? { hermesPortableCommandAuthority } : {}),
requalify,
});
requalify();
await requalify();
});
}

Expand Down Expand Up @@ -2296,13 +2298,13 @@ export async function prepareInteractiveSession(sandboxName: string): Promise<{
: undefined,
successLogs: [" Sandbox is ready. Connecting..."],
});
requalify();
await requalify();
// ── Inference route swap (#1248, #3390) ───────────────────────
const agent = agentRuntime.getSessionAgent(sandboxName);
const sb = hermesPortable
? await verifyHermesPortableInferenceRouteOrExit(sandboxName, agent)
: await ensureSandboxInferenceRouteOrExit(sandboxName, agent);
requalify();
await requalify();
if (hermesPortable) {
const authority = requireHermesPortableActiveLifecycleAuthority(
sandboxName,
Expand Down Expand Up @@ -2418,7 +2420,7 @@ async function prepareConnectSandboxWithinLifecycleFence(
// proof-backed requalification and lifecycle recovery exactly once;
// executable, socket, receipt, container, or policy drift still
// fails inside those authorities before another readiness read.
const requalified = probeTiming!.measure("authority", () =>
const requalified = await probeTiming!.measureAsync("authority", () =>
requalifyPortableAgentSandboxAuthority(
sandboxName,
portableAgentLifecycleAuthorityDeps(),
Expand All @@ -2437,7 +2439,7 @@ async function prepareConnectSandboxWithinLifecycleFence(
);
const registered = active.entry;
requalified.assertCurrent();
const recovery = probeTiming!.measure("lifecycle", () =>
const recovery = await probeTiming!.measureAsync("lifecycle", () =>
recoverPortableDemoSandboxLifecycleForConnect(
sandboxName,
registered,
Expand Down Expand Up @@ -2473,7 +2475,7 @@ async function prepareConnectSandboxWithinLifecycleFence(
requalified.kind === "already-current" && qualified.kind === "current";
}
if (qualified.kind === "requalification-required") {
const requalified = probeTiming!.measure("authority", () =>
const requalified = await probeTiming!.measureAsync("authority", () =>
requalifyPortableAgentSandboxAuthority(
sandboxName,
portableAgentLifecycleAuthorityDeps(),
Expand Down Expand Up @@ -2607,7 +2609,7 @@ async function prepareConnectSandboxWithinLifecycleFence(
return null;
}
if (!portableAuthorityReady) {
probeTiming!.measure("authority", () =>
await probeTiming!.measureAsync("authority", () =>
requalifyPortableAgentSandboxAuthority(
sandboxName,
portableAgentLifecycleAuthorityDeps(),
Expand Down Expand Up @@ -2783,7 +2785,7 @@ async function prepareConnectSandboxWithinLifecycleFence(
ensureLiveSandboxOrExit(sandboxName, { gatewayRecovery: "observe" }),
);
}
requalify();
await requalify();
await runSandboxConnectProbe(sandboxName, {
hermesPortable,
...(hermesPortableCommandAuthority ? { hermesPortableCommandAuthority } : {}),
Expand All @@ -2800,7 +2802,7 @@ async function prepareConnectSandboxWithinLifecycleFence(
}),
);
}
requalify();
await requalify();
},
});
const retainedCommand = hermesReadinessAuthority?.command;
Expand Down Expand Up @@ -2955,18 +2957,18 @@ async function prepareConnectSandboxWithinLifecycleFence(
throw new Error("Hermes portable lifecycle authority appeared during interactive connect");
}
};
const requalifyHermesPortableForConnect = (): {
const requalifyHermesPortableForConnect = async (): Promise<{
readonly env: NodeJS.ProcessEnv;
readonly executablePath: string;
readonly gatewayName: string;
} => {
}> => {
requalifyPortableDisposition();
const qualified = hermesAuthority;
if (!qualified) {
throw new Error("Hermes portable registry authority changed before interactive connect");
}
const gatewayName = resolveSandboxGatewayName(qualified.entry);
const recovery = recoverPortableDemoSandboxLifecycleForConnect(
const recovery = await recoverPortableDemoSandboxLifecycleForConnect(
sandboxName,
qualified.entry,
gatewayName,
Expand All @@ -2984,7 +2986,7 @@ async function prepareConnectSandboxWithinLifecycleFence(
requalifyPortableDisposition();
if (!hermesPortable) prepareHermesLightTerminalSkin(sandboxName, agent, process.env);
requalifyPortableDisposition();
const portableAuthority = hermesPortable ? requalifyHermesPortableForConnect() : null;
const portableAuthority = hermesPortable ? await requalifyHermesPortableForConnect() : null;
return {
executor: createCliOpenShellSandboxSessionExecutor({
hostCwd: ROOT,
Expand Down
53 changes: 46 additions & 7 deletions src/lib/actions/sandbox/gateway-state-observe-mode.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import * as portableAgentLifecycle from "../../onboard/experimental/portable-age
import * as registry from "../../state/registry";
import * as gatewaySelect from "./gateway-select";
import {
assertHermesPortableLifecycleForConnect,
captureHermesPortableInferenceRecoveryGateway,
getReconciledSandboxGatewayState,
recoverPortableDemoSandboxLifecycleForConnect,
Expand Down Expand Up @@ -145,11 +146,49 @@ describe("Hermes Portable inference recovery gateway", () => {
});

describe("Hermes Portable lifecycle recovery command authority", () => {
it("keeps connect authority registry reads live across deferred policy proof (#11479)", async () => {
const input = {
name: "alpha",
agent: "hermes",
gatewayName: "nemoclaw",
openshellDriver: "docker",
lifecycleGeneration: "generation-1",
} as registry.SandboxEntry;
let row = input;
vi.spyOn(registry, "getSandbox").mockImplementation(() => row);
let resolvePolicy!: () => void;
let notifyEntered!: () => void;
const policy = new Promise<void>((resolve) => {
resolvePolicy = resolve;
});
const entered = new Promise<void>((resolve) => {
notifyEntered = resolve;
});
vi.spyOn(
portableAgentLifecycle,
"assertHermesPortableAgentLifecycleAuthority",
).mockImplementation(async (name, _context, deps) => {
expect(deps?.readRegistry?.(name)).toBe(input);
notifyEntered();
await policy;
expect(deps?.readRegistry?.(name)).toBe(row);
throw new Error("registry authority disagrees with the active receipt");
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
});
const proof = assertHermesPortableLifecycleForConnect("alpha", input, "nemoclaw");
const rejected = expect(proof).rejects.toThrow(
"registry authority disagrees with the active receipt",
);
await entered;
row = { ...row, lifecycleGeneration: "generation-2" };
resolvePolicy();
await rejected;
});

afterEach(() => {
vi.restoreAllMocks();
});

it("uses transaction currentness for intermediate captures and full currentness at recovery boundaries", () => {
it("uses transaction currentness for intermediate captures and full currentness at recovery boundaries", async () => {
const assertCurrent = vi.fn();
const assertTransactionCurrent = vi.fn();
const capture = vi.spyOn(openshellRuntime, "captureResolvedOpenshell").mockReturnValue({
Expand All @@ -160,7 +199,7 @@ describe("Hermes Portable lifecycle recovery command authority", () => {
} as never);
const recover = vi
.spyOn(portableAgentLifecycle, "recoverPortableAgentSandboxLifecycle")
.mockImplementation((_sandboxName, _context, deps) => {
.mockImplementation(async (_sandboxName, _context, deps) => {
const recoveryDeps = deps!;
recoveryDeps.assertOpenShellExecutableAuthority?.({} as never, {}, {});
recoveryDeps.captureOpenshell?.(["sandbox", "exec", "--", "true"], 1_000);
Expand All @@ -170,7 +209,7 @@ describe("Hermes Portable lifecycle recovery command authority", () => {
});

expect(
recoverPortableDemoSandboxLifecycleForConnect(
await recoverPortableDemoSandboxLifecycleForConnect(
"alpha",
{
name: "alpha",
Expand All @@ -195,7 +234,7 @@ describe("Hermes Portable lifecycle recovery command authority", () => {
expect(assertTransactionCurrent).toHaveBeenCalledTimes(4);
});

it("rejects transaction drift around an intermediate capture", () => {
it("rejects transaction drift around an intermediate capture", async () => {
const assertCurrent = vi.fn();
const assertTransactionCurrent = vi
.fn()
Expand All @@ -210,13 +249,13 @@ describe("Hermes Portable lifecycle recovery command authority", () => {
stderr: "",
} as never);
vi.spyOn(portableAgentLifecycle, "recoverPortableAgentSandboxLifecycle").mockImplementation(
(_sandboxName, _context, deps) => {
async (_sandboxName, _context, deps) => {
deps!.captureOpenshell?.(["sandbox", "exec", "--", "true"], 1_000);
return { kind: "recovered" };
},
);

expect(() =>
await expect(() =>
recoverPortableDemoSandboxLifecycleForConnect(
"alpha",
{
Expand All @@ -234,7 +273,7 @@ describe("Hermes Portable lifecycle recovery command authority", () => {
executablePath: "/usr/bin/openshell",
},
),
).toThrow("transaction authority changed");
).rejects.toThrow("transaction authority changed");
expect(assertCurrent).toHaveBeenCalledTimes(2);
});
});
17 changes: 9 additions & 8 deletions src/lib/actions/sandbox/gateway-state.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ export { isTerminalSandboxPhase, TERMINAL_SANDBOX_PHASES };
import { selectSandboxOwningGateway } from "./gateway-select";
import {
gatewayNamePattern,
getKnownSandboxTarget,
getKnownSandboxTargetGatewayName,
getPersistedSandboxTargetGatewayName,
getSandboxTargetGatewayName,
Expand Down Expand Up @@ -218,15 +219,15 @@ export function captureSandboxOwnershipPhases(
return { output: result.output, status: result.status };
}
/** Recover a receipt-bound portable sandbox before the live lookup rejects a stopped container. */
export function recoverPortableDemoSandboxLifecycleForConnect(
export async function recoverPortableDemoSandboxLifecycleForConnect(
sandboxName: string,
sandbox: SandboxEntry | null,
gatewayName: string,
commandAuthority?: ReturnType<typeof qualifyHermesPortableOperatingCommandAuthority>,
lifecycleTiming?: HermesPortableLifecycleRecoveryTiming,
currentnessTiming?: HermesPortableCurrentnessTiming,
inspectionTiming?: HermesPortableContainerInspectionRecoveryTiming,
): PortableDemoLifecycleRecoveryResult {
): Promise<PortableDemoLifecycleRecoveryResult> {
const capture = (args: readonly string[], timeoutMs: number) => {
commandAuthority?.assertTransactionCurrent();
try {
Expand Down Expand Up @@ -256,7 +257,7 @@ export function recoverPortableDemoSandboxLifecycleForConnect(
};
commandAuthority?.assertCurrent();
try {
return recoverPortableAgentSandboxLifecycle(
return await recoverPortableAgentSandboxLifecycle(
sandboxName,
{
agent: sandbox?.agent,
Expand All @@ -283,7 +284,7 @@ export function recoverPortableDemoSandboxLifecycleForConnect(
}
: {}),
captureOpenshell: capture,
readRegistry: (name) => (sandbox?.name === name ? sandbox : null),
readRegistry: getKnownSandboxTarget,
...(lifecycleTiming ? { recoveryTiming: lifecycleTiming } : {}),
...(currentnessTiming ? { currentnessTiming } : {}),
...(inspectionTiming ? { inspectionTiming } : {}),
Expand All @@ -295,12 +296,12 @@ export function recoverPortableDemoSandboxLifecycleForConnect(
}

/** Requalify Hermes receipt authority without starting or mutating its sandbox. */
export function assertHermesPortableLifecycleForConnect(
export async function assertHermesPortableLifecycleForConnect(
sandboxName: string,
sandbox: SandboxEntry,
gatewayName: string,
): void {
assertHermesPortableAgentLifecycleAuthority(
): Promise<void> {
await assertHermesPortableAgentLifecycleAuthority(
sandboxName,
{
agent: sandbox.agent,
Expand All @@ -309,7 +310,7 @@ export function assertHermesPortableLifecycleForConnect(
openshellDriver: sandbox.openshellDriver,
provider: sandbox.provider,
},
{ readRegistry: (name: string) => (name === sandboxName ? sandbox : null) },
{ readRegistry: getKnownSandboxTarget },
);
}

Expand Down
Loading
Loading