Skip to content
5 changes: 3 additions & 2 deletions ci/source-architecture-budget.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
"src/lib/adapters/openshell/runtime.ts": 53,
"src/lib/adapters/openshell/timeouts.ts": 33,
"src/lib/agent/defs.ts": 32,
"src/lib/cli/branding.ts": 80,
"src/lib/cli/branding.ts": 81,
"src/lib/cli/nemoclaw-oclif-command.ts": 105,
"src/lib/cli/terminal-style.ts": 42,
"src/lib/core/json-types.ts": 34,
Expand All @@ -27,7 +27,7 @@
"src/lib/runner.ts": 84,
"src/lib/security/redact.ts": 52,
"src/lib/state/mcp-lifecycle-lock.ts": 21,
"src/lib/state/onboard-session.ts": 35,
"src/lib/state/onboard-session.ts": 36,
"src/lib/state/registry.ts": 96,
"src/lib/state/state-root.ts": 21,
"src/lib/subprocess-env.ts": 22,
Expand All @@ -52,6 +52,7 @@
"src/lib/inference/onboard-probes.ts": 21,
"src/lib/inference/vllm.ts": 20,
"src/lib/onboard.ts": 191,
"src/lib/onboard/command.ts": 23,
"src/lib/onboard/machine/handlers/sandbox.ts": 21,
"src/lib/policy/index.ts": 22,
"src/lib/sandbox/config.ts": 19,
Expand Down
43 changes: 43 additions & 0 deletions src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import {
resetDestroyModuleCache,
} from "../../../../test/helpers/destroy-flow-test-harness";
import type { RetainedSandboxRecoveryRecord } from "../../state/onboard-session/retained-sandbox-recovery";
import { OnboardLockContentionError } from "../../state/onboard-session";

function retainedRecoveryRecord(sandboxId = "sb-alpha"): RetainedSandboxRecoveryRecord {
return {
Expand Down Expand Up @@ -107,6 +108,48 @@ describe("destroySandbox retained recovery flow", () => {
},
);

it(
"reports holder PID and wait guidance when retained recovery cleanup hits a live onboard lock (#11052)",
{ timeout: 30_000 },
async () => {
const recovery = retainedRecoveryRecord();
const sandboxContainerId = "a".repeat(64);
const bootstrapContainerId = "b".repeat(64);
const identityRows = [sandboxContainerId, bootstrapContainerId]
.map((id) => `${id}\topenshell\tdefault\tsb-alpha`)
.join("\n");
const harness = createDestroyHarness({
sandboxPresent: false,
dockerOrphanIds: [bootstrapContainerId],
dockerRunResult: { status: 0, stdout: identityRows },
registryEntryOverrides: {
lifecycleGeneration: recovery.lifecycleGeneration!,
lifecycleLiveIdentityFingerprint: recovery.sandboxIdentityFingerprint!,
},
retainedRecoveryRecords: [recovery],
});
harness.resolveRetainedSandboxRecoverySpy.mockImplementation(() => {
throw new OnboardLockContentionError(4242);
});

await expect(harness.destroySandbox("alpha", { yes: true })).rejects.toThrow(
"process.exit(1)",
);

const errorOutput = harness.errorSpy.mock.calls.map((call) => String(call[0])).join("\n");
expect(errorOutput).toContain("could not clear its retained recovery record");
expect(errorOutput).toContain("another nemoclaw onboarding run is already in progress.");
expect(errorOutput).toContain("Lock holder PID: 4242.");
expect(errorOutput).toContain("Wait for the active onboarding run to finish.");
expect(errorOutput).toContain("after the active onboarding run finishes");
expect(errorOutput).not.toContain(
"Cannot update onboarding recovery while another onboarding run owns the lock.",
);
expect(harness.removeSandboxSpy).toHaveBeenCalledWith("alpha");
expect(exitSpy).toHaveBeenCalledWith(1);
},
);

it(
"preserves recovery when a foreign name-labeled container appears after continuity checks (#10547)",
{ timeout: 30_000 },
Expand Down
62 changes: 42 additions & 20 deletions src/lib/actions/sandbox/destroy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import fs from "node:fs";
import path from "node:path";

import { CLI_NAME } from "../../cli/branding";
import { CLI_NAME, formatOnboardLockContentionGuidance } from "../../cli/branding";
import { G, R, YW } from "../../cli/terminal-style";
import { prompt as askPrompt } from "../../credentials/store";
import {
Expand Down Expand Up @@ -607,6 +607,12 @@ export async function destroySandbox(
});
} catch (error) {
if (error instanceof SandboxDestroyExitRequest) process.exit(error.exitCode);
if (onboardSession.isOnboardLockContentionError(error)) {
for (const line of formatOnboardLockContentionGuidance(CLI_NAME, error.holderPid).lines) {
console.error(line);
}
process.exit(1);
}
throw error;
}
}
Expand Down Expand Up @@ -665,9 +671,7 @@ async function destroySandboxUnlocked(
: normalizeRuntimeProviderIdentity(null),
redact: redactDestroyError,
sandbox: registeredSandbox,
...(retainedSandboxIdentityFingerprint
? { retainedSandboxIdentityFingerprint }
: {}),
...(retainedSandboxIdentityFingerprint ? { retainedSandboxIdentityFingerprint } : {}),
});
};
const initialIdentity = portableContainerAuthority ? null : inspectContainerIdentity();
Expand Down Expand Up @@ -877,14 +881,13 @@ async function destroySandboxUnlocked(
runtimeSelection: destroyRuntimeSelection,
} = destructiveResult;

if (
destroyRuntimeSelection &&
cleanupGatewayName !== destroyRuntimeSelection.gatewayName
) {
if (destroyRuntimeSelection && cleanupGatewayName !== destroyRuntimeSelection.gatewayName) {
console.error(
` Sandbox '${sandboxName}' was deleted, but its cleanup target changed from '${destroyRuntimeSelection.gatewayName}' to '${cleanupGatewayName}'.`,
);
console.error(" Local ownership state was preserved. Restore the recorded gateway binding and retry destroy.");
console.error(
" Local ownership state was preserved. Restore the recorded gateway binding and retry destroy.",
);
preparedManagedLlamaCppCleanup?.abort();
requestSandboxDestroyExit(1);
}
Expand Down Expand Up @@ -966,11 +969,15 @@ async function destroySandboxUnlocked(
registeredSandboxCount: registry.listSandboxes().sandboxes.length,
sandboxStillRegistered: !!registry.getSandbox(sandboxName),
});
cleanupSandboxServices(sandboxName, {
stopHostServices: shouldStopHostServices,
}, {
runOpenshell: cleanupRunOpenshell,
});
cleanupSandboxServices(
sandboxName,
{
stopHostServices: shouldStopHostServices,
},
{
runOpenshell: cleanupRunOpenshell,
},
);
});
if (deleteSucceededOrAlreadyGone && commonLlamaCppAuthorityRetired === true) {
preparedManagedLlamaCppCleanup?.abort();
Expand Down Expand Up @@ -1127,9 +1134,21 @@ async function destroySandboxUnlocked(
recoveryResolved = onboardSession.resolveRetainedSandboxRecovery(retainedRecoveryAuthority);
} catch (error) {
console.error(
` Sandbox '${sandboxName}' resources are gone, but NemoClaw could not clear its retained recovery record: ${redactDestroyError(error)}`,
` Sandbox '${sandboxName}' resources are gone, but NemoClaw could not clear its retained recovery record.`,
);
console.error(` Re-run '${CLI_NAME} ${sandboxName} destroy --yes' to finish local cleanup.`);
if (onboardSession.isOnboardLockContentionError(error)) {
for (const line of formatOnboardLockContentionGuidance(CLI_NAME, error.holderPid).lines) {
console.error(line);
}
console.error(
` Re-run '${CLI_NAME} ${sandboxName} destroy --yes' after the active onboarding run finishes.`,
);
} else {
console.error(` ${redactDestroyError(error)}`);
console.error(
` Re-run '${CLI_NAME} ${sandboxName} destroy --yes' to finish local cleanup.`,
);
}
requestSandboxDestroyExit(1);
}
if (!recoveryResolved) {
Expand All @@ -1143,10 +1162,13 @@ async function destroySandboxUnlocked(
}
}
if (
shouldCleanupGatewayAfterConfirmedFinalDestroy({
deleteSucceededOrAlreadyGone,
removedRegistryEntry: removed,
}, cleanupCaptureOpenshell ? { captureOpenshell: cleanupCaptureOpenshell } : {})
shouldCleanupGatewayAfterConfirmedFinalDestroy(
{
deleteSucceededOrAlreadyGone,
removedRegistryEntry: removed,
},
cleanupCaptureOpenshell ? { captureOpenshell: cleanupCaptureOpenshell } : {},
)
) {
const shouldCleanupGateway = await resolveCleanupGatewayDecision(normalized);
if (shouldCleanupGateway) {
Expand Down
32 changes: 31 additions & 1 deletion src/lib/actions/sandbox/rebuild-preflight-confirmation.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -191,8 +191,9 @@ describe("rebuild preflight guards", () => {
expect(registerExitHandler).not.toHaveBeenCalled();
const output = error.mock.calls.flat().join("\n");
expect(output).toContain("another nemoclaw onboarding run is already in progress.");
expect(output).toContain("Wait for the other run to finish, then rerun rebuild.");
expect(output).toContain("Lock holder PID: 4242.");
expect(output).toContain("Wait for the active onboarding run to finish.");
expect(output).not.toContain("a stale nemoclaw onboarding lock is blocking rebuild.");
expect(output).not.toContain("remove the stale lock");
});

Expand All @@ -215,12 +216,41 @@ describe("rebuild preflight guards", () => {
expect(release).not.toHaveBeenCalled();
expect(registerExitHandler).not.toHaveBeenCalled();
const output = error.mock.calls.flat().join("\n");
expect(output).toContain("a stale nemoclaw onboarding lock is blocking rebuild.");
expect(output).toContain(
"Wait briefly, then rerun rebuild so verified stale-lock cleanup can finish.",
);
expect(output).not.toContain("another nemoclaw onboarding run is already in progress.");
expect(output).not.toContain("Lock holder PID:");
expect(output).not.toContain("remove the stale lock");
});

it("does not report a live onboarding holder when lock acquisition fails without a holder PID", () => {
vi.spyOn(onboardSession, "acquireOnboardLock").mockReturnValue({
acquired: false,
lockFile: "/tmp/nemoclaw-onboard.lock",
stale: false,
});
const release = vi
.spyOn(onboardSession, "releaseOnboardLock")
.mockImplementation(() => undefined);
const registerExitHandler = vi.spyOn(process, "once");
const error = vi.spyOn(console, "error").mockImplementation(() => undefined);
const bail = vi.fn() as unknown as (message: string, code?: number) => never;

expect(acquireRebuildOnboardLock("alpha", bail)).toBeNull();

expect(bail).toHaveBeenCalledWith("Could not acquire onboard lock before rebuild");
expect(release).not.toHaveBeenCalled();
expect(registerExitHandler).not.toHaveBeenCalled();
const output = error.mock.calls.flat().join("\n");
expect(output).toContain("could not acquire the nemoclaw onboarding lock.");
expect(output).toContain("Wait briefly, then rerun rebuild.");
expect(output).not.toContain("another nemoclaw onboarding run is already in progress.");
expect(output).not.toContain("a stale nemoclaw onboarding lock is blocking rebuild.");
expect(output).not.toContain("Lock holder PID:");
});

it("rejects a multi-agent sandbox before later rebuild work", () => {
const error = vi.spyOn(console, "error").mockImplementation(() => undefined);
const bail = (message: string): never => {
Expand Down
62 changes: 48 additions & 14 deletions src/lib/actions/sandbox/rebuild-preflight-guards.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import {
printOpenShellStateRpcIssue,
} from "../../adapters/openshell/gateway-drift";
import type { OpenShellRuntimeSelection } from "../../adapters/openshell/runtime-selection";
import { CLI_NAME } from "../../cli/branding";
import { CLI_NAME, formatOnboardLockContentionGuidance } from "../../cli/branding";
import {
checkGatewayRouteCompatibility,
formatGatewayRouteConflict,
Expand Down Expand Up @@ -323,14 +323,41 @@ export function getRebuildSandboxEntryOrBail(
return sb;
}

function printRebuildLiveLockGuidance(holderPid: number, bail: RebuildBail): void {
const guidance = formatOnboardLockContentionGuidance(CLI_NAME, holderPid);
printRebuildPreflightFailure(
guidance.summary,
guidance.lines
.slice(1)
.map((line) => line.trim())
.join(" "),
"Could not acquire onboard lock before rebuild",
bail,
);
}

/** Block rebuild before any live-state probe or cleanup can bypass retained recovery. */
function reportRebuildOnboardLockContention(error: unknown, bail: RebuildBail): boolean {
if (!onboardSession.isOnboardLockContentionError(error)) return false;
printRebuildLiveLockGuidance(error.holderPid, bail);
return true;
}

export function blockRebuildOnRetainedSandboxRecovery(
sandboxName: string,
bail: RebuildBail,
): boolean {
const retainedRecovery = onboardSession
.listRetainedSandboxRecoveryRecords()
.find((record) => record.sandboxName === sandboxName);
let retainedRecovery:
| ReturnType<typeof onboardSession.listRetainedSandboxRecoveryRecords>[number]
| undefined;
try {
retainedRecovery = onboardSession
.listRetainedSandboxRecoveryRecords()
.find((record) => record.sandboxName === sandboxName);
} catch (error) {
if (reportRebuildOnboardLockContention(error, bail)) return true;
throw error;
}
if (!retainedRecovery) return false;

console.error(
Expand Down Expand Up @@ -366,16 +393,23 @@ export function acquireRebuildOnboardLock(
`${CLI_NAME} ${sandboxName} rebuild --authoritative-resume`,
);
if (!lock.acquired) {
const pidDetail = lock.holderPid ? ` Lock holder PID: ${lock.holderPid}.` : "";
const remediation = lock.stale
? "Wait briefly, then rerun rebuild so verified stale-lock cleanup can finish."
: `Wait for the other run to finish, then rerun rebuild.${pidDetail}`;
printRebuildPreflightFailure(
`another ${CLI_NAME} onboarding run is already in progress.`,
remediation,
"Could not acquire onboard lock before rebuild",
bail,
);
if (!lock.stale && lock.holderPid !== undefined) {
printRebuildLiveLockGuidance(lock.holderPid, bail);
} else if (lock.stale) {
printRebuildPreflightFailure(
`a stale ${CLI_NAME} onboarding lock is blocking rebuild.`,
"Wait briefly, then rerun rebuild so verified stale-lock cleanup can finish.",
"Could not acquire onboard lock before rebuild",
bail,
);
} else {
printRebuildPreflightFailure(
`could not acquire the ${CLI_NAME} onboarding lock.`,
"Wait briefly, then rerun rebuild.",
"Could not acquire onboard lock before rebuild",
bail,
);
}
return null;
}
let released = false;
Expand Down
16 changes: 16 additions & 0 deletions src/lib/cli/branding.ts
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,22 @@ const branding = getAgentBranding();
/** CLI binary name for usage strings — e.g. "nemoclaw", "nemohermes", or "nemo-deepagents". */
export const CLI_NAME: string = branding.cli;

/** Shared live onboarding-lock contention copy for CLI command boundaries. */
export function formatOnboardLockContentionGuidance(
cli: string,
holderPid: number,
): { readonly summary: string; readonly lines: readonly string[] } {
const summary = `another ${cli} onboarding run is already in progress.`;
return {
summary,
lines: [
` ${summary}`,
` Lock holder PID: ${holderPid}.`,
" Wait for the active onboarding run to finish.",
],
};
}

/** Title-case display name for headers — e.g. "NemoClaw", "NemoHermes", or "NemoDeepAgents". */
export const CLI_DISPLAY_NAME: string = branding.display;

Expand Down
13 changes: 13 additions & 0 deletions src/lib/onboard/command.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import fs from "node:fs";
import path from "node:path";

import { formatAgentAliasSuffix, resolveAgentNameAlias } from "../agent/aliases";
import { formatOnboardLockContentionGuidance } from "../cli/branding";
import { withCredentialOverrides } from "../credentials/scoped-overrides";
import { loadServingCatalog } from "../inference/serving/catalog-loader";
import { NEMOCLAW_SERVING_PRESET_ENV } from "../inference/serving/managed-cluster-discovery";
Expand Down Expand Up @@ -34,6 +35,8 @@ import {
type ExperimentalOnboardProfile,
PORTABLE_EXPERIMENTAL_PROFILE,
} from "./docker-driver-platform";
import { isOnboardLockContentionError } from "../state/onboard-session";
import { cliName } from "./branding";
import {
loadPortableInferenceDescriptor,
PORTABLE_INFERENCE_CREDENTIAL_ENV,
Expand Down Expand Up @@ -534,7 +537,17 @@ function reportOnboardCommandError(deps: RunOnboardCommandDeps, message: string)
return 1;
}

/** Convert known onboarding failures into stable CLI exit behavior. */
function handleOnboardCommandError(error: unknown, deps: RunOnboardCommandDeps): number | null {
// Live onboarding-lock contention: surface the shipped guard text (name the
// competing run, print the holder PID, say to wait) instead of the bare
// internal error (#11052).
if (isOnboardLockContentionError(error)) {
return reportOnboardCommandError(
deps,
formatOnboardLockContentionGuidance(cliName(), error.holderPid).lines.join("\n"),
);
}
const cancellationCode = promptCancellationCode(error);
if (cancellationCode === "SIGINT") {
// The prompt has already restored terminal state and re-raised SIGINT.
Expand Down
Loading