fix(rebuild): emit actionable diagnostic on all preflight failures (#7794)

docs/manage-sandboxes/recover-rebuild-sandboxes.mdx

@@ -150,6 +150,21 @@ Back up those paths outside the sandbox before you approve legacy recovery.
 $$nemoclaw <sandbox-name> rebuild
 ```
 
+### Resolve Rebuild Preflight Stops
+
+Before it backs up or deletes the existing sandbox, `rebuild` validates the recorded sandbox, gateway, policy, MCP, agent, and operation-lock state.
+When one of these checks fails, NemoClaw prints `Rebuild preflight failed`, explains how to recover, and ends with `Aborting rebuild`.
+At this boundary, the existing sandbox is unchanged and no sandbox data has been removed.
+
+Use the recovery guidance that matches the reported check:
+
+- Verify the sandbox name when its registry entry is missing.
+- Follow the printed OpenShell gateway recovery steps when the gateway schema is incompatible.
+- Repair the named pending baseline policy transition, then rerun `rebuild`.
+- Resolve an incomplete MCP destroy transaction before retrying.
+- Back up the sandbox state and recreate it with `$$nemoclaw onboard` when the record contains multiple agents. Transactional multi-agent rebuild is not supported.
+- Wait for another onboarding or rebuild operation to finish before retrying. If verified stale-lock cleanup is still in progress, wait briefly and rerun the command. Do not delete the lock manually.
+
 <AgentOnly variant="openclaw">
 The rebuild command preserves the mounted workspace and registered policies while recreating the container.
 When no host web-search key is staged, rebuild preflight reuses an existing Brave or Tavily credential only when the provider name, type, and credential key match the sandbox's binding on its recorded OpenShell gateway.

src/lib/actions/sandbox/rebuild-baseline-transition-preflight.test.ts

@@ -1,7 +1,7 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-import { beforeEach, describe, expect, it, vi } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 
 const mocks = vi.hoisted(() => ({
   assertMcpDestroyNotPending: vi.fn(),
@@ -43,6 +43,10 @@ vi.mock("./rebuild-preflight-target-phase", async (importOriginal) => ({
 
 import { runRebuildPreflightPhase } from "./rebuild-preflight-phase";
 
+afterEach(() => {
+  vi.restoreAllMocks();
+});
+
 describe("rebuild baseline transition preflight (#7194)", () => {
   beforeEach(() => {
     vi.clearAllMocks();
@@ -76,3 +80,38 @@ describe("rebuild baseline transition preflight (#7194)", () => {
     expect(mocks.prepareTargets).not.toHaveBeenCalled();
   });
 });
+
+describe("rebuild MCP destroy marker preflight (#7794)", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    mocks.getSandbox.mockReturnValue({
+      name: "alpha",
+      agent: "openclaw",
+      mcp: {
+        bridges: {},
+        destroyPreparedAt: "2026-06-27T01:00:00.000Z",
+      },
+    });
+    mocks.assertMcpDestroyNotPending.mockImplementation(() => {
+      throw new Error("Sandbox 'alpha' has an incomplete MCP destroy transaction");
+    });
+  });
+
+  it("prints the safe-abort diagnostic and stops before later rebuild phases", async () => {
+    const error = vi.spyOn(console, "error").mockImplementation(() => undefined);
+
+    await expect(runRebuildPreflightPhase("alpha", ["--yes"])).resolves.toBeNull();
+
+    const output = error.mock.calls.flat().join("\n");
+    expect(output).toContain("Rebuild preflight failed:");
+    expect(output).toContain("a pending MCP destroy transaction blocks rebuild.");
+    expect(output).toContain("Resolve the pending MCP state before retrying rebuild.");
+    expect(output).toContain("Aborting rebuild");
+    expect(output).toContain("sandbox is untouched, no data was lost.");
+    expect(mocks.bail).toHaveBeenCalledWith(
+      "Sandbox 'alpha' has an incomplete MCP destroy transaction",
+    );
+    expect(mocks.confirmRebuildIntent).not.toHaveBeenCalled();
+    expect(mocks.prepareTargets).not.toHaveBeenCalled();
+  });
+});

src/lib/actions/sandbox/rebuild-gateway-drift.test.ts

@@ -120,6 +120,22 @@ describe("rebuild gateway drift preflight", () => {
     expect(recoverNamedGatewayRuntimeSpy).not.toHaveBeenCalled();
   });
 
+  it("prints the safe-abort diagnostic before bailing on gateway schema drift (#7794)", () => {
+    vi.mocked(gatewayDrift.detectOpenShellStateRpcPreflightIssue).mockReturnValue(driftIssue);
+    const nonThrowingBail = vi.fn();
+
+    expect(
+      checkRebuildGatewaySchemaPreflight("alpha", makeSandboxEntry(), nonThrowingBail as never),
+    ).toBe(false);
+
+    const diagnostics = errorSpy.mock.calls.flat().join("\n");
+    expect(diagnostics).toContain("Rebuild preflight failed:");
+    expect(diagnostics).toContain("OpenShell gateway schema is incompatible with this rebuild.");
+    expect(diagnostics).toContain("Follow the gateway recovery guidance above");
+    expect(diagnostics).toContain("Aborting rebuild — sandbox is untouched, no data was lost.");
+    expect(nonThrowingBail).toHaveBeenCalledWith("OpenShell gateway schema mismatch.");
+  });
+
   it.each([
     {
       recordedGateway: "nemoclaw",

src/lib/actions/sandbox/rebuild-preflight-confirmation.test.ts

@@ -4,13 +4,17 @@
 import { afterEach, describe, expect, it, vi } from "vitest";
 import * as openshellResolve from "../../adapters/openshell/resolve";
 import { redact } from "../../security/redact";
+import * as onboardSession from "../../state/onboard-session";
 import * as sandboxSession from "../../state/sandbox-session";
 import {
   confirmSandboxRebuildIfNeeded,
   countActiveSandboxSessionsForRebuild,
   createRebuildCommandContext,
 } from "./rebuild-preflight-confirmation";
-import { isSingleAgentRebuildSupported } from "./rebuild-preflight-guards";
+import {
+  acquireRebuildOnboardLock,
+  isSingleAgentRebuildSupported,
+} from "./rebuild-preflight-guards";
 
 afterEach(() => {
   vi.restoreAllMocks();
@@ -167,6 +171,58 @@ describe("createRebuildCommandContext bail behaviour (#6376)", () => {
 });
 
 describe("rebuild preflight guards", () => {
+  it("stops after a failed onboard-lock acquisition without releasing another run's lock (#7794)", () => {
+    vi.spyOn(onboardSession, "acquireOnboardLock").mockReturnValue({
+      acquired: false,
+      lockFile: "/tmp/nemoclaw-onboard.lock",
+      stale: false,
+      holderPid: 4242,
+      holderCommand: "nemoclaw onboard",
+    });
+    const release = vi
+      .spyOn(onboardSession, "releaseOnboardLock")
+      .mockImplementation(() => undefined);
+    const registerExitHandler = vi.spyOn(process, "once");
+    const error = vi.spyOn(console, "error").mockImplementation(() => undefined);
+    const bail = vi.fn() as unknown as (message: string, code?: number) => never;
+
+    expect(acquireRebuildOnboardLock("alpha", bail)).toBeNull();
+
+    expect(bail).toHaveBeenCalledWith("Could not acquire onboard lock before rebuild");
+    expect(release).not.toHaveBeenCalled();
+    expect(registerExitHandler).not.toHaveBeenCalled();
+    const output = error.mock.calls.flat().join("\n");
+    expect(output).toContain("another nemoclaw onboarding run is already in progress.");
+    expect(output).toContain("Wait for the other run to finish, then rerun rebuild.");
+    expect(output).toContain("Lock holder PID: 4242.");
+    expect(output).not.toContain("remove the stale lock");
+  });
+
+  it("waits for verified cleanup when stale-lock contention exhausts retries (#7794)", () => {
+    vi.spyOn(onboardSession, "acquireOnboardLock").mockReturnValue({
+      acquired: false,
+      lockFile: "/tmp/nemoclaw-onboard.lock",
+      stale: true,
+    });
+    const release = vi
+      .spyOn(onboardSession, "releaseOnboardLock")
+      .mockImplementation(() => undefined);
+    const registerExitHandler = vi.spyOn(process, "once");
+    const error = vi.spyOn(console, "error").mockImplementation(() => undefined);
+    const bail = vi.fn() as unknown as (message: string, code?: number) => never;
+
+    expect(acquireRebuildOnboardLock("alpha", bail)).toBeNull();
+
+    expect(bail).toHaveBeenCalledWith("Could not acquire onboard lock before rebuild");
+    expect(release).not.toHaveBeenCalled();
+    expect(registerExitHandler).not.toHaveBeenCalled();
+    const output = error.mock.calls.flat().join("\n");
+    expect(output).toContain(
+      "Wait briefly, then rerun rebuild so verified stale-lock cleanup can finish.",
+    );
+    expect(output).not.toContain("remove the stale lock");
+  });
+
   it("rejects a multi-agent sandbox before later rebuild work", () => {
     const error = vi.spyOn(console, "error").mockImplementation(() => undefined);
     const bail = (message: string): never => {
@@ -181,7 +237,7 @@ describe("rebuild preflight guards", () => {
     ).toThrow("Multi-agent sandbox rebuild is not yet supported");
 
     const output = error.mock.calls.flat().join("\n");
-    expect(output).toContain("Multi-agent sandbox rebuild is not yet supported");
+    expect(output).toContain("multi-agent sandbox rebuild is not yet supported.");
     expect(output).toContain("Back up state manually");
   });
 

src/lib/actions/sandbox/rebuild-preflight-error.test.ts

@@ -0,0 +1,51 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+import { printRebuildPreflightFailure } from "./rebuild-preflight-error";
+
+describe("printRebuildPreflightFailure (#7794)", () => {
+  beforeEach(() => {
+    vi.spyOn(console, "error").mockImplementation(() => {});
+  });
+
+  it("outputs 'Aborting rebuild' and calls bail with the bail message", () => {
+    const bail = vi.fn() as unknown as (message: string, code?: number) => never;
+
+    printRebuildPreflightFailure("policy apply failed.", "Check network.", "policy error", bail);
+
+    const output = (console.error as ReturnType<typeof vi.fn>).mock.calls
+      .map((c) => c[0])
+      .join("\n");
+    expect(output).toMatch(/Aborting rebuild/i);
+    expect(output).toContain("sandbox is untouched, no data was lost.");
+    expect(bail).toHaveBeenCalledWith("policy error");
+  });
+
+  it("includes the summary and detail in output", () => {
+    const bail = vi.fn() as unknown as (message: string, code?: number) => never;
+
+    printRebuildPreflightFailure("something broke.", "Try again.", "broke", bail);
+
+    const output = (console.error as ReturnType<typeof vi.fn>).mock.calls
+      .map((c) => c[0])
+      .join("\n");
+    expect(output).toContain("something broke.");
+    expect(output).toContain("Try again.");
+  });
+
+  it("preserves an explicit bail exit code", () => {
+    const bail = vi.fn() as unknown as (message: string, code?: number) => never;
+
+    printRebuildPreflightFailure(
+      "policy restore is pending.",
+      "Repair the pending transition.",
+      "pending policy transition",
+      bail,
+      1,
+    );
+
+    expect(bail).toHaveBeenCalledWith("pending policy transition", 1);
+  });
+});

src/lib/actions/sandbox/rebuild-preflight-error.ts

@@ -9,10 +9,15 @@ export function printRebuildPreflightFailure(
   detail: string,
   bailMessage: string,
   bail: RebuildBail,
+  bailCode?: number,
 ): void {
   console.error("");
   console.error(`  ${_RD}Rebuild preflight failed:${R} ${summary}`);
   console.error(`  ${detail}`);
-  console.error("  Sandbox is untouched — no data was lost.");
-  bail(bailMessage);
+  console.error("  Aborting rebuild — sandbox is untouched, no data was lost.");
+  if (bailCode === undefined) {
+    bail(bailMessage);
+  } else {
+    bail(bailMessage, bailCode);
+  }
 }

src/lib/actions/sandbox/rebuild-preflight-guards.ts

@@ -243,7 +243,12 @@ export function checkRebuildGatewaySchemaPreflight(
       action: `rebuilding sandbox '${sandboxName}'`,
       command: `${CLI_NAME} ${sandboxName} rebuild`,
     });
-    bail("OpenShell gateway schema mismatch.");
+    printRebuildPreflightFailure(
+      "OpenShell gateway schema is incompatible with this rebuild.",
+      "Follow the gateway recovery guidance above, then rerun rebuild.",
+      "OpenShell gateway schema mismatch.",
+      bail,
+    );
     return false;
   }
   return true;
@@ -263,8 +268,12 @@ export function getRebuildSandboxEntryOrBail(
 ): RebuildSandboxEntry | null {
   const sb = registry.getSandbox(sandboxName) as RebuildSandboxEntry | null;
   if (!sb) {
-    console.error(`  Sandbox '${sandboxName}' not found in registry.`);
-    bail(`Sandbox '${sandboxName}' not found in registry.`);
+    printRebuildPreflightFailure(
+      `sandbox '${sandboxName}' not found in registry.`,
+      "Verify the sandbox name and rerun rebuild.",
+      `Sandbox '${sandboxName}' not found in registry.`,
+      bail,
+    );
     return null;
   }
   return sb;
@@ -280,12 +289,13 @@ export function blockRebuildOnPendingBaselineTransition(
   if (!transition) return false;
 
   const key = transition.exclusion.key;
-  console.error("");
-  console.error(
-    `  Baseline policy ${transition.operation} for '${key}' needs repair before rebuild.`,
+  printRebuildPreflightFailure(
+    `baseline policy ${transition.operation} for '${key}' needs repair before rebuild.`,
+    `Re-run: ${CLI_NAME} ${sandboxName} policy ${transition.operation} ${key}`,
+    `Pending baseline policy ${transition.operation} for '${key}' blocks rebuild.`,
+    bail,
+    1,
   );
-  console.error(`  Re-run: ${CLI_NAME} ${sandboxName} policy ${transition.operation} ${key}`);
-  bail(`Pending baseline policy ${transition.operation} for '${key}' blocks rebuild.`, 1);
   return true;
 }
 
@@ -294,23 +304,36 @@ export function isSingleAgentRebuildSupported(
   bail: RebuildBail,
 ): boolean {
   if (sb.agents && sb.agents.length > 1) {
-    console.error("  Multi-agent sandbox rebuild is not yet supported.");
-    console.error(`  Back up state manually and recreate with \`${CLI_NAME} onboard\`.`);
-    bail("Multi-agent sandbox rebuild is not yet supported.");
+    printRebuildPreflightFailure(
+      "multi-agent sandbox rebuild is not yet supported.",
+      `Back up state manually and recreate with \`${CLI_NAME} onboard\`.`,
+      "Multi-agent sandbox rebuild is not yet supported.",
+      bail,
+    );
     return false;
   }
   return true;
 }
 
-export function acquireRebuildOnboardLock(sandboxName: string, bail: RebuildBail): () => void {
+export function acquireRebuildOnboardLock(
+  sandboxName: string,
+  bail: RebuildBail,
+): (() => void) | null {
   const lock = onboardSession.acquireOnboardLock(
     `${CLI_NAME} ${sandboxName} rebuild --authoritative-resume`,
   );
   if (!lock.acquired) {
-    console.error(`  Another ${CLI_NAME} onboarding run is already in progress.`);
-    if (lock.holderPid) console.error(`  Lock holder PID: ${lock.holderPid}`);
-    console.error("  Sandbox is untouched — no data was lost.");
-    bail("Could not acquire onboard lock before rebuild");
+    const pidDetail = lock.holderPid ? ` Lock holder PID: ${lock.holderPid}.` : "";
+    const remediation = lock.stale
+      ? "Wait briefly, then rerun rebuild so verified stale-lock cleanup can finish."
+      : `Wait for the other run to finish, then rerun rebuild.${pidDetail}`;
+    printRebuildPreflightFailure(
+      `another ${CLI_NAME} onboarding run is already in progress.`,
+      remediation,
+      "Could not acquire onboard lock before rebuild",
+      bail,
+    );
+    return null;
   }
   let released = false;
   const release = () => {

src/lib/actions/sandbox/rebuild-preflight-phase.ts

@@ -149,7 +149,12 @@ export async function runRebuildPreflightPhase(
   try {
     assertMcpDestroyNotPending(sandboxEntry);
   } catch (error) {
-    bail(error instanceof Error ? error.message : String(error));
+    printRebuildPreflightFailure(
+      "a pending MCP destroy transaction blocks rebuild.",
+      "Resolve the pending MCP state before retrying rebuild.",
+      error instanceof Error ? error.message : String(error),
+      bail,
+    );
     return null;
   }
   const confirmedEntrySnapshot = JSON.stringify(sandboxEntry);
@@ -241,6 +246,7 @@ export async function runRebuildPreflightPhase(
   let retainBaseImagePreflight = false;
   try {
     const releaseOnboardLock = acquireRebuildOnboardLock(sandboxName, bail);
+    if (!releaseOnboardLock) return null;
     let retainOnboardLock = false;
     try {
       assertRebuildEntryUnchanged(sandboxName, JSON.stringify(expectedSandboxEntry), bail);