fix(hermes): preserve MCP reload and Discord handshake

docs/manage-sandboxes/recover-rebuild-sandboxes.mdx

@@ -222,8 +222,9 @@ $$nemoclaw <sandbox-name> gateway-token --quiet
 ```
 
 Before post-restore repairs, NemoClaw verifies that the recreated sandbox still identifies as Hermes and exits nonzero if its identity does not match the rebuild target.
-After state restore, NemoClaw restores managed MCP configuration through the normal lifecycle, then restarts the Hermes gateway and verifies or recovers its health before performing final MCP reconciliation.
-The gateway starts during recreation and reads its durable state before the restore replaces it, so the restart is what binds the running gateway to the restored state.
+After state restore, NemoClaw restarts the Hermes gateway so it reads the restored durable state, then restores managed MCP configuration through the normal lifecycle.
+MCP restoration performs an acknowledged gateway reload, so NemoClaw finishes by verifying the final running gateway and its managed MCP state without replacing that verified process again.
+The gateway starts during recreation and reads its durable state before the restore replaces it, which is why the first post-restore restart must happen before managed MCP restoration.
 `rebuild` exits nonzero instead of reporting success when it cannot verify final gateway health or managed MCP state.
 Follow the printed recovery guidance, using `$$nemoclaw <sandbox-name> gateway restart` first for gateway health, `$$nemoclaw <sandbox-name> recover` when the restart does not restore verified health, and `$$nemoclaw <sandbox-name> mcp restart` for incomplete managed MCP restoration.
 

src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts

@@ -10,6 +10,8 @@ import {
 import {
   ensureHermesGatewayAfterStateRestore,
   ensureHermesGatewayAfterStateRestoreForCronGate,
+  verifyHermesGatewayAfterStateRestore,
+  verifyHermesGatewayAfterStateRestoreForCronGate,
 } from "./rebuild-hermes-post-restore";
 
 const RESTART_SUCCEEDED = {
@@ -164,6 +166,71 @@ describe("binding the Hermes gateway to restored state", () => {
     ).toEqual({ state: "unverified" });
     expect(observeHermesCronReplacement).toHaveBeenCalledTimes(2);
   });
+
+  it("verifies the final cron-bound gateway without restarting after MCP restoration (#8472)", () => {
+    const original = { pid: 41, start_time: 902, drain_token: "restore-token" };
+    const replacement = { pid: 77, start_time: 903, drain_token: "restore-token" };
+    const restartSandboxGateway = vi.fn(() => RESTART_SUCCEEDED);
+    const observeHermesCronReplacement = vi.fn(() => replacement);
+
+    expect(
+      verifyHermesGatewayAfterStateRestoreForCronGate("alpha", "hermes", "restarted", original, {
+        restartSandboxGateway,
+        observeHermesCronReplacement,
+        checkAndRecoverSandboxProcesses: () => ({
+          checked: true,
+          wasRunning: true,
+          recovered: false,
+        }),
+      }),
+    ).toEqual({ state: "healthy", replacementIdentity: replacement });
+    expect(restartSandboxGateway).not.toHaveBeenCalled();
+    expect(observeHermesCronReplacement).toHaveBeenCalledTimes(2);
+  });
+
+  it("rejects unstable final cron identity without restarting after MCP restoration (#8472)", () => {
+    const restartSandboxGateway = vi.fn(() => RESTART_SUCCEEDED);
+    const observeHermesCronReplacement = vi
+      .fn()
+      .mockReturnValueOnce({ pid: 77, start_time: 903, drain_token: "restore-token" })
+      .mockReturnValueOnce({ pid: 88, start_time: 904, drain_token: "restore-token" });
+
+    expect(
+      verifyHermesGatewayAfterStateRestoreForCronGate(
+        "alpha",
+        "hermes",
+        "restarted",
+        { pid: 41, start_time: 902, drain_token: "restore-token" },
+        {
+          restartSandboxGateway,
+          observeHermesCronReplacement,
+          checkAndRecoverSandboxProcesses: () => ({
+            checked: true,
+            wasRunning: true,
+            recovered: false,
+          }),
+        },
+      ),
+    ).toEqual({ state: "unverified" });
+    expect(restartSandboxGateway).not.toHaveBeenCalled();
+  });
+
+  it("preserves MCP reconciliation refusal during restart-free final verification (#7084)", () => {
+    const restartSandboxGateway = vi.fn(() => RESTART_SUCCEEDED);
+
+    expect(
+      verifyHermesGatewayAfterStateRestore("alpha", "hermes", "restarted", {
+        restartSandboxGateway,
+        checkAndRecoverSandboxProcesses: () => ({
+          checked: true,
+          wasRunning: true,
+          recovered: false,
+          mcpReconciliationRefused: true,
+        }),
+      }),
+    ).toBe("unverified");
+    expect(restartSandboxGateway).not.toHaveBeenCalled();
+  });
 });
 
 describe("Hermes gateway post-restore recheck", () => {
@@ -267,7 +334,7 @@ describe("Hermes rebuild post-restore verification", () => {
     expect(harness.restoreMcpBridgesAfterRebuildSpy).toHaveBeenCalledWith("alpha", [mcpEntry]);
   });
 
-  it("accepts restored MCP configuration only after final gateway recovery (#7084)", async () => {
+  it("restores MCP after gateway restart and before final health verification (#7084)", async () => {
     const mcpEntry = {
       server: "blender",
       providerName: "nemoclaw-mcp-alpha-blender",
@@ -276,9 +343,9 @@ describe("Hermes rebuild post-restore verification", () => {
       agentName: "hermes",
       checkAndRecoverSandboxProcesses: () => ({
         checked: true,
-        wasRunning: false,
-        recovered: true,
-        forwardRecovered: true,
+        wasRunning: true,
+        recovered: false,
+        forwardRecovered: false,
       }),
       mcpPreparation: {
         entries: [mcpEntry],
@@ -293,11 +360,14 @@ describe("Hermes rebuild post-restore verification", () => {
     ).resolves.toBeUndefined();
 
     expect(harness.restoreMcpBridgesAfterRebuildSpy).toHaveBeenCalledWith("alpha", [mcpEntry]);
+    expect(harness.restartSandboxGatewaySpy.mock.invocationCallOrder[0]).toBeLessThan(
+      harness.restoreMcpBridgesAfterRebuildSpy.mock.invocationCallOrder[0],
+    );
     expect(harness.restoreMcpBridgesAfterRebuildSpy.mock.invocationCallOrder[0]).toBeLessThan(
       harness.checkAndRecoverSandboxProcessesSpy.mock.invocationCallOrder[0],
     );
     expect(harness.logSpy).toHaveBeenCalledWith(
-      expect.stringContaining("Hermes gateway recovered after state restore"),
+      expect.stringContaining("Hermes gateway restarted and verified after state restore"),
     );
   });
 

src/lib/actions/sandbox/rebuild-hermes-post-restore.ts

@@ -79,6 +79,11 @@ export type HermesPostRestoreGatewayState =
   | "recovered"
   | "unverified";
 
+export type HermesPostRestoreGatewayRestartState =
+  | "not-applicable"
+  | "restarted"
+  | "restart-failed";
+
 type GatewayRecoveryObservation = {
   checked: boolean;
   wasRunning: boolean | null;
@@ -116,20 +121,24 @@ export interface HermesPostRestoreGatewayVerification {
  * pre-restore result for the life of the process — the WhatsApp bridge reads
  * its paired session that way — so the gateway can be alive and healthy while
  * still serving the state the rebuild replaced. A liveness check cannot see
- * that difference, so restart first and let the check report on the process
- * that restart produced. `relaunchManagedSupervisorSession` already restarts
- * after its own restore for the same reason.
+ * that difference, so restart before runtime restoration and let the final
+ * check report on the process left by every intervening acknowledged reload.
+ * `relaunchManagedSupervisorSession` already restarts after its own restore
+ * for the same reason.
  *
- * A gated rebuild keeps the root-owned cron drain active while this function
- * replaces and verifies the gateway. The caller then completes the held
- * transaction against the replacement process before dispatch can resume.
+ * The split restart/verify exports let rebuild insert MCP restoration between
+ * those two steps. Hermes MCP restoration performs its own acknowledged
+ * gateway reload, so a later unconditional restart would discard the runtime
+ * identity whose MCP load just converged. A gated rebuild keeps the root-owned
+ * cron drain active across restart, MCP restoration, and final verification.
  */
 export function ensureHermesGatewayAfterStateRestore(
   sandboxName: string,
   agentName: string,
   deps: HermesPostRestoreGatewayDeps = {},
 ): HermesPostRestoreGatewayState {
-  return ensureHermesGatewayAfterStateRestoreImpl(sandboxName, agentName, deps).state;
+  const restartState = restartHermesGatewayAfterStateRestore(sandboxName, agentName, deps);
+  return verifyHermesGatewayAfterStateRestore(sandboxName, agentName, restartState, deps);
 }
 
 export function ensureHermesGatewayAfterStateRestoreForCronGate(
@@ -138,7 +147,49 @@ export function ensureHermesGatewayAfterStateRestoreForCronGate(
   originalIdentity: HermesCronRestoreIdentity,
   deps: HermesPostRestoreGatewayDeps = {},
 ): HermesPostRestoreGatewayVerification {
-  return ensureHermesGatewayAfterStateRestoreImpl(sandboxName, agentName, deps, originalIdentity);
+  const restartState = restartHermesGatewayAfterStateRestore(sandboxName, agentName, deps);
+  return verifyHermesGatewayAfterStateRestoreForCronGate(
+    sandboxName,
+    agentName,
+    restartState,
+    originalIdentity,
+    deps,
+  );
+}
+
+export function restartHermesGatewayAfterStateRestore(
+  sandboxName: string,
+  agentName: string,
+  deps: HermesPostRestoreGatewayDeps = {},
+): HermesPostRestoreGatewayRestartState {
+  if (agentName !== "hermes") return "not-applicable";
+  const restart = deps.restartSandboxGateway ?? processRecovery.restartSandboxGateway;
+  return restart(sandboxName, { quiet: true }).ok ? "restarted" : "restart-failed";
+}
+
+export function verifyHermesGatewayAfterStateRestore(
+  sandboxName: string,
+  agentName: string,
+  restartState: HermesPostRestoreGatewayRestartState,
+  deps: HermesPostRestoreGatewayDeps = {},
+): HermesPostRestoreGatewayState {
+  return verifyHermesGatewayAfterStateRestoreImpl(sandboxName, agentName, restartState, deps).state;
+}
+
+export function verifyHermesGatewayAfterStateRestoreForCronGate(
+  sandboxName: string,
+  agentName: string,
+  restartState: HermesPostRestoreGatewayRestartState,
+  originalIdentity: HermesCronRestoreIdentity,
+  deps: HermesPostRestoreGatewayDeps = {},
+): HermesPostRestoreGatewayVerification {
+  return verifyHermesGatewayAfterStateRestoreImpl(
+    sandboxName,
+    agentName,
+    restartState,
+    deps,
+    originalIdentity,
+  );
 }
 
 function sameGatewayIdentity(
@@ -148,15 +199,15 @@ function sameGatewayIdentity(
   return left.pid === right.pid && left.start_time === right.start_time;
 }
 
-function ensureHermesGatewayAfterStateRestoreImpl(
+function verifyHermesGatewayAfterStateRestoreImpl(
   sandboxName: string,
   agentName: string,
+  restartState: HermesPostRestoreGatewayRestartState,
   deps: HermesPostRestoreGatewayDeps,
   originalIdentity?: HermesCronRestoreIdentity,
 ): HermesPostRestoreGatewayVerification {
   if (agentName !== "hermes") return { state: "not-applicable" };
-  const restart = deps.restartSandboxGateway ?? processRecovery.restartSandboxGateway;
-  const restarted = restart(sandboxName, { quiet: true }).ok;
+  const restarted = restartState === "restarted";
   const checkAndRecover =
     deps.checkAndRecoverSandboxProcesses ?? processRecovery.checkAndRecoverSandboxProcesses;
   const observeReplacement = deps.observeHermesCronReplacement ?? observeHermesCronReplacement;