test(e2e): stabilize release lifecycle checks

test/e2e/live/hermes-skill-lifecycle.ts

@@ -92,7 +92,10 @@ export async function assertHermesSkillLifecycle({
     ["test", "-f", `/sandbox/.hermes/skills/${HERMES_SKILL_ID}/SKILL.md`],
     "phase-4-hermes-skill-disk-check",
   );
-  const skillList = await exec(["hermes", "skills", "list"], "phase-4-hermes-skills-list");
+  const skillList = await exec(
+    ["env", "COLUMNS=240", "hermes", "skills", "list"],
+    "phase-4-hermes-skills-list",
+  );
   expect(stripAnsi(resultText(skillList))).toContain(HERMES_SKILL_ID);
 
   const sessionsBeforeSkill = await exec(

test/e2e/live/mcp-bridge-hermes-http.ts

@@ -10,13 +10,23 @@ export const HERMES_MCP_FAILURE_CAPTURE_BYTES = 4_096;
 export const HERMES_MCP_FAILURE_PREVIEW_CHARS = 1_024;
 export const HERMES_MCP_RESPONSE_FILE_BYTES = 65_536;
 
-interface HermesMcpCommandResult {
+export interface HermesMcpCommandResult {
   exitCode: number | null;
   signal?: NodeJS.Signals | null;
   stdout: string;
   stderr: string;
 }
 
+export function isHermesGatewayDrainingResponse(result: HermesMcpCommandResult): boolean {
+  if (result.exitCode !== 0 || parseHttpStatus(result.stderr) !== 503) return false;
+  try {
+    const body = JSON.parse(result.stdout) as { error?: { code?: unknown } };
+    return body.error?.code === "gateway_draining";
+  } catch {
+    return false;
+  }
+}
+
 const FAILURE_BODY_EMITTER = [
   "import os, pathlib, sys",
   "raw = pathlib.Path(sys.argv[1]).read_bytes()",

test/e2e/live/mcp-bridge-reliability.ts

@@ -1,7 +1,14 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+import {
+  type HermesMcpCommandResult,
+  isHermesGatewayDrainingResponse,
+} from "./mcp-bridge-hermes-http.ts";
+
 const ANSI_ESCAPE = /\u001b\[[0-9;]*m/gu;
+const HERMES_GATEWAY_DRAINING_RETRIES = 3;
+const HERMES_GATEWAY_DRAINING_RETRY_DELAY_MS = 5_000;
 const HERMES_RESTART_TRANSPORT_FAILURE_SUFFIX = [
   `Error: x code: 'Unknown error', message: "h2 protocol error: error reading a body`,
   `| from connection", source: hyper::Error(Body, Error { kind: Io(Custom`,
@@ -70,3 +77,22 @@ export async function retryAfterHermesRestartTransportFailure<T>(options: {
   }
   return options.retry();
 }
+
+export async function retryHermesGatewayDraining<T extends HermesMcpCommandResult>(options: {
+  initialResult: T;
+  retry: (attempt: number) => Promise<T>;
+  wait?: (milliseconds: number) => Promise<void>;
+}): Promise<T> {
+  const wait =
+    options.wait ?? ((milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)));
+  let result = options.initialResult;
+  for (
+    let attempt = 1;
+    attempt <= HERMES_GATEWAY_DRAINING_RETRIES && isHermesGatewayDrainingResponse(result);
+    attempt += 1
+  ) {
+    await wait(HERMES_GATEWAY_DRAINING_RETRY_DELAY_MS);
+    result = await options.retry(attempt);
+  }
+  return result;
+}

test/e2e/live/mcp-bridge.test.ts

@@ -47,7 +47,10 @@ import {
   requireMcpBridgeTlsCaCert,
 } from "./mcp-bridge-onboard-env.ts";
 import { MCP_BRIDGE_PHASES } from "./mcp-bridge-phases.ts";
-import { retryAfterHermesRestartTransportFailure } from "./mcp-bridge-reliability.ts";
+import {
+  retryAfterHermesRestartTransportFailure,
+  retryHermesGatewayDraining,
+} from "./mcp-bridge-reliability.ts";
 import {
   buildMcpDnsRebindingProbeScript,
   captureManagedMcpPolicy,
@@ -567,17 +570,28 @@ async function assertRealAdapterToolCall(
             buildHermesMcpChatProbeScript(hermesPayload, options.resultToken),
           ].join("\n")
         : `nemoclaw-start dcode -n ${JSON.stringify(prompt)}`;
-  const result = await sandbox.execShell(
-    options.sandboxName,
-    trustedSandboxShellScript(["set -eu", command].join("\n")),
-    {
-      artifactName: options.artifactName,
-      env: buildAvailabilityProbeEnv(),
-      captureLimitBytes: options.agent === "hermes" ? HERMES_MCP_FAILURE_CAPTURE_BYTES : undefined,
-      redactionValues: options.agent === "hermes" ? hermesRedactionValues : [],
-      timeoutMs: 5 * 60_000,
-    },
-  );
+  const runToolCall = (artifactName: string) =>
+    sandbox.execShell(
+      options.sandboxName,
+      trustedSandboxShellScript(["set -eu", command].join("\n")),
+      {
+        artifactName,
+        env: buildAvailabilityProbeEnv(),
+        captureLimitBytes:
+          options.agent === "hermes" ? HERMES_MCP_FAILURE_CAPTURE_BYTES : undefined,
+        redactionValues: options.agent === "hermes" ? hermesRedactionValues : [],
+        timeoutMs: 5 * 60_000,
+      },
+    );
+  const initialResult = await runToolCall(options.artifactName);
+  const result =
+    options.agent === "hermes"
+      ? await retryHermesGatewayDraining({
+          initialResult,
+          retry: (attempt) =>
+            runToolCall(`${options.artifactName}-gateway-draining-retry-${attempt}`),
+        })
+      : initialResult;
   const assertResponse =
     options.agent === "hermes"
       ? () => assertHermesMcpHttpResponse(result, hermesRedactionValues)

test/e2e/live/shields-config.test.ts

@@ -982,7 +982,7 @@ test("shields-config: live Shields lifecycle restores stopped OpenClaw under bot
   progress.phase("recover shields after a dead restore timer");
   const timerDown = await runNemoclaw(
     host,
-    [SANDBOX_NAME, "shields", "down", "--timeout", "10s", "--reason", "Auto-restore timer E2E"],
+    [SANDBOX_NAME, "shields", "down", "--timeout", "60s", "--reason", "Auto-restore timer E2E"],
     { artifactName: "phase-9-shields-down-timer" },
   );
   expect(timerDown.exitCode, resultText(timerDown)).toBe(0);

test/e2e/support/mcp-bridge-hermes-http.test.ts

@@ -14,6 +14,7 @@ import {
   HERMES_MCP_FAILURE_PREVIEW_CHARS,
   HERMES_MCP_HTTP_STATUS_MARKER,
   HERMES_MCP_RESULT_TOKEN_MARKER,
+  isHermesGatewayDrainingResponse,
 } from "../live/mcp-bridge-hermes-http.ts";
 
 const TIMEOUT_MS = 5_000;
@@ -116,4 +117,22 @@ describe("Hermes MCP HTTP failure diagnostics", () => {
       ),
     ).not.toThrow();
   });
+
+  it("classifies only the exact Hermes gateway draining response", () => {
+    const draining = JSON.stringify({ error: { code: "gateway_draining" } });
+    expect(isHermesGatewayDrainingResponse(httpResult(503, draining))).toBe(true);
+    expect(isHermesGatewayDrainingResponse(httpResult(500, draining))).toBe(false);
+    expect(isHermesGatewayDrainingResponse(httpResult(503, "not-json"))).toBe(false);
+    expect(
+      isHermesGatewayDrainingResponse(
+        httpResult(503, JSON.stringify({ error: { code: "other" } })),
+      ),
+    ).toBe(false);
+    expect(
+      isHermesGatewayDrainingResponse({
+        ...httpResult(503, draining),
+        exitCode: 1,
+      }),
+    ).toBe(false);
+  });
 });

test/e2e/support/mcp-bridge-reliability.test.ts

@@ -6,8 +6,20 @@ import { describe, expect, it, vi } from "vitest";
 import {
   isHermesRestartTransportFailure,
   retryAfterHermesRestartTransportFailure,
+  retryHermesGatewayDraining,
 } from "../live/mcp-bridge-reliability.ts";
 
+const HTTP_STATUS_MARKER = "NEMOCLAW_HERMES_MCP_HTTP_STATUS=";
+
+function gatewayResult(status: number, code: string) {
+  return {
+    exitCode: 0,
+    signal: null,
+    stdout: JSON.stringify({ error: { code } }),
+    stderr: `${HTTP_STATUS_MARKER}${status}\n`,
+  };
+}
+
 const HERMES_BROKEN_PIPE = `  Effective egress that would be opened:
     policy 'mcp-bridge-concurrent':
       - fixture.trycloudflare.com:443 (protocol: rest, enforcement: enforce)
@@ -115,4 +127,49 @@ describe("MCP bridge transient classification", () => {
     ).rejects.toThrow("requires a verified committed bridge");
     expect(retry).not.toHaveBeenCalled();
   });
+
+  it("retries the exact gateway draining response with a bounded delay", async () => {
+    const passing = gatewayResult(200, "none");
+    const retry = vi
+      .fn<(attempt: number) => Promise<typeof passing>>()
+      .mockResolvedValueOnce(gatewayResult(503, "gateway_draining"))
+      .mockResolvedValueOnce(passing);
+    const wait = vi.fn(async () => undefined);
+
+    await expect(
+      retryHermesGatewayDraining({
+        initialResult: gatewayResult(503, "gateway_draining"),
+        retry,
+        wait,
+      }),
+    ).resolves.toBe(passing);
+    expect(retry).toHaveBeenCalledTimes(2);
+    expect(wait).toHaveBeenCalledTimes(2);
+    expect(wait).toHaveBeenNthCalledWith(1, 5_000);
+    expect(wait).toHaveBeenNthCalledWith(2, 5_000);
+  });
+
+  it("stops after three gateway draining retries", async () => {
+    const draining = gatewayResult(503, "gateway_draining");
+    const retry = vi.fn(async () => draining);
+    const wait = vi.fn(async () => undefined);
+
+    await expect(
+      retryHermesGatewayDraining({ initialResult: draining, retry, wait }),
+    ).resolves.toBe(draining);
+    expect(retry).toHaveBeenCalledTimes(3);
+    expect(wait).toHaveBeenCalledTimes(3);
+  });
+
+  it("does not retry a different Hermes HTTP failure", async () => {
+    const failed = gatewayResult(503, "other");
+    const retry = vi.fn(async () => gatewayResult(200, "none"));
+    const wait = vi.fn(async () => undefined);
+
+    await expect(retryHermesGatewayDraining({ initialResult: failed, retry, wait })).resolves.toBe(
+      failed,
+    );
+    expect(retry).not.toHaveBeenCalled();
+    expect(wait).not.toHaveBeenCalled();
+  });
 });