test(e2e): wait for stable gateway recovery state

test/e2e/fixtures/clients/gateway.ts

@@ -1,8 +1,11 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+import { randomBytes } from "node:crypto";
+
 import { buildAvailabilityProbeEnv } from "../availability-env.ts";
 import type { NemoClawInstance } from "../phases/onboarding.ts";
+import { pollUntil } from "../polling.ts";
 import type { ShellProbeResult, ShellProbeRunOptions } from "../shell-probe.ts";
 import { assertExitZero } from "./command.ts";
 import type { HostCliClient } from "./host.ts";
@@ -68,6 +71,14 @@ export interface ExpectPidStableOptions extends ShellProbeRunOptions {
   pollIntervalSeconds?: number;
 }
 
+export interface WaitForMissingManagedSupervisorOptions {
+  attempts?: number;
+  delayMs?: number;
+  settleMs?: number;
+  sleep?: (milliseconds: number) => Promise<void>;
+  onRetry?: (attempt: number) => void;
+}
+
 export interface GatewayProcessIdentity {
   pid: number;
   startIdentity: string;
@@ -78,6 +89,21 @@ export interface HostGatewayRuntime {
   id: string;
 }
 
+function isMissingManagedSupervisorProof(result: ShellProbeResult): boolean {
+  const stderrLines = result.stderr
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter(Boolean);
+  return (
+    result.exitCode === 1 &&
+    result.timedOut === false &&
+    result.signal === null &&
+    result.stdout.trim() === "" &&
+    stderrLines.length === 1 &&
+    stderrLines[0] === "SUPERVISOR_NOT_RUNNING"
+  );
+}
+
 export class GatewayClient {
   private readonly host: HostCliClient;
   private readonly sandbox: SandboxClient;
@@ -168,6 +194,68 @@ export class GatewayClient {
     return result;
   }
 
+  /**
+   * Wait until the trusted root controller proves that a restarted legacy
+   * container has no managed supervisor. This keeps the live E2E recovery test
+   * out of the brief process-churn window after OpenShell becomes reachable.
+   * The fixture owns this wait because Docker restart and OpenShell readiness
+   * can complete before the container process table settles, while production
+   * recovery must keep treating uncertain controller output as terminal. Remove
+   * the wait when OpenShell readiness guarantees the controller absence proof
+   * remains stable across the settle interval.
+   */
+  async waitForMissingManagedSupervisor(
+    containerId: string,
+    options: WaitForMissingManagedSupervisorOptions = {},
+  ): Promise<void> {
+    const attempts = options.attempts ?? 12;
+    const delayMs = options.delayMs ?? 3_000;
+    const settleMs = options.settleMs ?? delayMs;
+    const sleep =
+      options.sleep ??
+      ((milliseconds: number) => new Promise<void>((resolve) => setTimeout(resolve, milliseconds)));
+
+    await pollUntil({
+      artifactPrefix: "legacy-restart-supervisor-absent",
+      attempts,
+      delayMs,
+      sleep,
+      probe: async (_attempt, artifactName) => {
+        const runProbe = async (name: string) =>
+          await this.host.command(
+            "docker",
+            [
+              "exec",
+              "--env",
+              "LD_PRELOAD=",
+              "--env",
+              "PYTHONPATH=",
+              "--user",
+              "root",
+              containerId,
+              "/usr/local/bin/nemoclaw-gateway-control",
+              "probe",
+              randomBytes(32).toString("hex"),
+            ],
+            {
+              artifactName: name,
+              env: probeEnv(),
+              timeoutMs: 30_000,
+            },
+          );
+        const initial = await runProbe(artifactName);
+        if (!isMissingManagedSupervisorProof(initial) || settleMs <= 0) return initial;
+        await sleep(settleMs);
+        return await runProbe(`${artifactName}-after-settle`);
+      },
+      accept: (result, attempt) => {
+        if (isMissingManagedSupervisorProof(result)) return true;
+        options.onRetry?.(attempt);
+        return false;
+      },
+    });
+  }
+
   // ─── Guard-chain recovery probes (#2478, #2701) ────────────────────
 
   /**

test/e2e/live/gateway-guard-recovery.test.ts

@@ -467,6 +467,9 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701)
   });
   expect(legacyRestart.exitCode, resultText(legacyRestart)).toBe(0);
   await waitForSandboxExecAfterContainerRestart(host, instance.sandboxName, progress);
+  await gateway.waitForMissingManagedSupervisor(legacyContainerId, {
+    onRetry: (attempt) => progress.event(`managed supervisor absence proof retry ${attempt}`),
+  });
 
   progress.phase("recover legacy managed supervisor and inference");
   const legacyCredentialCanary = "nemoclaw-e2e-recovery-secret-6635";

test/e2e/support/e2e-recovery-helpers.test.ts

@@ -21,6 +21,8 @@ interface ScriptedReply {
   stdout?: string;
   stderr?: string;
   exitCode?: number | null;
+  signal?: NodeJS.Signals | null;
+  timedOut?: boolean;
 }
 
 /**
@@ -35,6 +37,8 @@ class ScriptedRunner implements CommandRunner {
   readonly calls: RunnerCall[] = [];
   private replies: ScriptedReply[] = [];
 
+  constructor(private readonly observe?: (call: RunnerCall, reply: ScriptedReply) => void) {}
+
   queue(...replies: ScriptedReply[]): void {
     this.replies.push(...replies);
   }
@@ -43,13 +47,15 @@ class ScriptedRunner implements CommandRunner {
     command: TrustedShellCommand,
     options?: ShellProbeRunOptions,
   ): Promise<ShellProbeResult> {
-    this.calls.push({ command: command.command, args: [...command.args], options });
+    const call = { command: command.command, args: [...command.args], options };
+    this.calls.push(call);
     const reply = this.replies.shift() ?? {};
+    this.observe?.(call, reply);
     return {
       command: [command.command, ...command.args],
       exitCode: reply.exitCode ?? 0,
-      signal: null,
-      timedOut: false,
+      signal: reply.signal ?? null,
+      timedOut: reply.timedOut ?? false,
       stdout: reply.stdout ?? "",
       stderr: reply.stderr ?? "",
       artifacts: {
@@ -89,6 +95,145 @@ function buildGateway(runner: ScriptedRunner): GatewayClient {
 }
 
 describe("GatewayClient recovery helpers (#2701)", () => {
+  describe("waitForMissingManagedSupervisor", () => {
+    it("waits for the exact missing-supervisor proof and a quiet settle", async () => {
+      const events: string[] = [];
+      const runner = new ScriptedRunner((_call, reply) => {
+        events.push(
+          reply.stderr === "SUPERVISOR_NOT_RUNNING\n" ? "probe-not-running" : "probe-unavailable",
+        );
+      });
+      runner.queue(
+        {
+          exitCode: 1,
+          stderr: "SUPERVISOR_UNAVAILABLE\nNEMOCLAW_CONTROL_STAGE=discover-supervisor\n",
+        },
+        { exitCode: 1, stderr: "SUPERVISOR_NOT_RUNNING\n" },
+        { exitCode: 1, stderr: "SUPERVISOR_NOT_RUNNING\n" },
+      );
+      const gateway = buildGateway(runner);
+      const sleep = vi.fn(async (milliseconds: number) => {
+        events.push(`sleep-${milliseconds}`);
+      });
+      const onRetry = vi.fn();
+
+      await gateway.waitForMissingManagedSupervisor("container-123", {
+        attempts: 3,
+        delayMs: 3_000,
+        settleMs: 3_000,
+        sleep,
+        onRetry,
+      });
+
+      expect(onRetry).toHaveBeenCalledOnce();
+      expect(onRetry).toHaveBeenCalledWith(1);
+      expect(events).toEqual([
+        "probe-unavailable",
+        "sleep-3000",
+        "probe-not-running",
+        "sleep-3000",
+        "probe-not-running",
+      ]);
+      expect(runner.calls).toHaveLength(3);
+      for (const call of runner.calls) {
+        expect(call.command).toBe("docker");
+        expect(call.args.slice(0, -1)).toEqual([
+          "exec",
+          "--env",
+          "LD_PRELOAD=",
+          "--env",
+          "PYTHONPATH=",
+          "--user",
+          "root",
+          "container-123",
+          "/usr/local/bin/nemoclaw-gateway-control",
+          "probe",
+        ]);
+        expect(call.args.at(-1)).toMatch(/^[0-9a-f]{64}$/);
+      }
+    });
+
+    it("does not accept a composite missing-supervisor diagnostic", async () => {
+      const runner = new ScriptedRunner();
+      runner.queue({
+        exitCode: 1,
+        stderr: "SUPERVISOR_NOT_RUNNING\nNEMOCLAW_CONTROL_STAGE=discover-supervisor\n",
+      });
+      const gateway = buildGateway(runner);
+
+      await expect(
+        gateway.waitForMissingManagedSupervisor("container-123", {
+          attempts: 1,
+          delayMs: 0,
+          settleMs: 0,
+        }),
+      ).rejects.toThrow(/polling exhausted/);
+    });
+
+    it("does not accept missing-supervisor output with stdout", async () => {
+      const runner = new ScriptedRunner();
+      runner.queue({
+        exitCode: 1,
+        stdout: "unexpected output\n",
+        stderr: "SUPERVISOR_NOT_RUNNING\n",
+      });
+      const gateway = buildGateway(runner);
+
+      await expect(
+        gateway.waitForMissingManagedSupervisor("container-123", {
+          attempts: 1,
+          delayMs: 0,
+          settleMs: 0,
+        }),
+      ).rejects.toThrow(/polling exhausted/);
+    });
+
+    it.each([
+      { condition: "timed out", reply: { timedOut: true } },
+      { condition: "was terminated by a signal", reply: { signal: "SIGTERM" as const } },
+    ])("does not accept a probe that $condition", async ({ reply }) => {
+      const runner = new ScriptedRunner();
+      runner.queue({ exitCode: 1, stderr: "SUPERVISOR_NOT_RUNNING\n", ...reply });
+      const gateway = buildGateway(runner);
+
+      await expect(
+        gateway.waitForMissingManagedSupervisor("container-123", {
+          attempts: 1,
+          delayMs: 0,
+          settleMs: 0,
+        }),
+      ).rejects.toThrow(/polling exhausted/);
+    });
+
+    it("retries when supervisor absence changes during the settle interval", async () => {
+      const runner = new ScriptedRunner();
+      runner.queue(
+        { exitCode: 1, stderr: "SUPERVISOR_NOT_RUNNING\n" },
+        { exitCode: 0, stdout: "SUPERVISOR_RUNNING\n" },
+        { exitCode: 1, stderr: "SUPERVISOR_NOT_RUNNING\n" },
+        { exitCode: 0, stdout: "SUPERVISOR_RUNNING\n" },
+      );
+      const gateway = buildGateway(runner);
+      const sleep = vi.fn(async () => undefined);
+      const onRetry = vi.fn();
+
+      await expect(
+        gateway.waitForMissingManagedSupervisor("container-123", {
+          attempts: 2,
+          delayMs: 0,
+          settleMs: 3_000,
+          sleep,
+          onRetry,
+        }),
+      ).rejects.toThrow(/polling exhausted/);
+
+      expect(sleep).toHaveBeenCalledTimes(2);
+      expect(onRetry).toHaveBeenNthCalledWith(1, 1);
+      expect(onRetry).toHaveBeenNthCalledWith(2, 2);
+      expect(runner.calls).toHaveLength(4);
+    });
+  });
+
   describe("expectGuardChainActive", () => {
     it("passes when proxy-env.sh contains the default safety-net + ciao markers", async () => {
       const runner = new ScriptedRunner();