refactor(e2e): add bounded polling primitives for live readiness checks

test/e2e/fixtures/polling.ts

@@ -0,0 +1,63 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+export interface PollAttempt<T> {
+  attempt: number;
+  artifactName: string;
+  value: T;
+}
+
+export interface PollOptions<T> {
+  artifactPrefix: string;
+  probe: (attempt: number, artifactName: string) => Promise<T>;
+  accept: (value: T, attempt: number) => boolean;
+  attempts?: number;
+  deadlineMs?: number;
+  delayMs?: number | ((attempt: number) => number);
+  signal?: AbortSignal;
+  terminal?: (value: T, attempt: number) => string | undefined;
+  sleep?: (ms: number) => Promise<void>;
+  now?: () => number;
+}
+
+export type PollingFailureReason = "aborted" | "terminal" | "exhausted";
+
+export class PollingError<T> extends Error {
+  constructor(
+    message: string,
+    readonly lastAttempt?: PollAttempt<T>,
+    readonly reason: PollingFailureReason = "exhausted",
+  ) {
+    super(message);
+  }
+}
+
+export function pollingArtifactName(prefix: string, attempt: number): string {
+  return `${prefix}-attempt-${String(attempt).padStart(2, "0")}`;
+}
+
+export async function pollUntil<T>(options: PollOptions<T>): Promise<PollAttempt<T>> {
+  if (options.attempts === undefined && options.deadlineMs === undefined) {
+    throw new Error("pollUntil requires attempts or deadlineMs");
+  }
+  const now = options.now ?? Date.now;
+  const sleep =
+    options.sleep ?? ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
+  const deadline = options.deadlineMs === undefined ? undefined : now() + options.deadlineMs;
+  let lastAttempt: PollAttempt<T> | undefined;
+  for (let attempt = 1; ; attempt += 1) {
+    if (options.signal?.aborted) throw new PollingError("polling aborted", lastAttempt, "aborted");
+    if (options.attempts !== undefined && attempt > options.attempts) break;
+    if (deadline !== undefined && attempt > 1 && now() >= deadline) break;
+    const artifactName = pollingArtifactName(options.artifactPrefix, attempt);
+    const value = await options.probe(attempt, artifactName);
+    lastAttempt = { attempt, artifactName, value };
+    const terminal = options.terminal?.(value, attempt);
+    if (terminal) throw new PollingError(terminal, lastAttempt, "terminal");
+    if (options.accept(value, attempt)) return lastAttempt;
+    const delay =
+      typeof options.delayMs === "function" ? options.delayMs(attempt) : (options.delayMs ?? 0);
+    if (delay > 0) await sleep(delay);
+  }
+  throw new PollingError("polling exhausted its configured bound", lastAttempt);
+}

test/e2e/live/concurrent-gateway-ports.test.ts

@@ -19,6 +19,7 @@ import { validateSandboxName } from "../fixtures/clients/sandbox.ts";
 import { expect, test } from "../fixtures/e2e-test.ts";
 import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts";
 import { CLI_DIST_ENTRYPOINT, CLI_ENTRYPOINT } from "../fixtures/paths.ts";
+import { PollingError, pollUntil } from "../fixtures/polling.ts";
 import type { ShellProbeResult } from "../fixtures/shell-probe.ts";
 
 const SANDBOX_A = process.env.NEMOCLAW_CGP_SANDBOX_A ?? "e2e-cgp-a";
@@ -138,26 +139,35 @@ async function waitForSandboxReady(
   gatewayName: string,
   artifactPrefix: string,
 ): Promise<string> {
-  let lastPhase = "missing";
-  let lastOutput = "";
-  for (let attempt = 1; attempt <= PROBE_ATTEMPTS; attempt += 1) {
-    const result = await sandbox.openshell(["sandbox", "list", "-g", gatewayName], {
-      artifactName: `${artifactPrefix}-attempt-${attempt}`,
-      env: openshellEnvForGateway(gatewayName),
-      timeoutMs: 30_000,
+  try {
+    const result = await pollUntil({
+      artifactPrefix,
+      attempts: PROBE_ATTEMPTS,
+      delayMs: PROBE_DELAY_MS,
+      probe: async (_attempt, artifactName) => {
+        const probe = await sandbox.openshell(["sandbox", "list", "-g", gatewayName], {
+          artifactName,
+          env: openshellEnvForGateway(gatewayName),
+          timeoutMs: 30_000,
+        });
+        const output = resultText(probe);
+        return { output, phase: sandboxPhaseFromList(output, sandboxName) ?? "missing" };
+      },
+      accept: ({ phase }) => phase === "Ready" || phase === "Running",
+      terminal: ({ phase }) =>
+        phase === "Error" || phase === "Failed" || phase === "CrashLoopBackOff"
+          ? `${sandboxName} reached terminal phase '${phase}' on ${gatewayName}`
+          : undefined,
     });
-    lastOutput = resultText(result);
-    const phase = sandboxPhaseFromList(lastOutput, sandboxName);
-    if (phase) lastPhase = phase;
-    if (phase === "Ready" || phase === "Running") return phase;
-    if (phase === "Error" || phase === "Failed" || phase === "CrashLoopBackOff") {
-      throw new Error(`${sandboxName} reached terminal phase '${phase}' on ${gatewayName}`);
-    }
-    if (attempt < PROBE_ATTEMPTS) await sleep(PROBE_DELAY_MS);
+    return result.value.phase;
+  } catch (error) {
+    if (!(error instanceof PollingError)) throw error;
+    if (error.reason === "terminal") throw error;
+    const last = error.lastAttempt?.value;
+    throw new Error(
+      `${sandboxName} did not reach Ready/Running on ${gatewayName}; last phase '${last?.phase ?? "missing"}'\n${last?.output ?? ""}`,
+    );
   }
-  throw new Error(
-    `${sandboxName} did not reach Ready/Running on ${gatewayName}; last phase '${lastPhase}'\n${lastOutput}`,
-  );
 }
 
 async function expectPortListening(

test/e2e/live/network-policy-denied-log.ts

@@ -1,6 +1,8 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+import { PollingError, pollUntil } from "../fixtures/polling.ts";
+
 export type DeniedReasonLogProof = {
   line: string;
   reason: string;
@@ -30,13 +32,23 @@ export async function pollDeniedReasonLog(options: {
   settle: () => Promise<void>;
 }): Promise<DeniedReasonLogProof> {
   let latestLogs = "";
-  for (let attempt = 1; attempt <= options.attempts; attempt += 1) {
-    latestLogs = await options.readLogs(attempt);
-    const proof = deniedReasonLogProof(latestLogs, options.endpoint);
-    if (proof) return proof;
-    await options.settle();
+  try {
+    const result = await pollUntil({
+      artifactPrefix: "network-policy-denied-log",
+      attempts: options.attempts,
+      delayMs: 1,
+      sleep: async () => options.settle(),
+      probe: async (attempt) => {
+        latestLogs = await options.readLogs(attempt);
+        return deniedReasonLogProof(latestLogs, options.endpoint);
+      },
+      accept: (proof) => proof !== null,
+    });
+    return result.value as DeniedReasonLogProof;
+  } catch (error) {
+    if (!(error instanceof PollingError)) throw error;
+    throw new Error(
+      `denied egress audit event for ${options.endpoint} did not settle into nemoclaw logs --tail 50:\n${latestLogs}`,
+    );
   }
-  throw new Error(
-    `denied egress audit event for ${options.endpoint} did not settle into nemoclaw logs --tail 50:\n${latestLogs}`,
-  );
 }

test/e2e/support/e2e-polling.test.ts

@@ -0,0 +1,77 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, expect, it, vi } from "vitest";
+import { PollingError, pollUntil } from "../fixtures/polling.ts";
+
+describe("bounded polling", () => {
+  it("numbers artifacts and returns the accepted attempt", async () => {
+    const probe = vi.fn(async (attempt: number) => attempt);
+    const result = await pollUntil({
+      artifactPrefix: "ready",
+      attempts: 3,
+      probe,
+      accept: (v) => v === 2,
+    });
+    expect(result).toEqual({ attempt: 2, artifactName: "ready-attempt-02", value: 2 });
+  });
+
+  it("supports backoff and exposes the last result on exhaustion", async () => {
+    const delays: number[] = [];
+    let error: PollingError<string> | undefined;
+    try {
+      await pollUntil({
+        artifactPrefix: "health",
+        attempts: 2,
+        delayMs: (attempt) => attempt * 10,
+        sleep: async (ms) => {
+          delays.push(ms);
+        },
+        probe: async (attempt) => `not-ready-${attempt}`,
+        accept: () => false,
+      });
+    } catch (caught) {
+      expect(caught).toBeInstanceOf(PollingError);
+      error = caught as PollingError<string>;
+    }
+    expect(delays).toEqual([10, 20]);
+    expect(error?.lastAttempt?.value).toBe("not-ready-2");
+    expect(error?.reason).toBe("exhausted");
+  });
+
+  it("honors deadlines, terminal states, and abort signals", async () => {
+    let now = 0;
+    await expect(
+      pollUntil({
+        artifactPrefix: "deadline",
+        deadlineMs: 5,
+        now: () => now,
+        probe: async () => {
+          now = 5;
+          return "pending";
+        },
+        accept: () => false,
+      }),
+    ).rejects.toThrow(/exhausted/);
+    await expect(
+      pollUntil({
+        artifactPrefix: "terminal",
+        attempts: 3,
+        probe: async () => "Failed",
+        accept: () => false,
+        terminal: (value) => (value === "Failed" ? "terminal failure" : undefined),
+      }),
+    ).rejects.toMatchObject({ reason: "terminal" });
+    const controller = new AbortController();
+    controller.abort();
+    await expect(
+      pollUntil({
+        artifactPrefix: "abort",
+        attempts: 1,
+        signal: controller.signal,
+        probe: async () => true,
+        accept: Boolean,
+      }),
+    ).rejects.toMatchObject({ reason: "aborted" });
+  });
+});