diff --git a/test/e2e/fixtures/polling.ts b/test/e2e/fixtures/polling.ts new file mode 100644 index 00000000000..b95f197dc32 --- /dev/null +++ b/test/e2e/fixtures/polling.ts @@ -0,0 +1,63 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export interface PollAttempt { + attempt: number; + artifactName: string; + value: T; +} + +export interface PollOptions { + artifactPrefix: string; + probe: (attempt: number, artifactName: string) => Promise; + accept: (value: T, attempt: number) => boolean; + attempts?: number; + deadlineMs?: number; + delayMs?: number | ((attempt: number) => number); + signal?: AbortSignal; + terminal?: (value: T, attempt: number) => string | undefined; + sleep?: (ms: number) => Promise; + now?: () => number; +} + +export type PollingFailureReason = "aborted" | "terminal" | "exhausted"; + +export class PollingError extends Error { + constructor( + message: string, + readonly lastAttempt?: PollAttempt, + readonly reason: PollingFailureReason = "exhausted", + ) { + super(message); + } +} + +export function pollingArtifactName(prefix: string, attempt: number): string { + return `${prefix}-attempt-${String(attempt).padStart(2, "0")}`; +} + +export async function pollUntil(options: PollOptions): Promise> { + if (options.attempts === undefined && options.deadlineMs === undefined) { + throw new Error("pollUntil requires attempts or deadlineMs"); + } + const now = options.now ?? Date.now; + const sleep = + options.sleep ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))); + const deadline = options.deadlineMs === undefined ? undefined : now() + options.deadlineMs; + let lastAttempt: PollAttempt | undefined; + for (let attempt = 1; ; attempt += 1) { + if (options.signal?.aborted) throw new PollingError("polling aborted", lastAttempt, "aborted"); + if (options.attempts !== undefined && attempt > options.attempts) break; + if (deadline !== undefined && attempt > 1 && now() >= deadline) break; + const artifactName = pollingArtifactName(options.artifactPrefix, attempt); + const value = await options.probe(attempt, artifactName); + lastAttempt = { attempt, artifactName, value }; + const terminal = options.terminal?.(value, attempt); + if (terminal) throw new PollingError(terminal, lastAttempt, "terminal"); + if (options.accept(value, attempt)) return lastAttempt; + const delay = + typeof options.delayMs === "function" ? options.delayMs(attempt) : (options.delayMs ?? 0); + if (delay > 0) await sleep(delay); + } + throw new PollingError("polling exhausted its configured bound", lastAttempt); +} diff --git a/test/e2e/live/concurrent-gateway-ports.test.ts b/test/e2e/live/concurrent-gateway-ports.test.ts index dce6689f1d8..068bd719537 100644 --- a/test/e2e/live/concurrent-gateway-ports.test.ts +++ b/test/e2e/live/concurrent-gateway-ports.test.ts @@ -19,6 +19,7 @@ import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts"; import { CLI_DIST_ENTRYPOINT, CLI_ENTRYPOINT } from "../fixtures/paths.ts"; +import { PollingError, pollUntil } from "../fixtures/polling.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; const SANDBOX_A = process.env.NEMOCLAW_CGP_SANDBOX_A ?? "e2e-cgp-a"; @@ -138,26 +139,35 @@ async function waitForSandboxReady( gatewayName: string, artifactPrefix: string, ): Promise { - let lastPhase = "missing"; - let lastOutput = ""; - for (let attempt = 1; attempt <= PROBE_ATTEMPTS; attempt += 1) { - const result = await sandbox.openshell(["sandbox", "list", "-g", gatewayName], { - artifactName: `${artifactPrefix}-attempt-${attempt}`, - env: openshellEnvForGateway(gatewayName), - timeoutMs: 30_000, + try { + const result = await pollUntil({ + artifactPrefix, + attempts: PROBE_ATTEMPTS, + delayMs: PROBE_DELAY_MS, + probe: async (_attempt, artifactName) => { + const probe = await sandbox.openshell(["sandbox", "list", "-g", gatewayName], { + artifactName, + env: openshellEnvForGateway(gatewayName), + timeoutMs: 30_000, + }); + const output = resultText(probe); + return { output, phase: sandboxPhaseFromList(output, sandboxName) ?? "missing" }; + }, + accept: ({ phase }) => phase === "Ready" || phase === "Running", + terminal: ({ phase }) => + phase === "Error" || phase === "Failed" || phase === "CrashLoopBackOff" + ? `${sandboxName} reached terminal phase '${phase}' on ${gatewayName}` + : undefined, }); - lastOutput = resultText(result); - const phase = sandboxPhaseFromList(lastOutput, sandboxName); - if (phase) lastPhase = phase; - if (phase === "Ready" || phase === "Running") return phase; - if (phase === "Error" || phase === "Failed" || phase === "CrashLoopBackOff") { - throw new Error(`${sandboxName} reached terminal phase '${phase}' on ${gatewayName}`); - } - if (attempt < PROBE_ATTEMPTS) await sleep(PROBE_DELAY_MS); + return result.value.phase; + } catch (error) { + if (!(error instanceof PollingError)) throw error; + if (error.reason === "terminal") throw error; + const last = error.lastAttempt?.value; + throw new Error( + `${sandboxName} did not reach Ready/Running on ${gatewayName}; last phase '${last?.phase ?? "missing"}'\n${last?.output ?? ""}`, + ); } - throw new Error( - `${sandboxName} did not reach Ready/Running on ${gatewayName}; last phase '${lastPhase}'\n${lastOutput}`, - ); } async function expectPortListening( diff --git a/test/e2e/live/network-policy-denied-log.ts b/test/e2e/live/network-policy-denied-log.ts index 6580e77a089..e1f4f3ed9c4 100644 --- a/test/e2e/live/network-policy-denied-log.ts +++ b/test/e2e/live/network-policy-denied-log.ts @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { PollingError, pollUntil } from "../fixtures/polling.ts"; + export type DeniedReasonLogProof = { line: string; reason: string; @@ -30,13 +32,23 @@ export async function pollDeniedReasonLog(options: { settle: () => Promise; }): Promise { let latestLogs = ""; - for (let attempt = 1; attempt <= options.attempts; attempt += 1) { - latestLogs = await options.readLogs(attempt); - const proof = deniedReasonLogProof(latestLogs, options.endpoint); - if (proof) return proof; - await options.settle(); + try { + const result = await pollUntil({ + artifactPrefix: "network-policy-denied-log", + attempts: options.attempts, + delayMs: 1, + sleep: async () => options.settle(), + probe: async (attempt) => { + latestLogs = await options.readLogs(attempt); + return deniedReasonLogProof(latestLogs, options.endpoint); + }, + accept: (proof) => proof !== null, + }); + return result.value as DeniedReasonLogProof; + } catch (error) { + if (!(error instanceof PollingError)) throw error; + throw new Error( + `denied egress audit event for ${options.endpoint} did not settle into nemoclaw logs --tail 50:\n${latestLogs}`, + ); } - throw new Error( - `denied egress audit event for ${options.endpoint} did not settle into nemoclaw logs --tail 50:\n${latestLogs}`, - ); } diff --git a/test/e2e/support/e2e-polling.test.ts b/test/e2e/support/e2e-polling.test.ts new file mode 100644 index 00000000000..de8a51e1027 --- /dev/null +++ b/test/e2e/support/e2e-polling.test.ts @@ -0,0 +1,77 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { PollingError, pollUntil } from "../fixtures/polling.ts"; + +describe("bounded polling", () => { + it("numbers artifacts and returns the accepted attempt", async () => { + const probe = vi.fn(async (attempt: number) => attempt); + const result = await pollUntil({ + artifactPrefix: "ready", + attempts: 3, + probe, + accept: (v) => v === 2, + }); + expect(result).toEqual({ attempt: 2, artifactName: "ready-attempt-02", value: 2 }); + }); + + it("supports backoff and exposes the last result on exhaustion", async () => { + const delays: number[] = []; + let error: PollingError | undefined; + try { + await pollUntil({ + artifactPrefix: "health", + attempts: 2, + delayMs: (attempt) => attempt * 10, + sleep: async (ms) => { + delays.push(ms); + }, + probe: async (attempt) => `not-ready-${attempt}`, + accept: () => false, + }); + } catch (caught) { + expect(caught).toBeInstanceOf(PollingError); + error = caught as PollingError; + } + expect(delays).toEqual([10, 20]); + expect(error?.lastAttempt?.value).toBe("not-ready-2"); + expect(error?.reason).toBe("exhausted"); + }); + + it("honors deadlines, terminal states, and abort signals", async () => { + let now = 0; + await expect( + pollUntil({ + artifactPrefix: "deadline", + deadlineMs: 5, + now: () => now, + probe: async () => { + now = 5; + return "pending"; + }, + accept: () => false, + }), + ).rejects.toThrow(/exhausted/); + await expect( + pollUntil({ + artifactPrefix: "terminal", + attempts: 3, + probe: async () => "Failed", + accept: () => false, + terminal: (value) => (value === "Failed" ? "terminal failure" : undefined), + }), + ).rejects.toMatchObject({ reason: "terminal" }); + const controller = new AbortController(); + controller.abort(); + await expect( + pollUntil({ + artifactPrefix: "abort", + attempts: 1, + signal: controller.signal, + probe: async () => true, + accept: Boolean, + }), + ).rejects.toMatchObject({ reason: "aborted" }); + }); +});