From f6b38caef13b969f1db439eadc61829181643033 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 22 Jun 2026 09:35:51 -0400 Subject: [PATCH 01/15] test(e2e): migrate test-tunnel-lifecycle.sh to vitest --- .github/workflows/e2e-vitest-scenarios.yaml | 91 ++++ .../live/tunnel-lifecycle.test.ts | 453 ++++++++++++++++++ .../e2e-scenarios-workflow.test.ts | 16 + 3 files changed, 560 insertions(+) create mode 100644 test/e2e-scenario/live/tunnel-lifecycle.test.ts diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index b8a7c6816a1..a1c240c7af3 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -2950,6 +2950,96 @@ jobs: docker logout docker.io || true rm -rf "${DOCKER_CONFIG}" + tunnel-lifecycle-vitest: + needs: generate-matrix + if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',tunnel-lifecycle-vitest,') || contains(format(',{0},', inputs.scenarios), ',tunnel-lifecycle,') }} + runs-on: ubuntu-latest + timeout-minutes: 75 + env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "tunnel-lifecycle" + DOCKER_CONFIG: ${{ github.workspace }}/.docker-config-tunnel-lifecycle + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/tunnel-lifecycle + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-tunnel-lifecycle" + OPENSHELL_GATEWAY: "nemoclaw" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Authenticate to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + shell: bash + run: | + set -euo pipefail + if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then + echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls." + exit 0 + fi + mkdir -p "${DOCKER_CONFIG}" + chmod 700 "${DOCKER_CONFIG}" + login_succeeded=0 + for attempt in 1 2 3; do + if echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then + login_succeeded=1 + break + fi + if [[ "$attempt" -lt 3 ]]; then + echo "::warning::Docker Hub login attempt ${attempt} failed; retrying." + sleep 5 + fi + done + if [[ "$login_succeeded" -ne 1 ]]; then + echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls." + fi + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Run tunnel lifecycle live test + # Migrated from test/e2e/test-tunnel-lifecycle.sh. This preserves the + # real Docker/OpenShell onboard, host cloudflared quick-tunnel, + # local-dashboard readiness, public tunnel probe, and stop/status + # cleanup boundaries under Vitest. + env: + NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + run: | + set -euo pipefail + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/tunnel-lifecycle.test.ts \ + --silent=false --reporter=default + + - name: Upload tunnel lifecycle artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-tunnel-lifecycle + path: e2e-artifacts/vitest/tunnel-lifecycle/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + + - name: Clean up Docker auth + if: always() + run: | + set -euo pipefail + docker logout docker.io || true + rm -rf "${DOCKER_CONFIG}" + # ── PR result comment (mirrors nightly-e2e.yaml's report-to-pr) ─────────── # Posts a results table on the open PR for the dispatching branch (or the # PR identified by `inputs.pr_number`). `if: always()` so the comment lands @@ -2996,6 +3086,7 @@ jobs: issue-2478-crash-loop-recovery-vitest, gateway-health-honest-vitest, channels-add-remove-vitest, + tunnel-lifecycle-vitest, ] if: ${{ always() && github.event_name == 'workflow_dispatch' }} permissions: diff --git a/test/e2e-scenario/live/tunnel-lifecycle.test.ts b/test/e2e-scenario/live/tunnel-lifecycle.test.ts new file mode 100644 index 00000000000..18354733c49 --- /dev/null +++ b/test/e2e-scenario/live/tunnel-lifecycle.test.ts @@ -0,0 +1,453 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Live Vitest replacement for test/e2e/test-tunnel-lifecycle.sh. + * + * Preserves the legacy real boundaries: Docker/OpenShell onboarding, the + * installed/source NemoClaw CLI, host `cloudflared`, the local dashboard origin, + * public trycloudflare reachability, cloudflared log diagnosis, and tunnel stop + * cleanup/status removal. + */ + +import fs from "node:fs"; +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { resultText } from "../fixtures/clients/index.ts"; +import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const TEST_SANDBOX_PREFIX = "e2e-tunnel-lifecycle"; +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? TEST_SANDBOX_PREFIX; +const LOCAL_DASHBOARD_PORT = process.env.NEMOCLAW_DASHBOARD_PORT ?? "18789"; +const TEST_TIMEOUT_MS = Number(process.env.NEMOCLAW_E2E_TIMEOUT_SECONDS ?? 3_600) * 1_000; +const ONBOARD_TIMEOUT_MS = 30 * 60_000; +const COMMAND_TIMEOUT_MS = 60_000; +const TUNNEL_URL_PATTERN = /https:\/\/[a-z0-9-]+\.trycloudflare\.com\b[\w./?%&=-]*/i; +const DASHBOARD_MARKER_PATTERN = /OpenClaw Control<\/title>|<openclaw-app/i; + +validateSandboxName(SANDBOX_NAME); + +type CurlProbe = { + httpCode: string; + body: string; + result: ShellProbeResult; +}; + +function assertTestOwnedSandboxName(): void { + if (!SANDBOX_NAME.startsWith(TEST_SANDBOX_PREFIX)) { + throw new Error( + `tunnel-lifecycle live test is destructive and only accepts sandbox names with prefix ${TEST_SANDBOX_PREFIX}; got ${SANDBOX_NAME}`, + ); + } +} + +function sleep(ms: number): Promise<void> { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + NEMOCLAW_POLICY_TIER: "open", + NEMOCLAW_AGENT: "openclaw", + NEMOCLAW_PROVIDER: "cloud", + OPENSHELL_GATEWAY: "nemoclaw", + ...(process.env.NEMOCLAW_DASHBOARD_PORT + ? { NEMOCLAW_DASHBOARD_PORT: process.env.NEMOCLAW_DASHBOARD_PORT } + : {}), + ...extra, + }; +} + +function isCloudflareTransientText(text: string): boolean { + return /failed to unmarshal quick Tunnel|quick tunnels? (are )?(temporarily )?disabled|failed to (dial|register)|tunnel server.*error|i\/o timeout|EOF.*tunnel|couldn.?t start tunnel|tunnel creation failed|bad gateway|\b50[234]\b/i.test( + text, + ); +} + +function isCloudflareTransientHttpCode(code: string): boolean { + return ["000", "502", "503", "504"].includes(code); +} + +function getCloudflaredLogPath(): string | undefined { + const sandboxLog = path.join("/tmp", `nemoclaw-services-${SANDBOX_NAME}`, "cloudflared.log"); + if (fs.existsSync(sandboxLog)) return sandboxLog; + let candidates: string[] = []; + try { + candidates = fs + .readdirSync("/tmp", { withFileTypes: true }) + .filter((entry) => entry.isDirectory() && entry.name.startsWith("nemoclaw-services-")) + .map((entry) => path.join("/tmp", entry.name, "cloudflared.log")) + .filter((candidate) => fs.existsSync(candidate)); + } catch { + return undefined; + } + return candidates.sort((a, b) => fs.statSync(b).mtimeMs - fs.statSync(a).mtimeMs).at(0); +} + +function readCloudflaredLog(): string { + const logPath = getCloudflaredLogPath(); + if (!logPath) return ""; + return fs.readFileSync(logPath, "utf8"); +} + +function cloudflaredLogTail(lines = 80): string { + const logPath = getCloudflaredLogPath(); + if (!logPath) return "(no cloudflared.log found under /tmp/nemoclaw-services-*/)"; + const text = fs.readFileSync(logPath, "utf8"); + return [`--- cloudflared.log (${logPath}, last ${lines} lines) ---`, ...text.split(/\r?\n/).slice(-lines)].join( + "\n", + ); +} + +function classifyCloudflaredLog(): + | "nemoclaw_no_spawn" + | "nemoclaw_capture_bug" + | "nemoclaw_local" + | "cloudflare" + | "unknown" { + const logPath = getCloudflaredLogPath(); + if (!logPath) return "nemoclaw_no_spawn"; + const log = fs.readFileSync(logPath, "utf8"); + if (TUNNEL_URL_PATTERN.test(log)) return "nemoclaw_capture_bug"; + if ( + /unable to reach the origin|connection refused.*127\.0\.0\.1|connection refused.*localhost|dial tcp.*127\.0\.0\.1.*refused/i.test( + log, + ) + ) { + return "nemoclaw_local"; + } + if (isCloudflareTransientText(log)) return "cloudflare"; + return "unknown"; +} + +function extractTunnelUrl(text: string): string | undefined { + return text.match(TUNNEL_URL_PATTERN)?.[0]; +} + +function parseCurlProbe(result: ShellProbeResult): CurlProbe { + const text = result.stdout; + const match = text.match(/\n__HTTP_CODE:(\d{3})\s*$/); + const httpCode = match?.[1] ?? "000"; + const body = match ? text.slice(0, match.index) : text; + return { httpCode, body, result }; +} + +async function bestEffort(run: () => Promise<unknown>): Promise<void> { + try { + await run(); + } catch { + // Cleanup remains best-effort so the primary E2E failure stays visible. + } +} + +test.skipIf(!shouldRunLiveE2EScenarios())( + "tunnel-lifecycle: cloudflared quick tunnel starts, serves OpenClaw, and stops cleanly", + { timeout: TEST_TIMEOUT_MS }, + async ({ artifacts, cleanup, host, secrets, skip }) => { + assertTestOwnedSandboxName(); + const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); + expect(apiKey.startsWith("nvapi-"), "NVIDIA_INFERENCE_API_KEY must start with nvapi-").toBe( + true, + ); + + await artifacts.writeJson("contract.json", { + legacySource: "test/e2e/test-tunnel-lifecycle.sh", + sandboxName: SANDBOX_NAME, + localDashboardPort: LOCAL_DASHBOARD_PORT, + preservedBoundaries: [ + "real Docker/OpenShell OpenClaw sandbox onboarding", + "host cloudflared binary and quick-tunnel registration", + "nemoclaw tunnel start/status/stop CLI commands", + "local dashboard origin readiness before tunnel attribution", + "public trycloudflare HTTP probe with dashboard marker assertion", + "cloudflared.log classification for NemoClaw-vs-Cloudflare failures", + ], + }); + + cleanup.add("stop cloudflared quick tunnel", async () => { + await bestEffort(() => + host.nemoclaw(["tunnel", "stop"], { + artifactName: "cleanup-tunnel-stop", + env: commandEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }), + ); + }); + cleanup.add(`destroy sandbox ${SANDBOX_NAME}`, async () => { + if (process.env.NEMOCLAW_E2E_KEEP_SANDBOX === "1") return; + await bestEffort(() => + host.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy-tunnel-lifecycle", + timeoutMs: 15 * 60_000, + }), + ); + }); + + const docker = await host.command("docker", ["info"], { + artifactName: "prereq-docker-info-tunnel-lifecycle", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + if (docker.exitCode !== 0) { + if (process.env.GITHUB_ACTIONS === "true") { + throw new Error(`Docker is required for tunnel lifecycle E2E: ${resultText(docker)}`); + } + skip("Docker is required for tunnel lifecycle E2E"); + } + + const cloudflared = await host.command( + "bash", + [ + "-lc", + [ + "set -euo pipefail", + "if command -v cloudflared >/dev/null 2>&1; then", + " cloudflared --version", + " exit 0", + "fi", + 'if [ "${GITHUB_ACTIONS:-}" != "true" ]; then', + ' echo "cloudflared not found" >&2', + " exit 127", + "fi", + "source test/e2e/lib/cloudflared-version-resolver.sh", + "sudo mkdir -p --mode=0755 /usr/share/keyrings", + "curl -fsSL https://pkg.cloudflare.com/cloudflare-main.gpg | sudo tee /usr/share/keyrings/cloudflare-main.gpg >/dev/null", + 'echo "deb [signed-by=/usr/share/keyrings/cloudflare-main.gpg] https://pkg.cloudflare.com/cloudflared $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/cloudflared.list >/dev/null', + "sudo apt-get update -qq", + 'available_versions="$(apt-cache madison cloudflared | awk \'{print $3}\')"', + 'cf_min_version="${CLOUDFLARED_MIN_VERSION:-$CLOUDFLARED_DEFAULT_MIN_VERSION}"', + 'if [ -n "${CLOUDFLARED_VERSION:-}" ]; then', + ' cf_version="$(cloudflared_resolve_package_version "$available_versions" "$cf_min_version" "$CLOUDFLARED_VERSION")"', + "else", + ' cf_version="$(cloudflared_resolve_package_version "$available_versions" "$cf_min_version")"', + "fi", + 'sudo apt-get install -y "cloudflared=${cf_version}"', + "cloudflared --version", + ].join("\n"), + ], + { + artifactName: "prereq-cloudflared-version", + cwd: REPO_ROOT, + env: { + ...buildAvailabilityProbeEnv(), + ...(process.env.CLOUDFLARED_VERSION + ? { CLOUDFLARED_VERSION: process.env.CLOUDFLARED_VERSION } + : {}), + ...(process.env.CLOUDFLARED_MIN_VERSION + ? { CLOUDFLARED_MIN_VERSION: process.env.CLOUDFLARED_MIN_VERSION } + : {}), + }, + timeoutMs: 5 * 60_000, + }, + ); + if (cloudflared.exitCode !== 0) { + if (process.env.GITHUB_ACTIONS === "true") { + throw new Error(`cloudflared is required for tunnel lifecycle E2E: ${resultText(cloudflared)}`); + } + skip("cloudflared is required for tunnel lifecycle E2E"); + } + + expect(fs.existsSync(path.join(REPO_ROOT, "install.sh"))).toBe(true); + await host.bestEffortCleanupSandbox(SANDBOX_NAME, { + artifactName: "pre-cleanup-nemoclaw-destroy-tunnel-lifecycle", + timeoutMs: 15 * 60_000, + }); + + const install = await host.command( + "bash", + ["install.sh", "--non-interactive", "--yes-i-accept-third-party-software"], + { + artifactName: "install-sh-tunnel-lifecycle", + cwd: REPO_ROOT, + env: commandEnv({ NVIDIA_INFERENCE_API_KEY: apiKey }), + redactionValues: [apiKey], + timeoutMs: ONBOARD_TIMEOUT_MS, + }, + ); + expect(install.exitCode, resultText(install)).toBe(0); + + await host.expectListed(SANDBOX_NAME, { artifactName: "post-install-nemoclaw-list" }); + + let localReady = false; + for (let attempt = 1; attempt <= 30; attempt += 1) { + const local = await host.command( + "curl", + ["-sS", "-o", "/dev/null", "-w", "%{http_code}", "--max-time", "5", `http://localhost:${LOCAL_DASHBOARD_PORT}/`], + { + artifactName: `local-dashboard-ready-${attempt}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 10_000, + }, + ); + const code = local.stdout.trim() || "000"; + if (code !== "000") { + localReady = true; + break; + } + await sleep(1_000); + } + expect( + localReady, + `[NemoClaw fault] Local OpenClaw dashboard not reachable on localhost:${LOCAL_DASHBOARD_PORT} after 30s; tunnel cannot proxy a dead origin.`, + ).toBe(true); + + const start = await host.nemoclaw(["tunnel", "start"], { + artifactName: "tunnel-start", + env: commandEnv(), + timeoutMs: 90_000, + }); + if (start.exitCode !== 0) { + await artifacts.writeText("cloudflared-log-after-start-failure.txt", cloudflaredLogTail()); + if (isCloudflareTransientText(resultText(start)) || classifyCloudflaredLog() === "cloudflare") { + await bestEffort(() => + host.nemoclaw(["tunnel", "stop"], { + artifactName: "tunnel-stop-after-cloudflare-start-failure", + env: commandEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }), + ); + skip( + `[Cloudflare fault] nemoclaw tunnel start exited ${start.exitCode ?? "unknown"} because quick-tunnel registration returned a transient external error.`, + ); + } + throw new Error( + `[NemoClaw fault] nemoclaw tunnel start failed with exit ${start.exitCode ?? "unknown"}: ${resultText(start)}`, + ); + } + + let tunnelUrl: string | undefined; + let lastStatusText = ""; + for (let attempt = 1; attempt <= 15; attempt += 1) { + const status = await host.nemoclaw(["status"], { + artifactName: `status-with-tunnel-url-${attempt}`, + env: commandEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }); + lastStatusText = resultText(status); + tunnelUrl = extractTunnelUrl(lastStatusText); + if (tunnelUrl) break; + await sleep(1_000); + } + + if (!tunnelUrl) { + await artifacts.writeText("cloudflared-log-without-status-url.txt", cloudflaredLogTail()); + const cfClass = classifyCloudflaredLog(); + await bestEffort(() => + host.nemoclaw(["tunnel", "stop"], { + artifactName: "tunnel-stop-after-missing-url", + env: commandEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }), + ); + if (cfClass === "cloudflare") { + skip("[Cloudflare fault] cloudflared failed to register a quick tunnel URL."); + } + let reason: string; + switch (cfClass) { + case "nemoclaw_no_spawn": + reason = "cloudflared.log missing — NemoClaw failed to spawn the cloudflared process"; + break; + case "nemoclaw_capture_bug": + reason = "cloudflared.log has a trycloudflare URL but nemoclaw status did not surface it"; + break; + case "nemoclaw_local": + reason = `cloudflared.log reports it cannot reach localhost:${LOCAL_DASHBOARD_PORT}`; + break; + default: + reason = `tunnel URL did not surface and cloudflared.log did not match a known pattern; status was:\n${lastStatusText}`; + } + throw new Error(`[NemoClaw fault] ${reason}`); + } + + let lastPublicProbe: CurlProbe | undefined; + let backoffMs = 2_000; + for (let attempt = 1; attempt <= 15; attempt += 1) { + const probe = parseCurlProbe( + await host.command( + "curl", + ["-sS", "-L", "--max-time", "30", "-w", "\n__HTTP_CODE:%{http_code}\n", tunnelUrl], + { + artifactName: `public-tunnel-probe-${attempt}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 35_000, + }, + ), + ); + lastPublicProbe = probe; + if (probe.httpCode === "200") break; + + const local = await host.command( + "curl", + ["-sS", "-o", "/dev/null", "-w", "%{http_code}", "--max-time", "5", `http://localhost:${LOCAL_DASHBOARD_PORT}/`], + { + artifactName: `local-dashboard-recheck-${attempt}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 10_000, + }, + ); + const localCode = local.stdout.trim() || "000"; + if (localCode === "000") { + throw new Error( + `[NemoClaw fault] Tunnel returned ${probe.httpCode} and local dashboard regressed during retry loop; likely sandbox/dashboard crash, not Cloudflare.`, + ); + } + await sleep(backoffMs); + backoffMs = Math.min(backoffMs * 2, 30_000); + } + + expect(lastPublicProbe, "public tunnel probe should have run").toBeTruthy(); + if (lastPublicProbe!.httpCode !== "200") { + if ( + isCloudflareTransientHttpCode(lastPublicProbe!.httpCode) || + isCloudflareTransientText(lastPublicProbe!.body) || + isCloudflareTransientText(readCloudflaredLog()) + ) { + skip( + `[Cloudflare fault] Tunnel URL never became reachable while local stayed healthy; last HTTP status ${lastPublicProbe!.httpCode}.`, + ); + } + throw new Error( + `[NemoClaw fault] Tunnel returned unexpected HTTP ${lastPublicProbe!.httpCode} while local stayed healthy; body prefix: ${lastPublicProbe!.body.slice(0, 200)}`, + ); + } + expect(lastPublicProbe!.body, "public tunnel must serve OpenClaw dashboard markers").toMatch( + DASHBOARD_MARKER_PATTERN, + ); + + const stop = await host.nemoclaw(["tunnel", "stop"], { + artifactName: "tunnel-stop", + env: commandEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }); + expect(stop.exitCode, resultText(stop)).toBe(0); + + let postStopUrl: string | undefined; + let statusReadable = false; + for (let attempt = 1; attempt <= 10; attempt += 1) { + const status = await host.nemoclaw(["status"], { + artifactName: `status-after-tunnel-stop-${attempt}`, + env: commandEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }); + if (status.exitCode !== 0) { + await sleep(1_000); + continue; + } + statusReadable = true; + postStopUrl = extractTunnelUrl(resultText(status)); + if (!postStopUrl) break; + await sleep(1_000); + } + expect(statusReadable, "nemoclaw status should be readable after tunnel stop").toBe(true); + expect(postStopUrl, "tunnel URL must be absent after nemoclaw tunnel stop").toBeUndefined(); + }, +); diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index 41195ebb07a..6c6f776f5af 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -521,6 +521,22 @@ describe("e2e-vitest-scenarios workflow boundary", () => { selectedFreeStandingJobs: ["channels-add-remove-vitest"], registryScenarios: [], }); + expect( + evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "tunnel-lifecycle" }), + ).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["tunnel-lifecycle-vitest"], + registryScenarios: [], + }); + expect( + evaluateE2eVitestWorkflowDispatchSelectors({ jobs: "tunnel-lifecycle-vitest" }), + ).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["tunnel-lifecycle-vitest"], + registryScenarios: [], + }); }); it("derives the free-standing inventory from workflow job metadata", () => { From df9a8f1e822ca981704749ffc4ac6abf676a86d6 Mon Sep 17 00:00:00 2001 From: Julie Yaunches <jyaunches@nvidia.com> Date: Mon, 22 Jun 2026 09:59:17 -0400 Subject: [PATCH 02/15] test(e2e): keep tunnel lifecycle test linear --- .../live/tunnel-lifecycle-helpers.ts | 458 ++++++++++++++++++ .../live/tunnel-lifecycle.test.ts | 440 +---------------- 2 files changed, 465 insertions(+), 433 deletions(-) create mode 100644 test/e2e-scenario/live/tunnel-lifecycle-helpers.ts diff --git a/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts b/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts new file mode 100644 index 00000000000..a18715b72ca --- /dev/null +++ b/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts @@ -0,0 +1,458 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Live Vitest replacement for test/e2e/test-tunnel-lifecycle.sh. + * + * Preserves the legacy real boundaries: Docker/OpenShell onboarding, the + * installed/source NemoClaw CLI, host `cloudflared`, the local dashboard origin, + * public trycloudflare reachability, cloudflared log diagnosis, and tunnel stop + * cleanup/status removal. + */ + +import fs from "node:fs"; +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { resultText } from "../fixtures/clients/index.ts"; +import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; +import type { E2EScenarioFixtures } from "../fixtures/e2e-test.ts"; +import { expect } from "../fixtures/e2e-test.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const TEST_SANDBOX_PREFIX = "e2e-tunnel-lifecycle"; +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? TEST_SANDBOX_PREFIX; +const LOCAL_DASHBOARD_PORT = process.env.NEMOCLAW_DASHBOARD_PORT ?? "18789"; +const TEST_TIMEOUT_MS = Number(process.env.NEMOCLAW_E2E_TIMEOUT_SECONDS ?? 3_600) * 1_000; +const ONBOARD_TIMEOUT_MS = 30 * 60_000; +const COMMAND_TIMEOUT_MS = 60_000; +const TUNNEL_URL_PATTERN = /https:\/\/[a-z0-9-]+\.trycloudflare\.com\b[\w./?%&=-]*/i; +const DASHBOARD_MARKER_PATTERN = /<title>OpenClaw Control<\/title>|<openclaw-app/i; + +validateSandboxName(SANDBOX_NAME); + +type CurlProbe = { + httpCode: string; + body: string; + result: ShellProbeResult; +}; + +function assertTestOwnedSandboxName(): void { + if (!SANDBOX_NAME.startsWith(TEST_SANDBOX_PREFIX)) { + throw new Error( + `tunnel-lifecycle live test is destructive and only accepts sandbox names with prefix ${TEST_SANDBOX_PREFIX}; got ${SANDBOX_NAME}`, + ); + } +} + +function sleep(ms: number): Promise<void> { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + NEMOCLAW_POLICY_TIER: "open", + NEMOCLAW_AGENT: "openclaw", + NEMOCLAW_PROVIDER: "cloud", + OPENSHELL_GATEWAY: "nemoclaw", + ...(process.env.NEMOCLAW_DASHBOARD_PORT + ? { NEMOCLAW_DASHBOARD_PORT: process.env.NEMOCLAW_DASHBOARD_PORT } + : {}), + ...extra, + }; +} + +function isCloudflareTransientText(text: string): boolean { + return /failed to unmarshal quick Tunnel|quick tunnels? (are )?(temporarily )?disabled|failed to (dial|register)|tunnel server.*error|i\/o timeout|EOF.*tunnel|couldn.?t start tunnel|tunnel creation failed|bad gateway|\b50[234]\b/i.test( + text, + ); +} + +function isCloudflareTransientHttpCode(code: string): boolean { + return ["000", "502", "503", "504"].includes(code); +} + +function getCloudflaredLogPath(): string | undefined { + const sandboxLog = path.join("/tmp", `nemoclaw-services-${SANDBOX_NAME}`, "cloudflared.log"); + if (fs.existsSync(sandboxLog)) return sandboxLog; + let candidates: string[] = []; + try { + candidates = fs + .readdirSync("/tmp", { withFileTypes: true }) + .filter((entry) => entry.isDirectory() && entry.name.startsWith("nemoclaw-services-")) + .map((entry) => path.join("/tmp", entry.name, "cloudflared.log")) + .filter((candidate) => fs.existsSync(candidate)); + } catch { + return undefined; + } + return candidates.sort((a, b) => fs.statSync(b).mtimeMs - fs.statSync(a).mtimeMs).at(0); +} + +function readCloudflaredLog(): string { + const logPath = getCloudflaredLogPath(); + if (!logPath) return ""; + return fs.readFileSync(logPath, "utf8"); +} + +function cloudflaredLogTail(lines = 80): string { + const logPath = getCloudflaredLogPath(); + if (!logPath) return "(no cloudflared.log found under /tmp/nemoclaw-services-*/)"; + const text = fs.readFileSync(logPath, "utf8"); + return [`--- cloudflared.log (${logPath}, last ${lines} lines) ---`, ...text.split(/\r?\n/).slice(-lines)].join( + "\n", + ); +} + +function classifyCloudflaredLog(): + | "nemoclaw_no_spawn" + | "nemoclaw_capture_bug" + | "nemoclaw_local" + | "cloudflare" + | "unknown" { + const logPath = getCloudflaredLogPath(); + if (!logPath) return "nemoclaw_no_spawn"; + const log = fs.readFileSync(logPath, "utf8"); + if (TUNNEL_URL_PATTERN.test(log)) return "nemoclaw_capture_bug"; + if ( + /unable to reach the origin|connection refused.*127\.0\.0\.1|connection refused.*localhost|dial tcp.*127\.0\.0\.1.*refused/i.test( + log, + ) + ) { + return "nemoclaw_local"; + } + if (isCloudflareTransientText(log)) return "cloudflare"; + return "unknown"; +} + +function extractTunnelUrl(text: string): string | undefined { + return text.match(TUNNEL_URL_PATTERN)?.[0]; +} + +function parseCurlProbe(result: ShellProbeResult): CurlProbe { + const text = result.stdout; + const match = text.match(/\n__HTTP_CODE:(\d{3})\s*$/); + const httpCode = match?.[1] ?? "000"; + const body = match ? text.slice(0, match.index) : text; + return { httpCode, body, result }; +} + +async function bestEffort(run: () => Promise<unknown>): Promise<void> { + try { + await run(); + } catch { + // Cleanup remains best-effort so the primary E2E failure stays visible. + } +} + +export const TUNNEL_LIFECYCLE_TEST_TIMEOUT_MS = TEST_TIMEOUT_MS; + +type TunnelLifecycleFixtures = Pick<E2EScenarioFixtures, "artifacts" | "cleanup" | "host" | "secrets"> & { + skip: (note?: string) => never; +}; + +export async function runTunnelLifecycleContract({ + artifacts, + cleanup, + host, + secrets, + skip, +}: TunnelLifecycleFixtures): Promise<void> { + assertTestOwnedSandboxName(); + const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); + + await artifacts.writeJson("contract.json", { + legacySource: "test/e2e/test-tunnel-lifecycle.sh", + sandboxName: SANDBOX_NAME, + localDashboardPort: LOCAL_DASHBOARD_PORT, + preservedBoundaries: [ + "real Docker/OpenShell OpenClaw sandbox onboarding", + "host cloudflared binary and quick-tunnel registration", + "nemoclaw tunnel start/status/stop CLI commands", + "local dashboard origin readiness before tunnel attribution", + "public trycloudflare HTTP probe with dashboard marker assertion", + "cloudflared.log classification for NemoClaw-vs-Cloudflare failures", + ], + }); + + cleanup.add("stop cloudflared quick tunnel", async () => { + await bestEffort(() => + host.nemoclaw(["tunnel", "stop"], { + artifactName: "cleanup-tunnel-stop", + env: commandEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }), + ); + }); + cleanup.add(`destroy sandbox ${SANDBOX_NAME}`, async () => { + if (process.env.NEMOCLAW_E2E_KEEP_SANDBOX === "1") return; + await bestEffort(() => + host.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy-tunnel-lifecycle", + timeoutMs: 15 * 60_000, + }), + ); + }); + + const docker = await host.command("docker", ["info"], { + artifactName: "prereq-docker-info-tunnel-lifecycle", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + if (docker.exitCode !== 0) { + if (process.env.GITHUB_ACTIONS === "true") { + throw new Error(`Docker is required for tunnel lifecycle E2E: ${resultText(docker)}`); + } + skip("Docker is required for tunnel lifecycle E2E"); + } + + const cloudflared = await host.command( + "bash", + [ + "-lc", + [ + "set -euo pipefail", + "if command -v cloudflared >/dev/null 2>&1; then", + " cloudflared --version", + " exit 0", + "fi", + 'if [ "${GITHUB_ACTIONS:-}" != "true" ]; then', + ' echo "cloudflared not found" >&2', + " exit 127", + "fi", + "source test/e2e/lib/cloudflared-version-resolver.sh", + "sudo mkdir -p --mode=0755 /usr/share/keyrings", + "curl -fsSL https://pkg.cloudflare.com/cloudflare-main.gpg | sudo tee /usr/share/keyrings/cloudflare-main.gpg >/dev/null", + 'echo "deb [signed-by=/usr/share/keyrings/cloudflare-main.gpg] https://pkg.cloudflare.com/cloudflared $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/cloudflared.list >/dev/null', + "sudo apt-get update -qq", + 'available_versions="$(apt-cache madison cloudflared | awk \'{print $3}\')"', + 'cf_min_version="${CLOUDFLARED_MIN_VERSION:-$CLOUDFLARED_DEFAULT_MIN_VERSION}"', + 'if [ -n "${CLOUDFLARED_VERSION:-}" ]; then', + ' cf_version="$(cloudflared_resolve_package_version "$available_versions" "$cf_min_version" "$CLOUDFLARED_VERSION")"', + "else", + ' cf_version="$(cloudflared_resolve_package_version "$available_versions" "$cf_min_version")"', + "fi", + 'sudo apt-get install -y "cloudflared=${cf_version}"', + "cloudflared --version", + ].join("\n"), + ], + { + artifactName: "prereq-cloudflared-version", + cwd: REPO_ROOT, + env: { + ...buildAvailabilityProbeEnv(), + ...(process.env.CLOUDFLARED_VERSION + ? { CLOUDFLARED_VERSION: process.env.CLOUDFLARED_VERSION } + : {}), + ...(process.env.CLOUDFLARED_MIN_VERSION + ? { CLOUDFLARED_MIN_VERSION: process.env.CLOUDFLARED_MIN_VERSION } + : {}), + }, + timeoutMs: 5 * 60_000, + }, + ); + if (cloudflared.exitCode !== 0) { + if (process.env.GITHUB_ACTIONS === "true") { + throw new Error(`cloudflared is required for tunnel lifecycle E2E: ${resultText(cloudflared)}`); + } + skip("cloudflared is required for tunnel lifecycle E2E"); + } + + expect(fs.existsSync(path.join(REPO_ROOT, "install.sh"))).toBe(true); + await host.bestEffortCleanupSandbox(SANDBOX_NAME, { + artifactName: "pre-cleanup-nemoclaw-destroy-tunnel-lifecycle", + timeoutMs: 15 * 60_000, + }); + + const install = await host.command( + "bash", + ["install.sh", "--non-interactive", "--yes-i-accept-third-party-software"], + { + artifactName: "install-sh-tunnel-lifecycle", + cwd: REPO_ROOT, + env: commandEnv({ NVIDIA_INFERENCE_API_KEY: apiKey }), + redactionValues: [apiKey], + timeoutMs: ONBOARD_TIMEOUT_MS, + }, + ); + expect(install.exitCode, resultText(install)).toBe(0); + + await host.expectListed(SANDBOX_NAME, { artifactName: "post-install-nemoclaw-list" }); + + let localReady = false; + for (let attempt = 1; attempt <= 30; attempt += 1) { + const local = await host.command( + "curl", + ["-sS", "-o", "/dev/null", "-w", "%{http_code}", "--max-time", "5", `http://localhost:${LOCAL_DASHBOARD_PORT}/`], + { + artifactName: `local-dashboard-ready-${attempt}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 10_000, + }, + ); + const code = local.stdout.trim() || "000"; + if (code !== "000") { + localReady = true; + break; + } + await sleep(1_000); + } + expect( + localReady, + `[NemoClaw fault] Local OpenClaw dashboard not reachable on localhost:${LOCAL_DASHBOARD_PORT} after 30s; tunnel cannot proxy a dead origin.`, + ).toBe(true); + + const start = await host.nemoclaw(["tunnel", "start"], { + artifactName: "tunnel-start", + env: commandEnv(), + timeoutMs: 90_000, + }); + if (start.exitCode !== 0) { + await artifacts.writeText("cloudflared-log-after-start-failure.txt", cloudflaredLogTail()); + if (isCloudflareTransientText(resultText(start)) || classifyCloudflaredLog() === "cloudflare") { + await bestEffort(() => + host.nemoclaw(["tunnel", "stop"], { + artifactName: "tunnel-stop-after-cloudflare-start-failure", + env: commandEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }), + ); + skip( + `[Cloudflare fault] nemoclaw tunnel start exited ${start.exitCode ?? "unknown"} because quick-tunnel registration returned a transient external error.`, + ); + } + throw new Error( + `[NemoClaw fault] nemoclaw tunnel start failed with exit ${start.exitCode ?? "unknown"}: ${resultText(start)}`, + ); + } + + let tunnelUrl: string | undefined; + let lastStatusText = ""; + for (let attempt = 1; attempt <= 15; attempt += 1) { + const status = await host.nemoclaw(["status"], { + artifactName: `status-with-tunnel-url-${attempt}`, + env: commandEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }); + lastStatusText = resultText(status); + tunnelUrl = extractTunnelUrl(lastStatusText); + if (tunnelUrl) break; + await sleep(1_000); + } + + if (!tunnelUrl) { + await artifacts.writeText("cloudflared-log-without-status-url.txt", cloudflaredLogTail()); + const cfClass = classifyCloudflaredLog(); + await bestEffort(() => + host.nemoclaw(["tunnel", "stop"], { + artifactName: "tunnel-stop-after-missing-url", + env: commandEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }), + ); + if (cfClass === "cloudflare") { + skip("[Cloudflare fault] cloudflared failed to register a quick tunnel URL."); + } + let reason: string; + switch (cfClass) { + case "nemoclaw_no_spawn": + reason = "cloudflared.log missing — NemoClaw failed to spawn the cloudflared process"; + break; + case "nemoclaw_capture_bug": + reason = "cloudflared.log has a trycloudflare URL but nemoclaw status did not surface it"; + break; + case "nemoclaw_local": + reason = `cloudflared.log reports it cannot reach localhost:${LOCAL_DASHBOARD_PORT}`; + break; + default: + reason = `tunnel URL did not surface and cloudflared.log did not match a known pattern; status was:\n${lastStatusText}`; + } + throw new Error(`[NemoClaw fault] ${reason}`); + } + + let lastPublicProbe: CurlProbe | undefined; + let backoffMs = 2_000; + for (let attempt = 1; attempt <= 15; attempt += 1) { + const probe = parseCurlProbe( + await host.command( + "curl", + ["-sS", "-L", "--max-time", "30", "-w", "\n__HTTP_CODE:%{http_code}\n", tunnelUrl], + { + artifactName: `public-tunnel-probe-${attempt}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 35_000, + }, + ), + ); + lastPublicProbe = probe; + if (probe.httpCode === "200") break; + + const local = await host.command( + "curl", + ["-sS", "-o", "/dev/null", "-w", "%{http_code}", "--max-time", "5", `http://localhost:${LOCAL_DASHBOARD_PORT}/`], + { + artifactName: `local-dashboard-recheck-${attempt}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 10_000, + }, + ); + const localCode = local.stdout.trim() || "000"; + if (localCode === "000") { + throw new Error( + `[NemoClaw fault] Tunnel returned ${probe.httpCode} and local dashboard regressed during retry loop; likely sandbox/dashboard crash, not Cloudflare.`, + ); + } + await sleep(backoffMs); + backoffMs = Math.min(backoffMs * 2, 30_000); + } + + expect(lastPublicProbe, "public tunnel probe should have run").toBeTruthy(); + if (lastPublicProbe!.httpCode !== "200") { + if ( + isCloudflareTransientHttpCode(lastPublicProbe!.httpCode) || + isCloudflareTransientText(lastPublicProbe!.body) || + isCloudflareTransientText(readCloudflaredLog()) + ) { + skip( + `[Cloudflare fault] Tunnel URL never became reachable while local stayed healthy; last HTTP status ${lastPublicProbe!.httpCode}.`, + ); + } + throw new Error( + `[NemoClaw fault] Tunnel returned unexpected HTTP ${lastPublicProbe!.httpCode} while local stayed healthy; body prefix: ${lastPublicProbe!.body.slice(0, 200)}`, + ); + } + expect(lastPublicProbe!.body, "public tunnel must serve OpenClaw dashboard markers").toMatch( + DASHBOARD_MARKER_PATTERN, + ); + + const stop = await host.nemoclaw(["tunnel", "stop"], { + artifactName: "tunnel-stop", + env: commandEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }); + expect(stop.exitCode, resultText(stop)).toBe(0); + + let postStopUrl: string | undefined; + let statusReadable = false; + for (let attempt = 1; attempt <= 10; attempt += 1) { + const status = await host.nemoclaw(["status"], { + artifactName: `status-after-tunnel-stop-${attempt}`, + env: commandEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }); + if (status.exitCode !== 0) { + await sleep(1_000); + continue; + } + statusReadable = true; + postStopUrl = extractTunnelUrl(resultText(status)); + if (!postStopUrl) break; + await sleep(1_000); + } + expect(statusReadable, "nemoclaw status should be readable after tunnel stop").toBe(true); + expect(postStopUrl, "tunnel URL must be absent after nemoclaw tunnel stop").toBeUndefined(); +} diff --git a/test/e2e-scenario/live/tunnel-lifecycle.test.ts b/test/e2e-scenario/live/tunnel-lifecycle.test.ts index 36ce3b656f8..fd257436fa9 100644 --- a/test/e2e-scenario/live/tunnel-lifecycle.test.ts +++ b/test/e2e-scenario/live/tunnel-lifecycle.test.ts @@ -10,441 +10,15 @@ * cleanup/status removal. */ -import fs from "node:fs"; -import path from "node:path"; - -import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; -import { resultText } from "../fixtures/clients/index.ts"; -import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; -import { expect, test } from "../fixtures/e2e-test.ts"; +import { test } from "../fixtures/e2e-test.ts"; import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; -import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; - -const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); -const TEST_SANDBOX_PREFIX = "e2e-tunnel-lifecycle"; -const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? TEST_SANDBOX_PREFIX; -const LOCAL_DASHBOARD_PORT = process.env.NEMOCLAW_DASHBOARD_PORT ?? "18789"; -const TEST_TIMEOUT_MS = Number(process.env.NEMOCLAW_E2E_TIMEOUT_SECONDS ?? 3_600) * 1_000; -const ONBOARD_TIMEOUT_MS = 30 * 60_000; -const COMMAND_TIMEOUT_MS = 60_000; -const TUNNEL_URL_PATTERN = /https:\/\/[a-z0-9-]+\.trycloudflare\.com\b[\w./?%&=-]*/i; -const DASHBOARD_MARKER_PATTERN = /<title>OpenClaw Control<\/title>|<openclaw-app/i; - -validateSandboxName(SANDBOX_NAME); - -type CurlProbe = { - httpCode: string; - body: string; - result: ShellProbeResult; -}; - -function assertTestOwnedSandboxName(): void { - if (!SANDBOX_NAME.startsWith(TEST_SANDBOX_PREFIX)) { - throw new Error( - `tunnel-lifecycle live test is destructive and only accepts sandbox names with prefix ${TEST_SANDBOX_PREFIX}; got ${SANDBOX_NAME}`, - ); - } -} - -function sleep(ms: number): Promise<void> { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { - return { - ...buildAvailabilityProbeEnv(), - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", - NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, - NEMOCLAW_POLICY_TIER: "open", - NEMOCLAW_AGENT: "openclaw", - NEMOCLAW_PROVIDER: "cloud", - OPENSHELL_GATEWAY: "nemoclaw", - ...(process.env.NEMOCLAW_DASHBOARD_PORT - ? { NEMOCLAW_DASHBOARD_PORT: process.env.NEMOCLAW_DASHBOARD_PORT } - : {}), - ...extra, - }; -} - -function isCloudflareTransientText(text: string): boolean { - return /failed to unmarshal quick Tunnel|quick tunnels? (are )?(temporarily )?disabled|failed to (dial|register)|tunnel server.*error|i\/o timeout|EOF.*tunnel|couldn.?t start tunnel|tunnel creation failed|bad gateway|\b50[234]\b/i.test( - text, - ); -} - -function isCloudflareTransientHttpCode(code: string): boolean { - return ["000", "502", "503", "504"].includes(code); -} - -function getCloudflaredLogPath(): string | undefined { - const sandboxLog = path.join("/tmp", `nemoclaw-services-${SANDBOX_NAME}`, "cloudflared.log"); - if (fs.existsSync(sandboxLog)) return sandboxLog; - let candidates: string[] = []; - try { - candidates = fs - .readdirSync("/tmp", { withFileTypes: true }) - .filter((entry) => entry.isDirectory() && entry.name.startsWith("nemoclaw-services-")) - .map((entry) => path.join("/tmp", entry.name, "cloudflared.log")) - .filter((candidate) => fs.existsSync(candidate)); - } catch { - return undefined; - } - return candidates.sort((a, b) => fs.statSync(b).mtimeMs - fs.statSync(a).mtimeMs).at(0); -} - -function readCloudflaredLog(): string { - const logPath = getCloudflaredLogPath(); - if (!logPath) return ""; - return fs.readFileSync(logPath, "utf8"); -} - -function cloudflaredLogTail(lines = 80): string { - const logPath = getCloudflaredLogPath(); - if (!logPath) return "(no cloudflared.log found under /tmp/nemoclaw-services-*/)"; - const text = fs.readFileSync(logPath, "utf8"); - return [`--- cloudflared.log (${logPath}, last ${lines} lines) ---`, ...text.split(/\r?\n/).slice(-lines)].join( - "\n", - ); -} - -function classifyCloudflaredLog(): - | "nemoclaw_no_spawn" - | "nemoclaw_capture_bug" - | "nemoclaw_local" - | "cloudflare" - | "unknown" { - const logPath = getCloudflaredLogPath(); - if (!logPath) return "nemoclaw_no_spawn"; - const log = fs.readFileSync(logPath, "utf8"); - if (TUNNEL_URL_PATTERN.test(log)) return "nemoclaw_capture_bug"; - if ( - /unable to reach the origin|connection refused.*127\.0\.0\.1|connection refused.*localhost|dial tcp.*127\.0\.0\.1.*refused/i.test( - log, - ) - ) { - return "nemoclaw_local"; - } - if (isCloudflareTransientText(log)) return "cloudflare"; - return "unknown"; -} - -function extractTunnelUrl(text: string): string | undefined { - return text.match(TUNNEL_URL_PATTERN)?.[0]; -} - -function parseCurlProbe(result: ShellProbeResult): CurlProbe { - const text = result.stdout; - const match = text.match(/\n__HTTP_CODE:(\d{3})\s*$/); - const httpCode = match?.[1] ?? "000"; - const body = match ? text.slice(0, match.index) : text; - return { httpCode, body, result }; -} - -async function bestEffort(run: () => Promise<unknown>): Promise<void> { - try { - await run(); - } catch { - // Cleanup remains best-effort so the primary E2E failure stays visible. - } -} +import { + runTunnelLifecycleContract, + TUNNEL_LIFECYCLE_TEST_TIMEOUT_MS, +} from "./tunnel-lifecycle-helpers.ts"; test.skipIf(!shouldRunLiveE2EScenarios())( "tunnel-lifecycle: cloudflared quick tunnel starts, serves OpenClaw, and stops cleanly", - { timeout: TEST_TIMEOUT_MS }, - async ({ artifacts, cleanup, host, secrets, skip }) => { - assertTestOwnedSandboxName(); - const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); - - await artifacts.writeJson("contract.json", { - legacySource: "test/e2e/test-tunnel-lifecycle.sh", - sandboxName: SANDBOX_NAME, - localDashboardPort: LOCAL_DASHBOARD_PORT, - preservedBoundaries: [ - "real Docker/OpenShell OpenClaw sandbox onboarding", - "host cloudflared binary and quick-tunnel registration", - "nemoclaw tunnel start/status/stop CLI commands", - "local dashboard origin readiness before tunnel attribution", - "public trycloudflare HTTP probe with dashboard marker assertion", - "cloudflared.log classification for NemoClaw-vs-Cloudflare failures", - ], - }); - - cleanup.add("stop cloudflared quick tunnel", async () => { - await bestEffort(() => - host.nemoclaw(["tunnel", "stop"], { - artifactName: "cleanup-tunnel-stop", - env: commandEnv(), - timeoutMs: COMMAND_TIMEOUT_MS, - }), - ); - }); - cleanup.add(`destroy sandbox ${SANDBOX_NAME}`, async () => { - if (process.env.NEMOCLAW_E2E_KEEP_SANDBOX === "1") return; - await bestEffort(() => - host.cleanupSandbox(SANDBOX_NAME, { - artifactName: "cleanup-nemoclaw-destroy-tunnel-lifecycle", - timeoutMs: 15 * 60_000, - }), - ); - }); - - const docker = await host.command("docker", ["info"], { - artifactName: "prereq-docker-info-tunnel-lifecycle", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, - }); - if (docker.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error(`Docker is required for tunnel lifecycle E2E: ${resultText(docker)}`); - } - skip("Docker is required for tunnel lifecycle E2E"); - } - - const cloudflared = await host.command( - "bash", - [ - "-lc", - [ - "set -euo pipefail", - "if command -v cloudflared >/dev/null 2>&1; then", - " cloudflared --version", - " exit 0", - "fi", - 'if [ "${GITHUB_ACTIONS:-}" != "true" ]; then', - ' echo "cloudflared not found" >&2', - " exit 127", - "fi", - "source test/e2e/lib/cloudflared-version-resolver.sh", - "sudo mkdir -p --mode=0755 /usr/share/keyrings", - "curl -fsSL https://pkg.cloudflare.com/cloudflare-main.gpg | sudo tee /usr/share/keyrings/cloudflare-main.gpg >/dev/null", - 'echo "deb [signed-by=/usr/share/keyrings/cloudflare-main.gpg] https://pkg.cloudflare.com/cloudflared $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/cloudflared.list >/dev/null', - "sudo apt-get update -qq", - 'available_versions="$(apt-cache madison cloudflared | awk \'{print $3}\')"', - 'cf_min_version="${CLOUDFLARED_MIN_VERSION:-$CLOUDFLARED_DEFAULT_MIN_VERSION}"', - 'if [ -n "${CLOUDFLARED_VERSION:-}" ]; then', - ' cf_version="$(cloudflared_resolve_package_version "$available_versions" "$cf_min_version" "$CLOUDFLARED_VERSION")"', - "else", - ' cf_version="$(cloudflared_resolve_package_version "$available_versions" "$cf_min_version")"', - "fi", - 'sudo apt-get install -y "cloudflared=${cf_version}"', - "cloudflared --version", - ].join("\n"), - ], - { - artifactName: "prereq-cloudflared-version", - cwd: REPO_ROOT, - env: { - ...buildAvailabilityProbeEnv(), - ...(process.env.CLOUDFLARED_VERSION - ? { CLOUDFLARED_VERSION: process.env.CLOUDFLARED_VERSION } - : {}), - ...(process.env.CLOUDFLARED_MIN_VERSION - ? { CLOUDFLARED_MIN_VERSION: process.env.CLOUDFLARED_MIN_VERSION } - : {}), - }, - timeoutMs: 5 * 60_000, - }, - ); - if (cloudflared.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error(`cloudflared is required for tunnel lifecycle E2E: ${resultText(cloudflared)}`); - } - skip("cloudflared is required for tunnel lifecycle E2E"); - } - - expect(fs.existsSync(path.join(REPO_ROOT, "install.sh"))).toBe(true); - await host.bestEffortCleanupSandbox(SANDBOX_NAME, { - artifactName: "pre-cleanup-nemoclaw-destroy-tunnel-lifecycle", - timeoutMs: 15 * 60_000, - }); - - const install = await host.command( - "bash", - ["install.sh", "--non-interactive", "--yes-i-accept-third-party-software"], - { - artifactName: "install-sh-tunnel-lifecycle", - cwd: REPO_ROOT, - env: commandEnv({ NVIDIA_INFERENCE_API_KEY: apiKey }), - redactionValues: [apiKey], - timeoutMs: ONBOARD_TIMEOUT_MS, - }, - ); - expect(install.exitCode, resultText(install)).toBe(0); - - await host.expectListed(SANDBOX_NAME, { artifactName: "post-install-nemoclaw-list" }); - - let localReady = false; - for (let attempt = 1; attempt <= 30; attempt += 1) { - const local = await host.command( - "curl", - ["-sS", "-o", "/dev/null", "-w", "%{http_code}", "--max-time", "5", `http://localhost:${LOCAL_DASHBOARD_PORT}/`], - { - artifactName: `local-dashboard-ready-${attempt}`, - env: buildAvailabilityProbeEnv(), - timeoutMs: 10_000, - }, - ); - const code = local.stdout.trim() || "000"; - if (code !== "000") { - localReady = true; - break; - } - await sleep(1_000); - } - expect( - localReady, - `[NemoClaw fault] Local OpenClaw dashboard not reachable on localhost:${LOCAL_DASHBOARD_PORT} after 30s; tunnel cannot proxy a dead origin.`, - ).toBe(true); - - const start = await host.nemoclaw(["tunnel", "start"], { - artifactName: "tunnel-start", - env: commandEnv(), - timeoutMs: 90_000, - }); - if (start.exitCode !== 0) { - await artifacts.writeText("cloudflared-log-after-start-failure.txt", cloudflaredLogTail()); - if (isCloudflareTransientText(resultText(start)) || classifyCloudflaredLog() === "cloudflare") { - await bestEffort(() => - host.nemoclaw(["tunnel", "stop"], { - artifactName: "tunnel-stop-after-cloudflare-start-failure", - env: commandEnv(), - timeoutMs: COMMAND_TIMEOUT_MS, - }), - ); - skip( - `[Cloudflare fault] nemoclaw tunnel start exited ${start.exitCode ?? "unknown"} because quick-tunnel registration returned a transient external error.`, - ); - } - throw new Error( - `[NemoClaw fault] nemoclaw tunnel start failed with exit ${start.exitCode ?? "unknown"}: ${resultText(start)}`, - ); - } - - let tunnelUrl: string | undefined; - let lastStatusText = ""; - for (let attempt = 1; attempt <= 15; attempt += 1) { - const status = await host.nemoclaw(["status"], { - artifactName: `status-with-tunnel-url-${attempt}`, - env: commandEnv(), - timeoutMs: COMMAND_TIMEOUT_MS, - }); - lastStatusText = resultText(status); - tunnelUrl = extractTunnelUrl(lastStatusText); - if (tunnelUrl) break; - await sleep(1_000); - } - - if (!tunnelUrl) { - await artifacts.writeText("cloudflared-log-without-status-url.txt", cloudflaredLogTail()); - const cfClass = classifyCloudflaredLog(); - await bestEffort(() => - host.nemoclaw(["tunnel", "stop"], { - artifactName: "tunnel-stop-after-missing-url", - env: commandEnv(), - timeoutMs: COMMAND_TIMEOUT_MS, - }), - ); - if (cfClass === "cloudflare") { - skip("[Cloudflare fault] cloudflared failed to register a quick tunnel URL."); - } - let reason: string; - switch (cfClass) { - case "nemoclaw_no_spawn": - reason = "cloudflared.log missing — NemoClaw failed to spawn the cloudflared process"; - break; - case "nemoclaw_capture_bug": - reason = "cloudflared.log has a trycloudflare URL but nemoclaw status did not surface it"; - break; - case "nemoclaw_local": - reason = `cloudflared.log reports it cannot reach localhost:${LOCAL_DASHBOARD_PORT}`; - break; - default: - reason = `tunnel URL did not surface and cloudflared.log did not match a known pattern; status was:\n${lastStatusText}`; - } - throw new Error(`[NemoClaw fault] ${reason}`); - } - - let lastPublicProbe: CurlProbe | undefined; - let backoffMs = 2_000; - for (let attempt = 1; attempt <= 15; attempt += 1) { - const probe = parseCurlProbe( - await host.command( - "curl", - ["-sS", "-L", "--max-time", "30", "-w", "\n__HTTP_CODE:%{http_code}\n", tunnelUrl], - { - artifactName: `public-tunnel-probe-${attempt}`, - env: buildAvailabilityProbeEnv(), - timeoutMs: 35_000, - }, - ), - ); - lastPublicProbe = probe; - if (probe.httpCode === "200") break; - - const local = await host.command( - "curl", - ["-sS", "-o", "/dev/null", "-w", "%{http_code}", "--max-time", "5", `http://localhost:${LOCAL_DASHBOARD_PORT}/`], - { - artifactName: `local-dashboard-recheck-${attempt}`, - env: buildAvailabilityProbeEnv(), - timeoutMs: 10_000, - }, - ); - const localCode = local.stdout.trim() || "000"; - if (localCode === "000") { - throw new Error( - `[NemoClaw fault] Tunnel returned ${probe.httpCode} and local dashboard regressed during retry loop; likely sandbox/dashboard crash, not Cloudflare.`, - ); - } - await sleep(backoffMs); - backoffMs = Math.min(backoffMs * 2, 30_000); - } - - expect(lastPublicProbe, "public tunnel probe should have run").toBeTruthy(); - if (lastPublicProbe!.httpCode !== "200") { - if ( - isCloudflareTransientHttpCode(lastPublicProbe!.httpCode) || - isCloudflareTransientText(lastPublicProbe!.body) || - isCloudflareTransientText(readCloudflaredLog()) - ) { - skip( - `[Cloudflare fault] Tunnel URL never became reachable while local stayed healthy; last HTTP status ${lastPublicProbe!.httpCode}.`, - ); - } - throw new Error( - `[NemoClaw fault] Tunnel returned unexpected HTTP ${lastPublicProbe!.httpCode} while local stayed healthy; body prefix: ${lastPublicProbe!.body.slice(0, 200)}`, - ); - } - expect(lastPublicProbe!.body, "public tunnel must serve OpenClaw dashboard markers").toMatch( - DASHBOARD_MARKER_PATTERN, - ); - - const stop = await host.nemoclaw(["tunnel", "stop"], { - artifactName: "tunnel-stop", - env: commandEnv(), - timeoutMs: COMMAND_TIMEOUT_MS, - }); - expect(stop.exitCode, resultText(stop)).toBe(0); - - let postStopUrl: string | undefined; - let statusReadable = false; - for (let attempt = 1; attempt <= 10; attempt += 1) { - const status = await host.nemoclaw(["status"], { - artifactName: `status-after-tunnel-stop-${attempt}`, - env: commandEnv(), - timeoutMs: COMMAND_TIMEOUT_MS, - }); - if (status.exitCode !== 0) { - await sleep(1_000); - continue; - } - statusReadable = true; - postStopUrl = extractTunnelUrl(resultText(status)); - if (!postStopUrl) break; - await sleep(1_000); - } - expect(statusReadable, "nemoclaw status should be readable after tunnel stop").toBe(true); - expect(postStopUrl, "tunnel URL must be absent after nemoclaw tunnel stop").toBeUndefined(); - }, + { timeout: TUNNEL_LIFECYCLE_TEST_TIMEOUT_MS }, + runTunnelLifecycleContract, ); From 7cd6b223045a83c55079a3beeb3e41a54e79c458 Mon Sep 17 00:00:00 2001 From: Julie Yaunches <jyaunches@nvidia.com> Date: Mon, 22 Jun 2026 10:07:17 -0400 Subject: [PATCH 03/15] test(e2e): use hosted inference for tunnel lifecycle --- test/e2e-scenario/live/tunnel-lifecycle-helpers.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts b/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts index a18715b72ca..52c937541a1 100644 --- a/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts +++ b/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts @@ -18,6 +18,7 @@ import { resultText } from "../fixtures/clients/index.ts"; import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; import type { E2EScenarioFixtures } from "../fixtures/e2e-test.ts"; import { expect } from "../fixtures/e2e-test.ts"; +import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); @@ -163,7 +164,8 @@ export async function runTunnelLifecycleContract({ skip, }: TunnelLifecycleFixtures): Promise<void> { assertTestOwnedSandboxName(); - const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); + const hosted = requireHostedInferenceConfig(secrets); + const apiKey = hosted.apiKey; await artifacts.writeJson("contract.json", { legacySource: "test/e2e/test-tunnel-lifecycle.sh", @@ -177,6 +179,7 @@ export async function runTunnelLifecycleContract({ "public trycloudflare HTTP probe with dashboard marker assertion", "cloudflared.log classification for NemoClaw-vs-Cloudflare failures", ], + inferenceCredential: hosted.contractLabel, }); cleanup.add("stop cloudflared quick tunnel", async () => { @@ -274,7 +277,11 @@ export async function runTunnelLifecycleContract({ { artifactName: "install-sh-tunnel-lifecycle", cwd: REPO_ROOT, - env: commandEnv({ NVIDIA_INFERENCE_API_KEY: apiKey }), + env: commandEnv({ + ...hosted.env, + NVIDIA_INFERENCE_API_KEY: apiKey, + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1", + }), redactionValues: [apiKey], timeoutMs: ONBOARD_TIMEOUT_MS, }, From 7b1ffdf8a5f1534fcad8c5198cdabad62391eb26 Mon Sep 17 00:00:00 2001 From: Julie Yaunches <jyaunches@nvidia.com> Date: Mon, 22 Jun 2026 10:14:45 -0400 Subject: [PATCH 04/15] style: format tunnel lifecycle helper --- .../live/tunnel-lifecycle-helpers.ts | 554 +++++++++--------- 1 file changed, 289 insertions(+), 265 deletions(-) diff --git a/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts b/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts index 52c937541a1..0e47fdaba52 100644 --- a/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts +++ b/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts @@ -104,9 +104,10 @@ function cloudflaredLogTail(lines = 80): string { const logPath = getCloudflaredLogPath(); if (!logPath) return "(no cloudflared.log found under /tmp/nemoclaw-services-*/)"; const text = fs.readFileSync(logPath, "utf8"); - return [`--- cloudflared.log (${logPath}, last ${lines} lines) ---`, ...text.split(/\r?\n/).slice(-lines)].join( - "\n", - ); + return [ + `--- cloudflared.log (${logPath}, last ${lines} lines) ---`, + ...text.split(/\r?\n/).slice(-lines), + ].join("\n"); } function classifyCloudflaredLog(): @@ -152,7 +153,10 @@ async function bestEffort(run: () => Promise<unknown>): Promise<void> { export const TUNNEL_LIFECYCLE_TEST_TIMEOUT_MS = TEST_TIMEOUT_MS; -type TunnelLifecycleFixtures = Pick<E2EScenarioFixtures, "artifacts" | "cleanup" | "host" | "secrets"> & { +type TunnelLifecycleFixtures = Pick< + E2EScenarioFixtures, + "artifacts" | "cleanup" | "host" | "secrets" +> & { skip: (note?: string) => never; }; @@ -163,303 +167,323 @@ export async function runTunnelLifecycleContract({ secrets, skip, }: TunnelLifecycleFixtures): Promise<void> { - assertTestOwnedSandboxName(); - const hosted = requireHostedInferenceConfig(secrets); - const apiKey = hosted.apiKey; - - await artifacts.writeJson("contract.json", { - legacySource: "test/e2e/test-tunnel-lifecycle.sh", - sandboxName: SANDBOX_NAME, - localDashboardPort: LOCAL_DASHBOARD_PORT, - preservedBoundaries: [ - "real Docker/OpenShell OpenClaw sandbox onboarding", - "host cloudflared binary and quick-tunnel registration", - "nemoclaw tunnel start/status/stop CLI commands", - "local dashboard origin readiness before tunnel attribution", - "public trycloudflare HTTP probe with dashboard marker assertion", - "cloudflared.log classification for NemoClaw-vs-Cloudflare failures", - ], - inferenceCredential: hosted.contractLabel, - }); + assertTestOwnedSandboxName(); + const hosted = requireHostedInferenceConfig(secrets); + const apiKey = hosted.apiKey; + + await artifacts.writeJson("contract.json", { + legacySource: "test/e2e/test-tunnel-lifecycle.sh", + sandboxName: SANDBOX_NAME, + localDashboardPort: LOCAL_DASHBOARD_PORT, + preservedBoundaries: [ + "real Docker/OpenShell OpenClaw sandbox onboarding", + "host cloudflared binary and quick-tunnel registration", + "nemoclaw tunnel start/status/stop CLI commands", + "local dashboard origin readiness before tunnel attribution", + "public trycloudflare HTTP probe with dashboard marker assertion", + "cloudflared.log classification for NemoClaw-vs-Cloudflare failures", + ], + inferenceCredential: hosted.contractLabel, + }); + + cleanup.add("stop cloudflared quick tunnel", async () => { + await bestEffort(() => + host.nemoclaw(["tunnel", "stop"], { + artifactName: "cleanup-tunnel-stop", + env: commandEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }), + ); + }); + cleanup.add(`destroy sandbox ${SANDBOX_NAME}`, async () => { + if (process.env.NEMOCLAW_E2E_KEEP_SANDBOX === "1") return; + await bestEffort(() => + host.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy-tunnel-lifecycle", + timeoutMs: 15 * 60_000, + }), + ); + }); + + const docker = await host.command("docker", ["info"], { + artifactName: "prereq-docker-info-tunnel-lifecycle", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + if (docker.exitCode !== 0) { + if (process.env.GITHUB_ACTIONS === "true") { + throw new Error(`Docker is required for tunnel lifecycle E2E: ${resultText(docker)}`); + } + skip("Docker is required for tunnel lifecycle E2E"); + } - cleanup.add("stop cloudflared quick tunnel", async () => { - await bestEffort(() => - host.nemoclaw(["tunnel", "stop"], { - artifactName: "cleanup-tunnel-stop", - env: commandEnv(), - timeoutMs: COMMAND_TIMEOUT_MS, - }), - ); - }); - cleanup.add(`destroy sandbox ${SANDBOX_NAME}`, async () => { - if (process.env.NEMOCLAW_E2E_KEEP_SANDBOX === "1") return; - await bestEffort(() => - host.cleanupSandbox(SANDBOX_NAME, { - artifactName: "cleanup-nemoclaw-destroy-tunnel-lifecycle", - timeoutMs: 15 * 60_000, - }), + const cloudflared = await host.command( + "bash", + [ + "-lc", + [ + "set -euo pipefail", + "if command -v cloudflared >/dev/null 2>&1; then", + " cloudflared --version", + " exit 0", + "fi", + 'if [ "${GITHUB_ACTIONS:-}" != "true" ]; then', + ' echo "cloudflared not found" >&2', + " exit 127", + "fi", + "source test/e2e/lib/cloudflared-version-resolver.sh", + "sudo mkdir -p --mode=0755 /usr/share/keyrings", + "curl -fsSL https://pkg.cloudflare.com/cloudflare-main.gpg | sudo tee /usr/share/keyrings/cloudflare-main.gpg >/dev/null", + 'echo "deb [signed-by=/usr/share/keyrings/cloudflare-main.gpg] https://pkg.cloudflare.com/cloudflared $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/cloudflared.list >/dev/null', + "sudo apt-get update -qq", + "available_versions=\"$(apt-cache madison cloudflared | awk '{print $3}')\"", + 'cf_min_version="${CLOUDFLARED_MIN_VERSION:-$CLOUDFLARED_DEFAULT_MIN_VERSION}"', + 'if [ -n "${CLOUDFLARED_VERSION:-}" ]; then', + ' cf_version="$(cloudflared_resolve_package_version "$available_versions" "$cf_min_version" "$CLOUDFLARED_VERSION")"', + "else", + ' cf_version="$(cloudflared_resolve_package_version "$available_versions" "$cf_min_version")"', + "fi", + 'sudo apt-get install -y "cloudflared=${cf_version}"', + "cloudflared --version", + ].join("\n"), + ], + { + artifactName: "prereq-cloudflared-version", + cwd: REPO_ROOT, + env: { + ...buildAvailabilityProbeEnv(), + ...(process.env.CLOUDFLARED_VERSION + ? { CLOUDFLARED_VERSION: process.env.CLOUDFLARED_VERSION } + : {}), + ...(process.env.CLOUDFLARED_MIN_VERSION + ? { CLOUDFLARED_MIN_VERSION: process.env.CLOUDFLARED_MIN_VERSION } + : {}), + }, + timeoutMs: 5 * 60_000, + }, + ); + if (cloudflared.exitCode !== 0) { + if (process.env.GITHUB_ACTIONS === "true") { + throw new Error( + `cloudflared is required for tunnel lifecycle E2E: ${resultText(cloudflared)}`, ); - }); - - const docker = await host.command("docker", ["info"], { - artifactName: "prereq-docker-info-tunnel-lifecycle", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, - }); - if (docker.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error(`Docker is required for tunnel lifecycle E2E: ${resultText(docker)}`); - } - skip("Docker is required for tunnel lifecycle E2E"); } + skip("cloudflared is required for tunnel lifecycle E2E"); + } + + expect(fs.existsSync(path.join(REPO_ROOT, "install.sh"))).toBe(true); + await host.bestEffortCleanupSandbox(SANDBOX_NAME, { + artifactName: "pre-cleanup-nemoclaw-destroy-tunnel-lifecycle", + timeoutMs: 15 * 60_000, + }); + + const install = await host.command( + "bash", + ["install.sh", "--non-interactive", "--yes-i-accept-third-party-software"], + { + artifactName: "install-sh-tunnel-lifecycle", + cwd: REPO_ROOT, + env: commandEnv({ + ...hosted.env, + NVIDIA_INFERENCE_API_KEY: apiKey, + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1", + }), + redactionValues: [apiKey], + timeoutMs: ONBOARD_TIMEOUT_MS, + }, + ); + expect(install.exitCode, resultText(install)).toBe(0); + + await host.expectListed(SANDBOX_NAME, { artifactName: "post-install-nemoclaw-list" }); - const cloudflared = await host.command( - "bash", + let localReady = false; + for (let attempt = 1; attempt <= 30; attempt += 1) { + const local = await host.command( + "curl", [ - "-lc", - [ - "set -euo pipefail", - "if command -v cloudflared >/dev/null 2>&1; then", - " cloudflared --version", - " exit 0", - "fi", - 'if [ "${GITHUB_ACTIONS:-}" != "true" ]; then', - ' echo "cloudflared not found" >&2', - " exit 127", - "fi", - "source test/e2e/lib/cloudflared-version-resolver.sh", - "sudo mkdir -p --mode=0755 /usr/share/keyrings", - "curl -fsSL https://pkg.cloudflare.com/cloudflare-main.gpg | sudo tee /usr/share/keyrings/cloudflare-main.gpg >/dev/null", - 'echo "deb [signed-by=/usr/share/keyrings/cloudflare-main.gpg] https://pkg.cloudflare.com/cloudflared $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/cloudflared.list >/dev/null', - "sudo apt-get update -qq", - 'available_versions="$(apt-cache madison cloudflared | awk \'{print $3}\')"', - 'cf_min_version="${CLOUDFLARED_MIN_VERSION:-$CLOUDFLARED_DEFAULT_MIN_VERSION}"', - 'if [ -n "${CLOUDFLARED_VERSION:-}" ]; then', - ' cf_version="$(cloudflared_resolve_package_version "$available_versions" "$cf_min_version" "$CLOUDFLARED_VERSION")"', - "else", - ' cf_version="$(cloudflared_resolve_package_version "$available_versions" "$cf_min_version")"', - "fi", - 'sudo apt-get install -y "cloudflared=${cf_version}"', - "cloudflared --version", - ].join("\n"), + "-sS", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "--max-time", + "5", + `http://localhost:${LOCAL_DASHBOARD_PORT}/`, ], { - artifactName: "prereq-cloudflared-version", - cwd: REPO_ROOT, - env: { - ...buildAvailabilityProbeEnv(), - ...(process.env.CLOUDFLARED_VERSION - ? { CLOUDFLARED_VERSION: process.env.CLOUDFLARED_VERSION } - : {}), - ...(process.env.CLOUDFLARED_MIN_VERSION - ? { CLOUDFLARED_MIN_VERSION: process.env.CLOUDFLARED_MIN_VERSION } - : {}), - }, - timeoutMs: 5 * 60_000, + artifactName: `local-dashboard-ready-${attempt}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 10_000, }, ); - if (cloudflared.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error(`cloudflared is required for tunnel lifecycle E2E: ${resultText(cloudflared)}`); - } - skip("cloudflared is required for tunnel lifecycle E2E"); + const code = local.stdout.trim() || "000"; + if (code !== "000") { + localReady = true; + break; } - - expect(fs.existsSync(path.join(REPO_ROOT, "install.sh"))).toBe(true); - await host.bestEffortCleanupSandbox(SANDBOX_NAME, { - artifactName: "pre-cleanup-nemoclaw-destroy-tunnel-lifecycle", - timeoutMs: 15 * 60_000, - }); - - const install = await host.command( - "bash", - ["install.sh", "--non-interactive", "--yes-i-accept-third-party-software"], - { - artifactName: "install-sh-tunnel-lifecycle", - cwd: REPO_ROOT, - env: commandEnv({ - ...hosted.env, - NVIDIA_INFERENCE_API_KEY: apiKey, - NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1", + await sleep(1_000); + } + expect( + localReady, + `[NemoClaw fault] Local OpenClaw dashboard not reachable on localhost:${LOCAL_DASHBOARD_PORT} after 30s; tunnel cannot proxy a dead origin.`, + ).toBe(true); + + const start = await host.nemoclaw(["tunnel", "start"], { + artifactName: "tunnel-start", + env: commandEnv(), + timeoutMs: 90_000, + }); + if (start.exitCode !== 0) { + await artifacts.writeText("cloudflared-log-after-start-failure.txt", cloudflaredLogTail()); + if (isCloudflareTransientText(resultText(start)) || classifyCloudflaredLog() === "cloudflare") { + await bestEffort(() => + host.nemoclaw(["tunnel", "stop"], { + artifactName: "tunnel-stop-after-cloudflare-start-failure", + env: commandEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, }), - redactionValues: [apiKey], - timeoutMs: ONBOARD_TIMEOUT_MS, - }, - ); - expect(install.exitCode, resultText(install)).toBe(0); - - await host.expectListed(SANDBOX_NAME, { artifactName: "post-install-nemoclaw-list" }); - - let localReady = false; - for (let attempt = 1; attempt <= 30; attempt += 1) { - const local = await host.command( - "curl", - ["-sS", "-o", "/dev/null", "-w", "%{http_code}", "--max-time", "5", `http://localhost:${LOCAL_DASHBOARD_PORT}/`], - { - artifactName: `local-dashboard-ready-${attempt}`, - env: buildAvailabilityProbeEnv(), - timeoutMs: 10_000, - }, ); - const code = local.stdout.trim() || "000"; - if (code !== "000") { - localReady = true; - break; - } - await sleep(1_000); + skip( + `[Cloudflare fault] nemoclaw tunnel start exited ${start.exitCode ?? "unknown"} because quick-tunnel registration returned a transient external error.`, + ); } - expect( - localReady, - `[NemoClaw fault] Local OpenClaw dashboard not reachable on localhost:${LOCAL_DASHBOARD_PORT} after 30s; tunnel cannot proxy a dead origin.`, - ).toBe(true); + throw new Error( + `[NemoClaw fault] nemoclaw tunnel start failed with exit ${start.exitCode ?? "unknown"}: ${resultText(start)}`, + ); + } - const start = await host.nemoclaw(["tunnel", "start"], { - artifactName: "tunnel-start", + let tunnelUrl: string | undefined; + let lastStatusText = ""; + for (let attempt = 1; attempt <= 15; attempt += 1) { + const status = await host.nemoclaw(["status"], { + artifactName: `status-with-tunnel-url-${attempt}`, env: commandEnv(), - timeoutMs: 90_000, + timeoutMs: COMMAND_TIMEOUT_MS, }); - if (start.exitCode !== 0) { - await artifacts.writeText("cloudflared-log-after-start-failure.txt", cloudflaredLogTail()); - if (isCloudflareTransientText(resultText(start)) || classifyCloudflaredLog() === "cloudflare") { - await bestEffort(() => - host.nemoclaw(["tunnel", "stop"], { - artifactName: "tunnel-stop-after-cloudflare-start-failure", - env: commandEnv(), - timeoutMs: COMMAND_TIMEOUT_MS, - }), - ); - skip( - `[Cloudflare fault] nemoclaw tunnel start exited ${start.exitCode ?? "unknown"} because quick-tunnel registration returned a transient external error.`, - ); - } - throw new Error( - `[NemoClaw fault] nemoclaw tunnel start failed with exit ${start.exitCode ?? "unknown"}: ${resultText(start)}`, - ); - } + lastStatusText = resultText(status); + tunnelUrl = extractTunnelUrl(lastStatusText); + if (tunnelUrl) break; + await sleep(1_000); + } - let tunnelUrl: string | undefined; - let lastStatusText = ""; - for (let attempt = 1; attempt <= 15; attempt += 1) { - const status = await host.nemoclaw(["status"], { - artifactName: `status-with-tunnel-url-${attempt}`, + if (!tunnelUrl) { + await artifacts.writeText("cloudflared-log-without-status-url.txt", cloudflaredLogTail()); + const cfClass = classifyCloudflaredLog(); + await bestEffort(() => + host.nemoclaw(["tunnel", "stop"], { + artifactName: "tunnel-stop-after-missing-url", env: commandEnv(), timeoutMs: COMMAND_TIMEOUT_MS, - }); - lastStatusText = resultText(status); - tunnelUrl = extractTunnelUrl(lastStatusText); - if (tunnelUrl) break; - await sleep(1_000); + }), + ); + if (cfClass === "cloudflare") { + skip("[Cloudflare fault] cloudflared failed to register a quick tunnel URL."); } - - if (!tunnelUrl) { - await artifacts.writeText("cloudflared-log-without-status-url.txt", cloudflaredLogTail()); - const cfClass = classifyCloudflaredLog(); - await bestEffort(() => - host.nemoclaw(["tunnel", "stop"], { - artifactName: "tunnel-stop-after-missing-url", - env: commandEnv(), - timeoutMs: COMMAND_TIMEOUT_MS, - }), - ); - if (cfClass === "cloudflare") { - skip("[Cloudflare fault] cloudflared failed to register a quick tunnel URL."); - } - let reason: string; - switch (cfClass) { - case "nemoclaw_no_spawn": - reason = "cloudflared.log missing — NemoClaw failed to spawn the cloudflared process"; - break; - case "nemoclaw_capture_bug": - reason = "cloudflared.log has a trycloudflare URL but nemoclaw status did not surface it"; - break; - case "nemoclaw_local": - reason = `cloudflared.log reports it cannot reach localhost:${LOCAL_DASHBOARD_PORT}`; - break; - default: - reason = `tunnel URL did not surface and cloudflared.log did not match a known pattern; status was:\n${lastStatusText}`; - } - throw new Error(`[NemoClaw fault] ${reason}`); + let reason: string; + switch (cfClass) { + case "nemoclaw_no_spawn": + reason = "cloudflared.log missing — NemoClaw failed to spawn the cloudflared process"; + break; + case "nemoclaw_capture_bug": + reason = "cloudflared.log has a trycloudflare URL but nemoclaw status did not surface it"; + break; + case "nemoclaw_local": + reason = `cloudflared.log reports it cannot reach localhost:${LOCAL_DASHBOARD_PORT}`; + break; + default: + reason = `tunnel URL did not surface and cloudflared.log did not match a known pattern; status was:\n${lastStatusText}`; } + throw new Error(`[NemoClaw fault] ${reason}`); + } - let lastPublicProbe: CurlProbe | undefined; - let backoffMs = 2_000; - for (let attempt = 1; attempt <= 15; attempt += 1) { - const probe = parseCurlProbe( - await host.command( - "curl", - ["-sS", "-L", "--max-time", "30", "-w", "\n__HTTP_CODE:%{http_code}\n", tunnelUrl], - { - artifactName: `public-tunnel-probe-${attempt}`, - env: buildAvailabilityProbeEnv(), - timeoutMs: 35_000, - }, - ), - ); - lastPublicProbe = probe; - if (probe.httpCode === "200") break; - - const local = await host.command( + let lastPublicProbe: CurlProbe | undefined; + let backoffMs = 2_000; + for (let attempt = 1; attempt <= 15; attempt += 1) { + const probe = parseCurlProbe( + await host.command( "curl", - ["-sS", "-o", "/dev/null", "-w", "%{http_code}", "--max-time", "5", `http://localhost:${LOCAL_DASHBOARD_PORT}/`], + ["-sS", "-L", "--max-time", "30", "-w", "\n__HTTP_CODE:%{http_code}\n", tunnelUrl], { - artifactName: `local-dashboard-recheck-${attempt}`, + artifactName: `public-tunnel-probe-${attempt}`, env: buildAvailabilityProbeEnv(), - timeoutMs: 10_000, + timeoutMs: 35_000, }, + ), + ); + lastPublicProbe = probe; + if (probe.httpCode === "200") break; + + const local = await host.command( + "curl", + [ + "-sS", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "--max-time", + "5", + `http://localhost:${LOCAL_DASHBOARD_PORT}/`, + ], + { + artifactName: `local-dashboard-recheck-${attempt}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 10_000, + }, + ); + const localCode = local.stdout.trim() || "000"; + if (localCode === "000") { + throw new Error( + `[NemoClaw fault] Tunnel returned ${probe.httpCode} and local dashboard regressed during retry loop; likely sandbox/dashboard crash, not Cloudflare.`, ); - const localCode = local.stdout.trim() || "000"; - if (localCode === "000") { - throw new Error( - `[NemoClaw fault] Tunnel returned ${probe.httpCode} and local dashboard regressed during retry loop; likely sandbox/dashboard crash, not Cloudflare.`, - ); - } - await sleep(backoffMs); - backoffMs = Math.min(backoffMs * 2, 30_000); } + await sleep(backoffMs); + backoffMs = Math.min(backoffMs * 2, 30_000); + } - expect(lastPublicProbe, "public tunnel probe should have run").toBeTruthy(); - if (lastPublicProbe!.httpCode !== "200") { - if ( - isCloudflareTransientHttpCode(lastPublicProbe!.httpCode) || - isCloudflareTransientText(lastPublicProbe!.body) || - isCloudflareTransientText(readCloudflaredLog()) - ) { - skip( - `[Cloudflare fault] Tunnel URL never became reachable while local stayed healthy; last HTTP status ${lastPublicProbe!.httpCode}.`, - ); - } - throw new Error( - `[NemoClaw fault] Tunnel returned unexpected HTTP ${lastPublicProbe!.httpCode} while local stayed healthy; body prefix: ${lastPublicProbe!.body.slice(0, 200)}`, + expect(lastPublicProbe, "public tunnel probe should have run").toBeTruthy(); + if (lastPublicProbe!.httpCode !== "200") { + if ( + isCloudflareTransientHttpCode(lastPublicProbe!.httpCode) || + isCloudflareTransientText(lastPublicProbe!.body) || + isCloudflareTransientText(readCloudflaredLog()) + ) { + skip( + `[Cloudflare fault] Tunnel URL never became reachable while local stayed healthy; last HTTP status ${lastPublicProbe!.httpCode}.`, ); } - expect(lastPublicProbe!.body, "public tunnel must serve OpenClaw dashboard markers").toMatch( - DASHBOARD_MARKER_PATTERN, + throw new Error( + `[NemoClaw fault] Tunnel returned unexpected HTTP ${lastPublicProbe!.httpCode} while local stayed healthy; body prefix: ${lastPublicProbe!.body.slice(0, 200)}`, ); + } + expect(lastPublicProbe!.body, "public tunnel must serve OpenClaw dashboard markers").toMatch( + DASHBOARD_MARKER_PATTERN, + ); - const stop = await host.nemoclaw(["tunnel", "stop"], { - artifactName: "tunnel-stop", + const stop = await host.nemoclaw(["tunnel", "stop"], { + artifactName: "tunnel-stop", + env: commandEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }); + expect(stop.exitCode, resultText(stop)).toBe(0); + + let postStopUrl: string | undefined; + let statusReadable = false; + for (let attempt = 1; attempt <= 10; attempt += 1) { + const status = await host.nemoclaw(["status"], { + artifactName: `status-after-tunnel-stop-${attempt}`, env: commandEnv(), timeoutMs: COMMAND_TIMEOUT_MS, }); - expect(stop.exitCode, resultText(stop)).toBe(0); - - let postStopUrl: string | undefined; - let statusReadable = false; - for (let attempt = 1; attempt <= 10; attempt += 1) { - const status = await host.nemoclaw(["status"], { - artifactName: `status-after-tunnel-stop-${attempt}`, - env: commandEnv(), - timeoutMs: COMMAND_TIMEOUT_MS, - }); - if (status.exitCode !== 0) { - await sleep(1_000); - continue; - } - statusReadable = true; - postStopUrl = extractTunnelUrl(resultText(status)); - if (!postStopUrl) break; + if (status.exitCode !== 0) { await sleep(1_000); + continue; } - expect(statusReadable, "nemoclaw status should be readable after tunnel stop").toBe(true); - expect(postStopUrl, "tunnel URL must be absent after nemoclaw tunnel stop").toBeUndefined(); + statusReadable = true; + postStopUrl = extractTunnelUrl(resultText(status)); + if (!postStopUrl) break; + await sleep(1_000); + } + expect(statusReadable, "nemoclaw status should be readable after tunnel stop").toBe(true); + expect(postStopUrl, "tunnel URL must be absent after nemoclaw tunnel stop").toBeUndefined(); } From c02800c667b45325d7d0a20c861b41812c5fe4b3 Mon Sep 17 00:00:00 2001 From: Julie Yaunches <jyaunches@nvidia.com> Date: Mon, 22 Jun 2026 10:35:54 -0400 Subject: [PATCH 05/15] test(e2e): address tunnel lifecycle advisor --- .github/workflows/e2e-vitest-scenarios.yaml | 1 + test/cloudflared-version-resolver.test.ts | 11 ++++- .../e2e-scenarios-workflow.test.ts | 21 ++++++++ test/e2e/lib/cloudflared-version-resolver.sh | 5 ++ tools/e2e-scenarios/workflow-boundary.mts | 49 +++++++++++++++++++ 5 files changed, 86 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 2ced920b931..e5516190ff9 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -4169,6 +4169,7 @@ jobs: FREE_STANDING_SCENARIO_ID: "tunnel-lifecycle" DOCKER_CONFIG: ${{ github.workspace }}/.docker-config-tunnel-lifecycle E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/tunnel-lifecycle + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_E2E_SCENARIOS: "1" NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" diff --git a/test/cloudflared-version-resolver.test.ts b/test/cloudflared-version-resolver.test.ts index 12a820a4190..750bc345cd0 100644 --- a/test/cloudflared-version-resolver.test.ts +++ b/test/cloudflared-version-resolver.test.ts @@ -23,6 +23,7 @@ if [[ "\${1:-}" != "--compare-versions" ]]; then fi rank() { case "\${1:-}" in + 2020.1.1) printf '20200101' ;; 2026.4.30) printf '20260430' ;; 2026.5.1~rc1) printf '20260500' ;; 2026.5.1) printf '20260501' ;; @@ -89,13 +90,21 @@ describe("cloudflared APT package resolver", () => { expect(result.stderr).toContain("meets minimum 2026.5.1"); }); - it("preserves exact CLOUDFLARED_VERSION overrides for emergency repro", () => { + it("preserves syntactically valid exact CLOUDFLARED_VERSION overrides for emergency repro", () => { const result = runResolver("2026.5.1\n2026.5.10", "2026.5.1", "2020.1.1"); expect(result.status).toBe(0); expect(result.stdout.trim()).toBe("2020.1.1"); }); + it("rejects invalid CLOUDFLARED_VERSION overrides before apt install", () => { + const result = runResolver("2026.5.1\n2026.5.10", "2026.5.1", "bad/min"); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("invalid CLOUDFLARED_VERSION"); + expect(result.stdout.trim()).toBe(""); + }); + it("rejects invalid minimum versions before comparing package versions", () => { const result = runResolver("2026.5.1", "bad/min"); diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index 4a28ac29588..6e8776b39ee 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -1061,6 +1061,27 @@ jobs: } }); + it("requires tunnel lifecycle to use the repo NemoClaw CLI boundary", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); + const workflowPath = path.join(tmp, "workflow.yaml"); + const workflow = readWorkflow() as { + jobs: Record<string, { env?: Record<string, unknown> }>; + }; + const job = workflow.jobs["tunnel-lifecycle-vitest"]; + expect(job).toBeDefined(); + job.env = { ...job.env }; + delete job.env.NEMOCLAW_CLI_BIN; + fs.writeFileSync(workflowPath, YAML.stringify(workflow)); + + try { + expect(validateE2eVitestScenariosWorkflowBoundary(workflowPath)).toContain( + "tunnel-lifecycle-vitest job must point NEMOCLAW_CLI_BIN at the repo CLI", + ); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + it("applies boundary checks to newly marked free-standing jobs", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); const workflowPath = path.join(tmp, "workflow.yaml"); diff --git a/test/e2e/lib/cloudflared-version-resolver.sh b/test/e2e/lib/cloudflared-version-resolver.sh index a56a93d252f..64e88aca36b 100755 --- a/test/e2e/lib/cloudflared-version-resolver.sh +++ b/test/e2e/lib/cloudflared-version-resolver.sh @@ -28,7 +28,12 @@ cloudflared_resolve_package_version() { # Emergency repro knob: install the exact requested version and let APT report # unavailable overrides, rather than silently substituting another package. + # Still validate Debian-version syntax before the sudo apt install boundary. if [[ -n "$override_version" ]]; then + if ! cloudflared_is_debian_version "$override_version"; then + printf 'ERROR: invalid CLOUDFLARED_VERSION %q\n' "$override_version" >&2 + return 1 + fi printf '%s\n' "$override_version" return 0 fi diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index 8decf10480e..31e91f07616 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -3138,6 +3138,53 @@ function validateModelRouterProviderRoutedInferenceVitestJob( requireRunContains(errors, cleanup, 'rm -rf "${DOCKER_CONFIG}"'); } +function validateTunnelLifecycleVitestJob(errors: string[], jobs: WorkflowRecord): void { + const jobName = "tunnel-lifecycle-vitest"; + const scenarioName = "tunnel-lifecycle"; + const job = asRecord(jobs[jobName]); + if (Object.keys(job).length === 0) { + errors.push("workflow missing tunnel-lifecycle-vitest job"); + return; + } + + if (job["runs-on"] !== "ubuntu-latest") { + errors.push("tunnel-lifecycle-vitest job must run on ubuntu-latest"); + } + if (job["timeout-minutes"] !== 75) { + errors.push("tunnel-lifecycle-vitest job must keep the 75 minute timeout"); + } + validateFreeStandingJobSelector(errors, jobs, jobName, scenarioName); + + const jobEnv = asRecord(job.env); + if (jobEnv.NEMOCLAW_CLI_BIN !== "${{ github.workspace }}/bin/nemoclaw.js") { + errors.push("tunnel-lifecycle-vitest job must point NEMOCLAW_CLI_BIN at the repo CLI"); + } + if (jobEnv.FREE_STANDING_VITEST_JOB !== "1") { + errors.push("tunnel-lifecycle-vitest job must set FREE_STANDING_VITEST_JOB=1"); + } + if (jobEnv.FREE_STANDING_SCENARIO_ID !== scenarioName) { + errors.push(`tunnel-lifecycle-vitest job must set FREE_STANDING_SCENARIO_ID=${scenarioName}`); + } + if (jobEnv.NEMOCLAW_RUN_E2E_SCENARIOS !== "1") { + errors.push("tunnel-lifecycle-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1"); + } + requireEnvDoesNotExposeSecret(errors, "tunnel-lifecycle-vitest job", jobEnv, "NVIDIA_INFERENCE_API_KEY"); + + const steps = asSteps(job.steps); + const buildCli = requireJobStep(errors, jobName, steps, "Build CLI"); + requireRunContains(errors, buildCli, "npm run build:cli"); + + const runVitest = requireJobStep(errors, jobName, steps, "Run tunnel lifecycle live test"); + const runVitestEnv = asRecord(runVitest?.env); + if (runVitestEnv.NVIDIA_INFERENCE_API_KEY !== "${{ secrets.NVIDIA_INFERENCE_API_KEY }}") { + errors.push( + "tunnel-lifecycle-vitest Vitest step must receive NVIDIA_INFERENCE_API_KEY from secrets", + ); + } + requireRunContains(errors, runVitest, "npx vitest run --project e2e-scenarios-live"); + requireRunContains(errors, runVitest, "test/e2e-scenario/live/tunnel-lifecycle.test.ts"); +} + function validateIssue2478CrashLoopRecoveryVitestJob(errors: string[], jobs: WorkflowRecord): void { const jobName = "issue-2478-crash-loop-recovery-vitest"; const scenarioName = "issue-2478-crash-loop-recovery"; @@ -4007,6 +4054,8 @@ export function validateE2eVitestScenariosWorkflowBoundary( validateIssue2478CrashLoopRecoveryVitestJob(errors, jobs); + validateTunnelLifecycleVitestJob(errors, jobs); + validateFreeStandingJobSelector( errors, jobs, From ec7f0c44220de4234ab2e60324b486910ed39cea Mon Sep 17 00:00:00 2001 From: Julie Yaunches <jyaunches@nvidia.com> Date: Mon, 22 Jun 2026 10:46:07 -0400 Subject: [PATCH 06/15] test(cli): relax sandbox status timeout --- test/cli/sandbox-status-json.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/cli/sandbox-status-json.test.ts b/test/cli/sandbox-status-json.test.ts index c3f3ffc5603..daf8de10fba 100644 --- a/test/cli/sandbox-status-json.test.ts +++ b/test/cli/sandbox-status-json.test.ts @@ -7,9 +7,9 @@ import net from "node:net"; import os from "node:os"; import path from "node:path"; -import { runWithEnv, writeSandboxRegistry } from "./helpers"; +import { runWithEnv, testTimeoutOptions, writeSandboxRegistry } from "./helpers"; -describe("CLI sandbox status JSON output", () => { +describe("CLI sandbox status JSON output", testTimeoutOptions(20_000), () => { it("sandbox status --json emits structured per-sandbox report", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-sandbox-status-json-")); const localBin = path.join(home, "bin"); From 669d3324ad500615f9b06ad4369e5870dbd3a8e5 Mon Sep 17 00:00:00 2001 From: Julie Yaunches <jyaunches@nvidia.com> Date: Mon, 22 Jun 2026 10:59:11 -0400 Subject: [PATCH 07/15] test(e2e): tighten tunnel advisor coverage --- .../live/tunnel-lifecycle-helpers.ts | 39 ++++----- .../e2e-scenarios-workflow.test.ts | 46 ++++++++++ .../tunnel-lifecycle-helpers.test.ts | 51 +++++++++++ tools/e2e-scenarios/workflow-boundary.mts | 85 ++++++++++++++++++- 4 files changed, 199 insertions(+), 22 deletions(-) create mode 100644 test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts diff --git a/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts b/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts index 0e47fdaba52..bdc6eab54b8 100644 --- a/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts +++ b/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts @@ -78,20 +78,19 @@ function isCloudflareTransientHttpCode(code: string): boolean { return ["000", "502", "503", "504"].includes(code); } -function getCloudflaredLogPath(): string | undefined { - const sandboxLog = path.join("/tmp", `nemoclaw-services-${SANDBOX_NAME}`, "cloudflared.log"); - if (fs.existsSync(sandboxLog)) return sandboxLog; - let candidates: string[] = []; - try { - candidates = fs - .readdirSync("/tmp", { withFileTypes: true }) - .filter((entry) => entry.isDirectory() && entry.name.startsWith("nemoclaw-services-")) - .map((entry) => path.join("/tmp", entry.name, "cloudflared.log")) - .filter((candidate) => fs.existsSync(candidate)); - } catch { - return undefined; - } - return candidates.sort((a, b) => fs.statSync(b).mtimeMs - fs.statSync(a).mtimeMs).at(0); +export function getCloudflaredLogPath( + logRoot = "/tmp", + sandboxName = SANDBOX_NAME, +): string | undefined { + // Source boundary: NemoClaw owns the per-sandbox cloudflared service log at + // /tmp/nemoclaw-services-${sandboxName}/cloudflared.log. If that exact file + // is missing, this live contract classifies the invalid state as + // `nemoclaw_no_spawn` instead of falling back to the newest /tmp log, because + // unrelated parallel/stale sandboxes can otherwise corrupt fault attribution. + // Remove this filesystem fallback point entirely once NemoClaw exposes + // machine-readable tunnel diagnostics from `nemoclaw status --json`. + const sandboxLog = path.join(logRoot, `nemoclaw-services-${sandboxName}`, "cloudflared.log"); + return fs.existsSync(sandboxLog) ? sandboxLog : undefined; } function readCloudflaredLog(): string { @@ -110,13 +109,11 @@ function cloudflaredLogTail(lines = 80): string { ].join("\n"); } -function classifyCloudflaredLog(): - | "nemoclaw_no_spawn" - | "nemoclaw_capture_bug" - | "nemoclaw_local" - | "cloudflare" - | "unknown" { - const logPath = getCloudflaredLogPath(); +export function classifyCloudflaredLog( + logRoot = "/tmp", + sandboxName = SANDBOX_NAME, +): "nemoclaw_no_spawn" | "nemoclaw_capture_bug" | "nemoclaw_local" | "cloudflare" | "unknown" { + const logPath = getCloudflaredLogPath(logRoot, sandboxName); if (!logPath) return "nemoclaw_no_spawn"; const log = fs.readFileSync(logPath, "utf8"); if (TUNNEL_URL_PATTERN.test(log)) return "nemoclaw_capture_bug"; diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index 6e8776b39ee..19a37a2f9c4 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -1082,6 +1082,52 @@ jobs: } }); + it("rejects tunnel lifecycle trusted-boundary drift", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); + const workflowPath = path.join(tmp, "workflow.yaml"); + const workflow = readWorkflow() as { + jobs: Record<string, { steps: Array<Record<string, unknown>> }>; + }; + const job = workflow.jobs["tunnel-lifecycle-vitest"]; + expect(job).toBeDefined(); + for (const step of job.steps) { + if (typeof step.uses === "string" && step.uses.startsWith("actions/checkout@")) { + step.with = { ...(step.with as Record<string, unknown>), "persist-credentials": true }; + } + if (step.name === "Install root dependencies") { + step.run = "npm install"; + } + if (step.name === "Upload tunnel lifecycle artifacts") { + step.with = { + ...(step.with as Record<string, unknown>), + path: "e2e-artifacts/vitest/", + "include-hidden-files": true, + }; + } + if (step.name === "Clean up Docker auth") { + step.if = "success()"; + step.run = 'set -euo pipefail\necho "missing Docker auth cleanup"\n'; + } + } + fs.writeFileSync(workflowPath, YAML.stringify(workflow)); + + try { + expect(validateE2eVitestScenariosWorkflowBoundary(workflowPath)).toEqual( + expect.arrayContaining([ + "tunnel-lifecycle-vitest checkout step must set persist-credentials=false", + "step 'Install root dependencies' run script must include npm ci --ignore-scripts", + "artifact upload path must include e2e-artifacts/vitest/tunnel-lifecycle/", + "tunnel-lifecycle-vitest artifact upload must set include-hidden-files: false", + "tunnel-lifecycle-vitest Docker auth cleanup must always run", + "step 'Clean up Docker auth' run script must include docker logout docker.io", + "step 'Clean up Docker auth' run script must include rm -rf \"${DOCKER_CONFIG}\"", + ]), + ); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + it("applies boundary checks to newly marked free-standing jobs", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); const workflowPath = path.join(tmp, "workflow.yaml"); diff --git a/test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts b/test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts new file mode 100644 index 00000000000..870e496430e --- /dev/null +++ b/test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + classifyCloudflaredLog, + getCloudflaredLogPath, +} from "../live/tunnel-lifecycle-helpers.ts"; + +describe("tunnel lifecycle cloudflared log attribution", () => { + it("does not attribute an unrelated newer cloudflared log to the current sandbox", () => { + const logRoot = fs.mkdtempSync(path.join(os.tmpdir(), "tunnel-lifecycle-logs-")); + const unrelatedDir = path.join(logRoot, "nemoclaw-services-other-sandbox"); + fs.mkdirSync(unrelatedDir, { recursive: true }); + fs.writeFileSync( + path.join(unrelatedDir, "cloudflared.log"), + "https://unrelated.trycloudflare.com captured by another run\n", + ); + + try { + expect(getCloudflaredLogPath(logRoot, "e2e-tunnel-lifecycle-current")).toBeUndefined(); + expect(classifyCloudflaredLog(logRoot, "e2e-tunnel-lifecycle-current")).toBe( + "nemoclaw_no_spawn", + ); + } finally { + fs.rmSync(logRoot, { recursive: true, force: true }); + } + }); + + it("classifies only the sandbox-specific cloudflared log", () => { + const logRoot = fs.mkdtempSync(path.join(os.tmpdir(), "tunnel-lifecycle-logs-")); + const sandboxDir = path.join(logRoot, "nemoclaw-services-e2e-tunnel-lifecycle-current"); + fs.mkdirSync(sandboxDir, { recursive: true }); + const sandboxLog = path.join(sandboxDir, "cloudflared.log"); + fs.writeFileSync(sandboxLog, "https://current.trycloudflare.com\n"); + + try { + expect(getCloudflaredLogPath(logRoot, "e2e-tunnel-lifecycle-current")).toBe(sandboxLog); + expect(classifyCloudflaredLog(logRoot, "e2e-tunnel-lifecycle-current")).toBe( + "nemoclaw_capture_bug", + ); + } finally { + fs.rmSync(logRoot, { recursive: true, force: true }); + } + }); +}); diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index 31e91f07616..2436e275302 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -3168,9 +3168,67 @@ function validateTunnelLifecycleVitestJob(errors: string[], jobs: WorkflowRecord if (jobEnv.NEMOCLAW_RUN_E2E_SCENARIOS !== "1") { errors.push("tunnel-lifecycle-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1"); } - requireEnvDoesNotExposeSecret(errors, "tunnel-lifecycle-vitest job", jobEnv, "NVIDIA_INFERENCE_API_KEY"); + requireEnvDoesNotExposeSecret( + errors, + "tunnel-lifecycle-vitest job", + jobEnv, + "NVIDIA_INFERENCE_API_KEY", + ); const steps = asSteps(job.steps); + requireNoDispatchInputInterpolation(errors, steps); + for (const step of steps) { + const stepName = `tunnel-lifecycle-vitest step '${step.name ?? step.uses ?? "<unnamed>"}'`; + const stepEnv = asRecord(step.env); + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "GITHUB_TOKEN"); + if (step.name !== "Authenticate to Docker Hub") { + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_USERNAME"); + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_TOKEN"); + requireNoDockerHubAuthInRun(errors, stepName, stringValue(step.run)); + } + } + + const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); + if (!checkout) { + errors.push("tunnel-lifecycle-vitest job missing checkout step"); + } + requireFullShaAction(errors, checkout, "tunnel-lifecycle-vitest checkout"); + if (asRecord(checkout?.with)["persist-credentials"] !== false) { + errors.push("tunnel-lifecycle-vitest checkout step must set persist-credentials=false"); + } + + const dockerLogin = requireJobStep(errors, jobName, steps, "Authenticate to Docker Hub"); + const dockerLoginEnv = asRecord(dockerLogin?.env); + if (dockerLoginEnv.DOCKERHUB_USERNAME !== "${{ secrets.DOCKERHUB_USERNAME }}") { + errors.push( + "tunnel-lifecycle-vitest Docker Hub auth must receive DOCKERHUB_USERNAME from secrets", + ); + } + if (dockerLoginEnv.DOCKERHUB_TOKEN !== "${{ secrets.DOCKERHUB_TOKEN }}") { + errors.push( + "tunnel-lifecycle-vitest Docker Hub auth must receive DOCKERHUB_TOKEN from secrets", + ); + } + requireRunContains(errors, dockerLogin, 'mkdir -p "${DOCKER_CONFIG}"'); + requireRunContains(errors, dockerLogin, 'chmod 700 "${DOCKER_CONFIG}"'); + requireRunContains(errors, dockerLogin, "docker login docker.io"); + requireRunContains(errors, dockerLogin, "--password-stdin"); + requireRunContains(errors, dockerLogin, "continuing with anonymous pulls"); + + const setupNode = namedStep(steps, "Set up Node"); + if (!setupNode) { + errors.push("tunnel-lifecycle-vitest job missing step: Set up Node"); + } + requireFullShaAction(errors, setupNode, "tunnel-lifecycle-vitest setup-node"); + + const installRootDependencies = requireJobStep( + errors, + jobName, + steps, + "Install root dependencies", + ); + requireRunContains(errors, installRootDependencies, "npm ci --ignore-scripts"); + const buildCli = requireJobStep(errors, jobName, steps, "Build CLI"); requireRunContains(errors, buildCli, "npm run build:cli"); @@ -3183,6 +3241,31 @@ function validateTunnelLifecycleVitestJob(errors: string[], jobs: WorkflowRecord } requireRunContains(errors, runVitest, "npx vitest run --project e2e-scenarios-live"); requireRunContains(errors, runVitest, "test/e2e-scenario/live/tunnel-lifecycle.test.ts"); + + const upload = requireJobStep(errors, jobName, steps, "Upload tunnel lifecycle artifacts"); + requireFullShaAction(errors, upload, "tunnel-lifecycle-vitest upload-artifact"); + const uploadWith = asRecord(upload?.with); + if (uploadWith.name !== "e2e-vitest-scenarios-tunnel-lifecycle") { + errors.push("tunnel-lifecycle-vitest artifact upload name must be stable"); + } + const uploadPath = stringValue(uploadWith.path); + requireUploadPathContains(errors, uploadPath, "e2e-artifacts/vitest/tunnel-lifecycle/"); + if (uploadWith["include-hidden-files"] !== false) { + errors.push("tunnel-lifecycle-vitest artifact upload must set include-hidden-files: false"); + } + if (uploadWith["if-no-files-found"] !== "ignore") { + errors.push("tunnel-lifecycle-vitest artifact upload must ignore missing fixture artifacts"); + } + if (uploadWith["retention-days"] !== 14) { + errors.push("tunnel-lifecycle-vitest artifact upload retention-days must be 14"); + } + + const cleanup = requireJobStep(errors, jobName, steps, "Clean up Docker auth"); + if (cleanup?.if !== "always()") { + errors.push("tunnel-lifecycle-vitest Docker auth cleanup must always run"); + } + requireRunContains(errors, cleanup, "docker logout docker.io"); + requireRunContains(errors, cleanup, 'rm -rf "${DOCKER_CONFIG}"'); } function validateIssue2478CrashLoopRecoveryVitestJob(errors: string[], jobs: WorkflowRecord): void { From 3d084dd67f59b0f90ef1b6b075b7efadcdbdbd8b Mon Sep 17 00:00:00 2001 From: Julie Yaunches <jyaunches@nvidia.com> Date: Mon, 22 Jun 2026 11:04:51 -0400 Subject: [PATCH 08/15] test(e2e): keep advisor tests linear --- .../e2e-scenarios-workflow.test.ts | 44 +++++++++++-------- .../tunnel-lifecycle-helpers.test.ts | 5 +-- 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index 19a37a2f9c4..f29f69437b0 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -1090,25 +1090,31 @@ jobs: }; const job = workflow.jobs["tunnel-lifecycle-vitest"]; expect(job).toBeDefined(); - for (const step of job.steps) { - if (typeof step.uses === "string" && step.uses.startsWith("actions/checkout@")) { - step.with = { ...(step.with as Record<string, unknown>), "persist-credentials": true }; - } - if (step.name === "Install root dependencies") { - step.run = "npm install"; - } - if (step.name === "Upload tunnel lifecycle artifacts") { - step.with = { - ...(step.with as Record<string, unknown>), - path: "e2e-artifacts/vitest/", - "include-hidden-files": true, - }; - } - if (step.name === "Clean up Docker auth") { - step.if = "success()"; - step.run = 'set -euo pipefail\necho "missing Docker auth cleanup"\n'; - } - } + const checkout = job.steps.find((step) => + String(step.uses ?? "").startsWith("actions/checkout@"), + ); + expect(checkout).toBeDefined(); + checkout!.with = { + ...(checkout!.with as Record<string, unknown>), + "persist-credentials": true, + }; + + const install = job.steps.find((step) => step.name === "Install root dependencies"); + expect(install).toBeDefined(); + install!.run = "npm install"; + + const upload = job.steps.find((step) => step.name === "Upload tunnel lifecycle artifacts"); + expect(upload).toBeDefined(); + upload!.with = { + ...(upload!.with as Record<string, unknown>), + path: "e2e-artifacts/vitest/", + "include-hidden-files": true, + }; + + const cleanup = job.steps.find((step) => step.name === "Clean up Docker auth"); + expect(cleanup).toBeDefined(); + cleanup!.if = "success()"; + cleanup!.run = 'set -euo pipefail\necho "missing Docker auth cleanup"\n'; fs.writeFileSync(workflowPath, YAML.stringify(workflow)); try { diff --git a/test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts b/test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts index 870e496430e..f6f4e5bfbe0 100644 --- a/test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts +++ b/test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts @@ -7,10 +7,7 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; -import { - classifyCloudflaredLog, - getCloudflaredLogPath, -} from "../live/tunnel-lifecycle-helpers.ts"; +import { classifyCloudflaredLog, getCloudflaredLogPath } from "../live/tunnel-lifecycle-helpers.ts"; describe("tunnel lifecycle cloudflared log attribution", () => { it("does not attribute an unrelated newer cloudflared log to the current sandbox", () => { From a1476b9ad33d231dd26c8b883af6ed80227262e0 Mon Sep 17 00:00:00 2001 From: Julie Yaunches <jyaunches@nvidia.com> Date: Mon, 22 Jun 2026 11:20:07 -0400 Subject: [PATCH 09/15] ci(e2e): isolate tunnel Docker auth --- .github/workflows/e2e-vitest-scenarios.yaml | 4 +++- .../e2e-scenarios-workflow.test.ts | 20 ++++++++++++++++++- tools/e2e-scenarios/workflow-boundary.mts | 17 ++++++++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index e5516190ff9..dd6697f7dec 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -4167,7 +4167,6 @@ jobs: env: FREE_STANDING_VITEST_JOB: "1" FREE_STANDING_SCENARIO_ID: "tunnel-lifecycle" - DOCKER_CONFIG: ${{ github.workspace }}/.docker-config-tunnel-lifecycle E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/tunnel-lifecycle NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_E2E_SCENARIOS: "1" @@ -4180,6 +4179,9 @@ jobs: with: persist-credentials: false + - name: Configure isolated Docker auth directory + run: echo "DOCKER_CONFIG=${RUNNER_TEMP}/docker-config-tunnel-lifecycle" >> "$GITHUB_ENV" + - name: Authenticate to Docker Hub env: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index f29f69437b0..b11b310c282 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -1086,10 +1086,18 @@ jobs: const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); const workflowPath = path.join(tmp, "workflow.yaml"); const workflow = readWorkflow() as { - jobs: Record<string, { steps: Array<Record<string, unknown>> }>; + jobs: Record< + string, + { env?: Record<string, unknown>; steps: Array<Record<string, unknown>> } + >; }; const job = workflow.jobs["tunnel-lifecycle-vitest"]; expect(job).toBeDefined(); + job.env = { + ...job.env, + DOCKER_CONFIG: "${{ github.workspace }}/e2e-artifacts/vitest/tunnel-lifecycle/docker-config", + }; + const checkout = job.steps.find((step) => String(step.uses ?? "").startsWith("actions/checkout@"), ); @@ -1099,6 +1107,13 @@ jobs: "persist-credentials": true, }; + const configureDockerAuth = job.steps.find( + (step) => step.name === "Configure isolated Docker auth directory", + ); + expect(configureDockerAuth).toBeDefined(); + configureDockerAuth!.run = + 'echo "DOCKER_CONFIG=${{ github.workspace }}/docker-config-tunnel-lifecycle" >> "$GITHUB_ENV"'; + const install = job.steps.find((step) => step.name === "Install root dependencies"); expect(install).toBeDefined(); install!.run = "npm install"; @@ -1120,6 +1135,9 @@ jobs: try { expect(validateE2eVitestScenariosWorkflowBoundary(workflowPath)).toEqual( expect.arrayContaining([ + "tunnel-lifecycle-vitest job must not set DOCKER_CONFIG at job level", + 'step \'Configure isolated Docker auth directory\' run script must include echo "DOCKER_CONFIG=${RUNNER_TEMP}/docker-config-tunnel-lifecycle" >> "$GITHUB_ENV"', + "step 'Configure isolated Docker auth directory' run script must not include ${{ github.workspace }}", "tunnel-lifecycle-vitest checkout step must set persist-credentials=false", "step 'Install root dependencies' run script must include npm ci --ignore-scripts", "artifact upload path must include e2e-artifacts/vitest/tunnel-lifecycle/", diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index 2436e275302..31d022dbd97 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -3156,6 +3156,9 @@ function validateTunnelLifecycleVitestJob(errors: string[], jobs: WorkflowRecord validateFreeStandingJobSelector(errors, jobs, jobName, scenarioName); const jobEnv = asRecord(job.env); + if ("DOCKER_CONFIG" in jobEnv) { + errors.push("tunnel-lifecycle-vitest job must not set DOCKER_CONFIG at job level"); + } if (jobEnv.NEMOCLAW_CLI_BIN !== "${{ github.workspace }}/bin/nemoclaw.js") { errors.push("tunnel-lifecycle-vitest job must point NEMOCLAW_CLI_BIN at the repo CLI"); } @@ -3197,6 +3200,20 @@ function validateTunnelLifecycleVitestJob(errors: string[], jobs: WorkflowRecord errors.push("tunnel-lifecycle-vitest checkout step must set persist-credentials=false"); } + const configureDockerAuth = requireJobStep( + errors, + jobName, + steps, + "Configure isolated Docker auth directory", + ); + requireRunContains( + errors, + configureDockerAuth, + 'echo "DOCKER_CONFIG=${RUNNER_TEMP}/docker-config-tunnel-lifecycle" >> "$GITHUB_ENV"', + ); + requireRunDoesNotContain(errors, configureDockerAuth, "${{ runner.temp }}"); + requireRunDoesNotContain(errors, configureDockerAuth, "${{ github.workspace }}"); + const dockerLogin = requireJobStep(errors, jobName, steps, "Authenticate to Docker Hub"); const dockerLoginEnv = asRecord(dockerLogin?.env); if (dockerLoginEnv.DOCKERHUB_USERNAME !== "${{ secrets.DOCKERHUB_USERNAME }}") { From 69cc272ea7ce79b165d2ade57c3a2f566f86d040 Mon Sep 17 00:00:00 2001 From: Julie Yaunches <jyaunches@nvidia.com> Date: Mon, 22 Jun 2026 11:30:37 -0400 Subject: [PATCH 10/15] test(e2e): stop tunnel probe redirects --- .../live/tunnel-lifecycle-helpers.ts | 23 +++++++++++-------- .../tunnel-lifecycle-helpers.test.ts | 17 +++++++++++++- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts b/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts index bdc6eab54b8..240b7e0be7c 100644 --- a/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts +++ b/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts @@ -132,6 +132,15 @@ function extractTunnelUrl(text: string): string | undefined { return text.match(TUNNEL_URL_PATTERN)?.[0]; } +export function publicTunnelProbeCurlArgs(tunnelUrl: string): string[] { + // Source boundary: the public tunnel URL already came from `nemoclaw status` + // and matched `*.trycloudflare.com`. Do not ask curl to follow redirects; + // a 3xx response is a tunnel/output contract failure unless NemoClaw grows a + // documented same-host redirect requirement. If that happens, replace this + // with explicit redirect target inspection before issuing a second request. + return ["-sS", "--max-time", "30", "-w", "\n__HTTP_CODE:%{http_code}\n", tunnelUrl]; +} + function parseCurlProbe(result: ShellProbeResult): CurlProbe { const text = result.stdout; const match = text.match(/\n__HTTP_CODE:(\d{3})\s*$/); @@ -397,15 +406,11 @@ export async function runTunnelLifecycleContract({ let backoffMs = 2_000; for (let attempt = 1; attempt <= 15; attempt += 1) { const probe = parseCurlProbe( - await host.command( - "curl", - ["-sS", "-L", "--max-time", "30", "-w", "\n__HTTP_CODE:%{http_code}\n", tunnelUrl], - { - artifactName: `public-tunnel-probe-${attempt}`, - env: buildAvailabilityProbeEnv(), - timeoutMs: 35_000, - }, - ), + await host.command("curl", publicTunnelProbeCurlArgs(tunnelUrl), { + artifactName: `public-tunnel-probe-${attempt}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 35_000, + }), ); lastPublicProbe = probe; if (probe.httpCode === "200") break; diff --git a/test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts b/test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts index f6f4e5bfbe0..ce9eaeb742c 100644 --- a/test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts +++ b/test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts @@ -7,9 +7,24 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; -import { classifyCloudflaredLog, getCloudflaredLogPath } from "../live/tunnel-lifecycle-helpers.ts"; +import { + classifyCloudflaredLog, + getCloudflaredLogPath, + publicTunnelProbeCurlArgs, +} from "../live/tunnel-lifecycle-helpers.ts"; describe("tunnel lifecycle cloudflared log attribution", () => { + it("does not follow redirects from the public trycloudflare probe", () => { + expect(publicTunnelProbeCurlArgs("https://current.trycloudflare.com/")).toEqual([ + "-sS", + "--max-time", + "30", + "-w", + "\n__HTTP_CODE:%{http_code}\n", + "https://current.trycloudflare.com/", + ]); + }); + it("does not attribute an unrelated newer cloudflared log to the current sandbox", () => { const logRoot = fs.mkdtempSync(path.join(os.tmpdir(), "tunnel-lifecycle-logs-")); const unrelatedDir = path.join(logRoot, "nemoclaw-services-other-sandbox"); From b2f5049b711934905bff3669b4e2f86ce38d2838 Mon Sep 17 00:00:00 2001 From: Julie Yaunches <jyaunches@nvidia.com> Date: Mon, 22 Jun 2026 12:20:56 -0400 Subject: [PATCH 11/15] test(e2e): cover tunnel log classifications --- .../tunnel-lifecycle-helpers.test.ts | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts b/test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts index ce9eaeb742c..d87fdb56a54 100644 --- a/test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts +++ b/test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts @@ -60,4 +60,38 @@ describe("tunnel lifecycle cloudflared log attribution", () => { fs.rmSync(logRoot, { recursive: true, force: true }); } }); + + it("classifies localhost/origin-refused logs as a NemoClaw local-origin fault", () => { + const logRoot = fs.mkdtempSync(path.join(os.tmpdir(), "tunnel-lifecycle-logs-")); + const sandboxDir = path.join(logRoot, "nemoclaw-services-e2e-tunnel-lifecycle-current"); + fs.mkdirSync(sandboxDir, { recursive: true }); + fs.writeFileSync( + path.join(sandboxDir, "cloudflared.log"), + "ERR Request failed error=\"Unable to reach the origin service. dial tcp 127.0.0.1:18789: connect: connection refused\"\n", + ); + + try { + expect(classifyCloudflaredLog(logRoot, "e2e-tunnel-lifecycle-current")).toBe( + "nemoclaw_local", + ); + } finally { + fs.rmSync(logRoot, { recursive: true, force: true }); + } + }); + + it("classifies representative quick-tunnel registration failures as Cloudflare faults", () => { + const logRoot = fs.mkdtempSync(path.join(os.tmpdir(), "tunnel-lifecycle-logs-")); + const sandboxDir = path.join(logRoot, "nemoclaw-services-e2e-tunnel-lifecycle-current"); + fs.mkdirSync(sandboxDir, { recursive: true }); + fs.writeFileSync( + path.join(sandboxDir, "cloudflared.log"), + "ERR failed to unmarshal quick Tunnel response: tunnel server returned 503 bad gateway\n", + ); + + try { + expect(classifyCloudflaredLog(logRoot, "e2e-tunnel-lifecycle-current")).toBe("cloudflare"); + } finally { + fs.rmSync(logRoot, { recursive: true, force: true }); + } + }); }); From 878e93a1c45fbe6aacfcec067ecb092942ec7138 Mon Sep 17 00:00:00 2001 From: Julie Yaunches <jyaunches@nvidia.com> Date: Mon, 22 Jun 2026 12:25:43 -0400 Subject: [PATCH 12/15] style: format tunnel helper tests --- .../e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts b/test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts index d87fdb56a54..8f395ade371 100644 --- a/test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts +++ b/test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts @@ -67,7 +67,7 @@ describe("tunnel lifecycle cloudflared log attribution", () => { fs.mkdirSync(sandboxDir, { recursive: true }); fs.writeFileSync( path.join(sandboxDir, "cloudflared.log"), - "ERR Request failed error=\"Unable to reach the origin service. dial tcp 127.0.0.1:18789: connect: connection refused\"\n", + 'ERR Request failed error="Unable to reach the origin service. dial tcp 127.0.0.1:18789: connect: connection refused"\n', ); try { From 908ee22c6c6692c6b01f3b4aa980d5a2f4cde248 Mon Sep 17 00:00:00 2001 From: Julie Yaunches <jyaunches@nvidia.com> Date: Mon, 22 Jun 2026 13:46:59 -0400 Subject: [PATCH 13/15] test(e2e): split tunnel workflow checks --- .../e2e-scenarios-workflow.test.ts | 107 -------------- ...tunnel-lifecycle-workflow-boundary.test.ts | 135 ++++++++++++++++++ 2 files changed, 135 insertions(+), 107 deletions(-) create mode 100644 test/e2e-scenario/support-tests/tunnel-lifecycle-workflow-boundary.test.ts diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index 227c99a99d2..95699519afa 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -594,22 +594,6 @@ describe("e2e-vitest-scenarios workflow boundary", () => { selectedFreeStandingJobs: ["channels-add-remove-vitest"], registryScenarios: [], }); - expect( - evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "tunnel-lifecycle" }), - ).toMatchObject({ - valid: true, - liveScenariosRuns: false, - selectedFreeStandingJobs: ["tunnel-lifecycle-vitest"], - registryScenarios: [], - }); - expect( - evaluateE2eVitestWorkflowDispatchSelectors({ jobs: "tunnel-lifecycle-vitest" }), - ).toMatchObject({ - valid: true, - liveScenariosRuns: false, - selectedFreeStandingJobs: ["tunnel-lifecycle-vitest"], - registryScenarios: [], - }); }, ); @@ -1078,97 +1062,6 @@ jobs: } }); - it("requires tunnel lifecycle to use the repo NemoClaw CLI boundary", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); - const workflowPath = path.join(tmp, "workflow.yaml"); - const workflow = readWorkflow() as { - jobs: Record<string, { env?: Record<string, unknown> }>; - }; - const job = workflow.jobs["tunnel-lifecycle-vitest"]; - expect(job).toBeDefined(); - job.env = { ...job.env }; - delete job.env.NEMOCLAW_CLI_BIN; - fs.writeFileSync(workflowPath, YAML.stringify(workflow)); - - try { - expect(validateE2eVitestScenariosWorkflowBoundary(workflowPath)).toContain( - "tunnel-lifecycle-vitest job must point NEMOCLAW_CLI_BIN at the repo CLI", - ); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("rejects tunnel lifecycle trusted-boundary drift", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); - const workflowPath = path.join(tmp, "workflow.yaml"); - const workflow = readWorkflow() as { - jobs: Record< - string, - { env?: Record<string, unknown>; steps: Array<Record<string, unknown>> } - >; - }; - const job = workflow.jobs["tunnel-lifecycle-vitest"]; - expect(job).toBeDefined(); - job.env = { - ...job.env, - DOCKER_CONFIG: "${{ github.workspace }}/e2e-artifacts/vitest/tunnel-lifecycle/docker-config", - }; - - const checkout = job.steps.find((step) => - String(step.uses ?? "").startsWith("actions/checkout@"), - ); - expect(checkout).toBeDefined(); - checkout!.with = { - ...(checkout!.with as Record<string, unknown>), - "persist-credentials": true, - }; - - const configureDockerAuth = job.steps.find( - (step) => step.name === "Configure isolated Docker auth directory", - ); - expect(configureDockerAuth).toBeDefined(); - configureDockerAuth!.run = - 'echo "DOCKER_CONFIG=${{ github.workspace }}/docker-config-tunnel-lifecycle" >> "$GITHUB_ENV"'; - - const install = job.steps.find((step) => step.name === "Install root dependencies"); - expect(install).toBeDefined(); - install!.run = "npm install"; - - const upload = job.steps.find((step) => step.name === "Upload tunnel lifecycle artifacts"); - expect(upload).toBeDefined(); - upload!.with = { - ...(upload!.with as Record<string, unknown>), - path: "e2e-artifacts/vitest/", - "include-hidden-files": true, - }; - - const cleanup = job.steps.find((step) => step.name === "Clean up Docker auth"); - expect(cleanup).toBeDefined(); - cleanup!.if = "success()"; - cleanup!.run = 'set -euo pipefail\necho "missing Docker auth cleanup"\n'; - fs.writeFileSync(workflowPath, YAML.stringify(workflow)); - - try { - expect(validateE2eVitestScenariosWorkflowBoundary(workflowPath)).toEqual( - expect.arrayContaining([ - "tunnel-lifecycle-vitest job must not set DOCKER_CONFIG at job level", - 'step \'Configure isolated Docker auth directory\' run script must include echo "DOCKER_CONFIG=${RUNNER_TEMP}/docker-config-tunnel-lifecycle" >> "$GITHUB_ENV"', - "step 'Configure isolated Docker auth directory' run script must not include ${{ github.workspace }}", - "tunnel-lifecycle-vitest checkout step must set persist-credentials=false", - "step 'Install root dependencies' run script must include npm ci --ignore-scripts", - "artifact upload path must include e2e-artifacts/vitest/tunnel-lifecycle/", - "tunnel-lifecycle-vitest artifact upload must set include-hidden-files: false", - "tunnel-lifecycle-vitest Docker auth cleanup must always run", - "step 'Clean up Docker auth' run script must include docker logout docker.io", - "step 'Clean up Docker auth' run script must include rm -rf \"${DOCKER_CONFIG}\"", - ]), - ); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - it("applies boundary checks to newly marked free-standing jobs", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); const workflowPath = path.join(tmp, "workflow.yaml"); diff --git a/test/e2e-scenario/support-tests/tunnel-lifecycle-workflow-boundary.test.ts b/test/e2e-scenario/support-tests/tunnel-lifecycle-workflow-boundary.test.ts new file mode 100644 index 00000000000..fcdc16910e7 --- /dev/null +++ b/test/e2e-scenario/support-tests/tunnel-lifecycle-workflow-boundary.test.ts @@ -0,0 +1,135 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; +import YAML from "yaml"; + +import { + evaluateE2eVitestWorkflowDispatchSelectors, + validateE2eVitestScenariosWorkflowBoundary, +} from "../../../tools/e2e-scenarios/workflow-boundary.mts"; + +function readWorkflow(): Record<string, unknown> { + return YAML.parse( + fs.readFileSync( + path.join(process.cwd(), ".github/workflows/e2e-vitest-scenarios.yaml"), + "utf-8", + ), + ) as Record<string, unknown>; +} + +describe("tunnel lifecycle workflow boundary", () => { + it("maps the tunnel lifecycle selector to its free-standing Vitest job", () => { + expect( + evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "tunnel-lifecycle" }), + ).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["tunnel-lifecycle-vitest"], + registryScenarios: [], + }); + expect( + evaluateE2eVitestWorkflowDispatchSelectors({ jobs: "tunnel-lifecycle-vitest" }), + ).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["tunnel-lifecycle-vitest"], + registryScenarios: [], + }); + }); + + it("requires the tunnel lifecycle job to use the repo NemoClaw CLI boundary", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); + const workflowPath = path.join(tmp, "workflow.yaml"); + const workflow = readWorkflow() as { + jobs: Record<string, { env?: Record<string, unknown> }>; + }; + const job = workflow.jobs["tunnel-lifecycle-vitest"]; + expect(job).toBeDefined(); + job.env = { ...job.env }; + delete job.env.NEMOCLAW_CLI_BIN; + fs.writeFileSync(workflowPath, YAML.stringify(workflow)); + + try { + expect(validateE2eVitestScenariosWorkflowBoundary(workflowPath)).toContain( + "tunnel-lifecycle-vitest job must point NEMOCLAW_CLI_BIN at the repo CLI", + ); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("rejects tunnel lifecycle trusted-boundary drift", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); + const workflowPath = path.join(tmp, "workflow.yaml"); + const workflow = readWorkflow() as { + jobs: Record< + string, + { env?: Record<string, unknown>; steps: Array<Record<string, unknown>> } + >; + }; + const job = workflow.jobs["tunnel-lifecycle-vitest"]; + expect(job).toBeDefined(); + job.env = { + ...job.env, + DOCKER_CONFIG: "${{ github.workspace }}/e2e-artifacts/vitest/tunnel-lifecycle/docker-config", + }; + + const checkout = job.steps.find((step) => + String(step.uses ?? "").startsWith("actions/checkout@"), + ); + expect(checkout).toBeDefined(); + checkout!.with = { + ...(checkout!.with as Record<string, unknown>), + "persist-credentials": true, + }; + + const configureDockerAuth = job.steps.find( + (step) => step.name === "Configure isolated Docker auth directory", + ); + expect(configureDockerAuth).toBeDefined(); + configureDockerAuth!.run = + 'echo "DOCKER_CONFIG=${{ github.workspace }}/docker-config-tunnel-lifecycle" >> "$GITHUB_ENV"'; + + const install = job.steps.find((step) => step.name === "Install root dependencies"); + expect(install).toBeDefined(); + install!.run = "npm install"; + + const upload = job.steps.find((step) => step.name === "Upload tunnel lifecycle artifacts"); + expect(upload).toBeDefined(); + upload!.with = { + ...(upload!.with as Record<string, unknown>), + path: "e2e-artifacts/vitest/", + "include-hidden-files": true, + }; + + const cleanup = job.steps.find((step) => step.name === "Clean up Docker auth"); + expect(cleanup).toBeDefined(); + cleanup!.if = "success()"; + cleanup!.run = 'set -euo pipefail\necho "missing Docker auth cleanup"\n'; + fs.writeFileSync(workflowPath, YAML.stringify(workflow)); + + try { + expect(validateE2eVitestScenariosWorkflowBoundary(workflowPath)).toEqual( + expect.arrayContaining([ + "tunnel-lifecycle-vitest job must not set DOCKER_CONFIG at job level", + 'step \'Configure isolated Docker auth directory\' run script must include echo "DOCKER_CONFIG=${RUNNER_TEMP}/docker-config-tunnel-lifecycle" >> "$GITHUB_ENV"', + "step 'Configure isolated Docker auth directory' run script must not include ${{ github.workspace }}", + "tunnel-lifecycle-vitest checkout step must set persist-credentials=false", + "step 'Install root dependencies' run script must include npm ci --ignore-scripts", + "artifact upload path must include e2e-artifacts/vitest/tunnel-lifecycle/", + "tunnel-lifecycle-vitest artifact upload must set include-hidden-files: false", + "tunnel-lifecycle-vitest Docker auth cleanup must always run", + "step 'Clean up Docker auth' run script must include docker logout docker.io", + "step 'Clean up Docker auth' run script must include rm -rf \"${DOCKER_CONFIG}\"", + ]), + ); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); +}); From cda9f20bf47f5324a2f06eaccfc213bf58e14e1b Mon Sep 17 00:00:00 2001 From: Julie Yaunches <jyaunches@nvidia.com> Date: Mon, 22 Jun 2026 14:07:40 -0400 Subject: [PATCH 14/15] test(e2e): tighten tunnel cleanup boundaries --- .../live/tunnel-lifecycle-helpers.ts | 66 +++++++++---- .../tunnel-lifecycle-helpers.test.ts | 94 +++++++++++++++++++ ...tunnel-lifecycle-workflow-boundary.test.ts | 6 ++ tools/e2e-scenarios/workflow-boundary.mts | 4 + 4 files changed, 151 insertions(+), 19 deletions(-) diff --git a/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts b/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts index 240b7e0be7c..fcf5b50b705 100644 --- a/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts +++ b/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts @@ -153,10 +153,16 @@ async function bestEffort(run: () => Promise<unknown>): Promise<void> { try { await run(); } catch { - // Cleanup remains best-effort so the primary E2E failure stays visible. + // Inline recovery remains best-effort so the primary E2E failure stays visible. } } +function isBenignTunnelStopFailure(text: string): boolean { + return /no active tunnel|no tunnel.*running|tunnel.*not.*running|already stopped|cloudflared.*not.*running|no cloudflared/i.test( + text, + ); +} + export const TUNNEL_LIFECYCLE_TEST_TIMEOUT_MS = TEST_TIMEOUT_MS; type TunnelLifecycleFixtures = Pick< @@ -166,6 +172,45 @@ type TunnelLifecycleFixtures = Pick< skip: (note?: string) => never; }; +type TunnelLifecycleCleanupHost = Pick<E2EScenarioFixtures["host"], "cleanupSandbox" | "nemoclaw">; + +type TunnelLifecycleCleanupRegistry = Pick<E2EScenarioFixtures["cleanup"], "add">; + +export function registerTunnelLifecycleCleanup( + cleanup: TunnelLifecycleCleanupRegistry, + host: TunnelLifecycleCleanupHost, +): void { + // CleanupRegistry runs callbacks in reverse registration order. Register the + // sandbox destroy first so host `cloudflared` is stopped before the sandbox is + // torn down on early failures. Source boundary: `nemoclaw tunnel stop` owns + // quick-tunnel process cleanup; `cleanupSandbox` owns the Docker/OpenShell + // sandbox and only suppresses already-missing sandboxes. Keep both callbacks + // strict so unexpected cleanup failures surface in cleanup.json. Removal + // condition: replace this ordering guard once NemoClaw exposes one atomic + // machine-readable lifecycle cleanup that stops tunnels before destroying the + // sandbox. + cleanup.add(`destroy sandbox ${SANDBOX_NAME}`, async () => { + if (process.env.NEMOCLAW_E2E_KEEP_SANDBOX === "1") return; + await host.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy-tunnel-lifecycle", + timeoutMs: 15 * 60_000, + }); + }); + cleanup.add("stop cloudflared quick tunnel", async () => { + const stop = await host.nemoclaw(["tunnel", "stop"], { + artifactName: "cleanup-tunnel-stop", + env: commandEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }); + if (stop.exitCode === 0) return; + const text = resultText(stop); + if (isBenignTunnelStopFailure(text)) return; + throw new Error( + `[NemoClaw fault] cleanup tunnel stop failed with exit ${stop.exitCode ?? "unknown"}: ${text}`, + ); + }); +} + export async function runTunnelLifecycleContract({ artifacts, cleanup, @@ -192,24 +237,7 @@ export async function runTunnelLifecycleContract({ inferenceCredential: hosted.contractLabel, }); - cleanup.add("stop cloudflared quick tunnel", async () => { - await bestEffort(() => - host.nemoclaw(["tunnel", "stop"], { - artifactName: "cleanup-tunnel-stop", - env: commandEnv(), - timeoutMs: COMMAND_TIMEOUT_MS, - }), - ); - }); - cleanup.add(`destroy sandbox ${SANDBOX_NAME}`, async () => { - if (process.env.NEMOCLAW_E2E_KEEP_SANDBOX === "1") return; - await bestEffort(() => - host.cleanupSandbox(SANDBOX_NAME, { - artifactName: "cleanup-nemoclaw-destroy-tunnel-lifecycle", - timeoutMs: 15 * 60_000, - }), - ); - }); + registerTunnelLifecycleCleanup(cleanup, host); const docker = await host.command("docker", ["info"], { artifactName: "prereq-docker-info-tunnel-lifecycle", diff --git a/test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts b/test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts index 8f395ade371..e683b614019 100644 --- a/test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts +++ b/test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts @@ -7,12 +7,106 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; +import { CleanupRegistry } from "../fixtures/cleanup.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; import { classifyCloudflaredLog, getCloudflaredLogPath, publicTunnelProbeCurlArgs, + registerTunnelLifecycleCleanup, } from "../live/tunnel-lifecycle-helpers.ts"; +function shellResult(overrides: Partial<ShellProbeResult> = {}): ShellProbeResult { + return { + command: ["nemoclaw"], + exitCode: 0, + signal: null, + timedOut: false, + stdout: "", + stderr: "", + artifacts: { + stdout: "stdout.txt", + stderr: "stderr.txt", + result: "result.json", + }, + ...overrides, + }; +} + +describe("tunnel lifecycle cleanup registration", () => { + it("stops the tunnel before destroying the sandbox during registered cleanup", async () => { + const calls: string[] = []; + const cleanup = new CleanupRegistry(); + registerTunnelLifecycleCleanup(cleanup, { + cleanupSandbox: async () => { + calls.push("destroy"); + }, + nemoclaw: async () => { + calls.push("stop"); + return shellResult(); + }, + }); + + const result = await cleanup.runAll(); + + expect(result.failures).toEqual([]); + expect(calls).toEqual(["stop", "destroy"]); + }); + + it("surfaces unexpected tunnel-stop cleanup failures", async () => { + const cleanup = new CleanupRegistry(); + registerTunnelLifecycleCleanup(cleanup, { + cleanupSandbox: async () => {}, + nemoclaw: async () => + shellResult({ + exitCode: 1, + stderr: "permission denied while stopping cloudflared", + }), + }); + + const result = await cleanup.runAll(); + + expect(result.failures).toEqual([ + { + name: "stop cloudflared quick tunnel", + message: + "[NemoClaw fault] cleanup tunnel stop failed with exit 1: permission denied while stopping cloudflared", + }, + ]); + }); + + it("surfaces unexpected sandbox-destroy cleanup failures", async () => { + const cleanup = new CleanupRegistry(); + registerTunnelLifecycleCleanup(cleanup, { + cleanupSandbox: async () => { + throw new Error("docker daemon denied sandbox destroy"); + }, + nemoclaw: async () => shellResult(), + }); + + const result = await cleanup.runAll(); + + expect(result.failures).toEqual([ + { + name: "destroy sandbox e2e-tunnel-lifecycle", + message: "docker daemon denied sandbox destroy", + }, + ]); + }); + + it("suppresses already-stopped tunnel cleanup states", async () => { + const cleanup = new CleanupRegistry(); + registerTunnelLifecycleCleanup(cleanup, { + cleanupSandbox: async () => {}, + nemoclaw: async () => shellResult({ exitCode: 1, stderr: "no active tunnel" }), + }); + + const result = await cleanup.runAll(); + + expect(result.failures).toEqual([]); + }); +}); + describe("tunnel lifecycle cloudflared log attribution", () => { it("does not follow redirects from the public trycloudflare probe", () => { expect(publicTunnelProbeCurlArgs("https://current.trycloudflare.com/")).toEqual([ diff --git a/test/e2e-scenario/support-tests/tunnel-lifecycle-workflow-boundary.test.ts b/test/e2e-scenario/support-tests/tunnel-lifecycle-workflow-boundary.test.ts index fcdc16910e7..cf0389cfd9a 100644 --- a/test/e2e-scenario/support-tests/tunnel-lifecycle-workflow-boundary.test.ts +++ b/test/e2e-scenario/support-tests/tunnel-lifecycle-workflow-boundary.test.ts @@ -97,6 +97,10 @@ describe("tunnel lifecycle workflow boundary", () => { const install = job.steps.find((step) => step.name === "Install root dependencies"); expect(install).toBeDefined(); + install!.env = { + NVIDIA_INFERENCE_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}", + NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}", + }; install!.run = "npm install"; const upload = job.steps.find((step) => step.name === "Upload tunnel lifecycle artifacts"); @@ -120,6 +124,8 @@ describe("tunnel lifecycle workflow boundary", () => { 'step \'Configure isolated Docker auth directory\' run script must include echo "DOCKER_CONFIG=${RUNNER_TEMP}/docker-config-tunnel-lifecycle" >> "$GITHUB_ENV"', "step 'Configure isolated Docker auth directory' run script must not include ${{ github.workspace }}", "tunnel-lifecycle-vitest checkout step must set persist-credentials=false", + "tunnel-lifecycle-vitest step 'Install root dependencies' env must not include NVIDIA_INFERENCE_API_KEY", + "tunnel-lifecycle-vitest step 'Install root dependencies' env must not include NVIDIA_API_KEY", "step 'Install root dependencies' run script must include npm ci --ignore-scripts", "artifact upload path must include e2e-artifacts/vitest/tunnel-lifecycle/", "tunnel-lifecycle-vitest artifact upload must set include-hidden-files: false", diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index b07ad5ff440..37ed4932d66 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -3184,6 +3184,10 @@ function validateTunnelLifecycleVitestJob(errors: string[], jobs: WorkflowRecord const stepName = `tunnel-lifecycle-vitest step '${step.name ?? step.uses ?? "<unnamed>"}'`; const stepEnv = asRecord(step.env); requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "GITHUB_TOKEN"); + if (step.name !== "Run tunnel lifecycle live test") { + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_INFERENCE_API_KEY"); + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_API_KEY"); + } if (step.name !== "Authenticate to Docker Hub") { requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_USERNAME"); requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_TOKEN"); From ee4643f86fe84b9e8c080bdcc104f66ccee7dc50 Mon Sep 17 00:00:00 2001 From: Julie Yaunches <jyaunches@nvidia.com> Date: Mon, 22 Jun 2026 14:22:14 -0400 Subject: [PATCH 15/15] ci(e2e): isolate cloudflared prerequisite --- .github/workflows/e2e-vitest-scenarios.yaml | 22 ++++++++ .../live/tunnel-lifecycle-helpers.ts | 50 ++----------------- ...tunnel-lifecycle-workflow-boundary.test.ts | 22 ++++++++ tools/e2e-scenarios/workflow-boundary.mts | 35 +++++++++++++ 4 files changed, 84 insertions(+), 45 deletions(-) diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 529059d4ed8..e421618325a 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -4440,6 +4440,28 @@ jobs: - name: Build CLI run: npm run build:cli + - name: Install and verify cloudflared prerequisite + run: | + set -euo pipefail + if command -v cloudflared >/dev/null 2>&1; then + cloudflared --version + exit 0 + fi + source test/e2e/lib/cloudflared-version-resolver.sh + sudo mkdir -p --mode=0755 /usr/share/keyrings + curl -fsSL https://pkg.cloudflare.com/cloudflare-main.gpg | sudo tee /usr/share/keyrings/cloudflare-main.gpg >/dev/null + echo "deb [signed-by=/usr/share/keyrings/cloudflare-main.gpg] https://pkg.cloudflare.com/cloudflared $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/cloudflared.list >/dev/null + sudo apt-get update -qq + available_versions="$(apt-cache madison cloudflared | awk '{print $3}')" + cf_min_version="${CLOUDFLARED_MIN_VERSION:-$CLOUDFLARED_DEFAULT_MIN_VERSION}" + if [ -n "${CLOUDFLARED_VERSION:-}" ]; then + cf_version="$(cloudflared_resolve_package_version "$available_versions" "$cf_min_version" "$CLOUDFLARED_VERSION")" + else + cf_version="$(cloudflared_resolve_package_version "$available_versions" "$cf_min_version")" + fi + sudo apt-get install -y "cloudflared=${cf_version}" + cloudflared --version + - name: Run tunnel lifecycle live test # Migrated from test/e2e/test-tunnel-lifecycle.sh. This preserves the # real Docker/OpenShell onboard, host cloudflared quick-tunnel, diff --git a/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts b/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts index fcf5b50b705..587be74ffe1 100644 --- a/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts +++ b/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts @@ -251,51 +251,11 @@ export async function runTunnelLifecycleContract({ skip("Docker is required for tunnel lifecycle E2E"); } - const cloudflared = await host.command( - "bash", - [ - "-lc", - [ - "set -euo pipefail", - "if command -v cloudflared >/dev/null 2>&1; then", - " cloudflared --version", - " exit 0", - "fi", - 'if [ "${GITHUB_ACTIONS:-}" != "true" ]; then', - ' echo "cloudflared not found" >&2', - " exit 127", - "fi", - "source test/e2e/lib/cloudflared-version-resolver.sh", - "sudo mkdir -p --mode=0755 /usr/share/keyrings", - "curl -fsSL https://pkg.cloudflare.com/cloudflare-main.gpg | sudo tee /usr/share/keyrings/cloudflare-main.gpg >/dev/null", - 'echo "deb [signed-by=/usr/share/keyrings/cloudflare-main.gpg] https://pkg.cloudflare.com/cloudflared $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/cloudflared.list >/dev/null', - "sudo apt-get update -qq", - "available_versions=\"$(apt-cache madison cloudflared | awk '{print $3}')\"", - 'cf_min_version="${CLOUDFLARED_MIN_VERSION:-$CLOUDFLARED_DEFAULT_MIN_VERSION}"', - 'if [ -n "${CLOUDFLARED_VERSION:-}" ]; then', - ' cf_version="$(cloudflared_resolve_package_version "$available_versions" "$cf_min_version" "$CLOUDFLARED_VERSION")"', - "else", - ' cf_version="$(cloudflared_resolve_package_version "$available_versions" "$cf_min_version")"', - "fi", - 'sudo apt-get install -y "cloudflared=${cf_version}"', - "cloudflared --version", - ].join("\n"), - ], - { - artifactName: "prereq-cloudflared-version", - cwd: REPO_ROOT, - env: { - ...buildAvailabilityProbeEnv(), - ...(process.env.CLOUDFLARED_VERSION - ? { CLOUDFLARED_VERSION: process.env.CLOUDFLARED_VERSION } - : {}), - ...(process.env.CLOUDFLARED_MIN_VERSION - ? { CLOUDFLARED_MIN_VERSION: process.env.CLOUDFLARED_MIN_VERSION } - : {}), - }, - timeoutMs: 5 * 60_000, - }, - ); + const cloudflared = await host.command("cloudflared", ["--version"], { + artifactName: "prereq-cloudflared-version", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); if (cloudflared.exitCode !== 0) { if (process.env.GITHUB_ACTIONS === "true") { throw new Error( diff --git a/test/e2e-scenario/support-tests/tunnel-lifecycle-workflow-boundary.test.ts b/test/e2e-scenario/support-tests/tunnel-lifecycle-workflow-boundary.test.ts index cf0389cfd9a..35cc3d88e29 100644 --- a/test/e2e-scenario/support-tests/tunnel-lifecycle-workflow-boundary.test.ts +++ b/test/e2e-scenario/support-tests/tunnel-lifecycle-workflow-boundary.test.ts @@ -103,6 +103,20 @@ describe("tunnel lifecycle workflow boundary", () => { }; install!.run = "npm install"; + const cloudflared = job.steps.find( + (step) => step.name === "Install and verify cloudflared prerequisite", + ); + expect(cloudflared).toBeDefined(); + cloudflared!.env = { + NVIDIA_INFERENCE_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}", + NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}", + }; + cloudflared!.run = "cloudflared --version"; + + const runTunnel = job.steps.find((step) => step.name === "Run tunnel lifecycle live test"); + expect(runTunnel).toBeDefined(); + runTunnel!.run = `${String(runTunnel!.run ?? "")}\nsudo apt-get install -y cloudflared`; + const upload = job.steps.find((step) => step.name === "Upload tunnel lifecycle artifacts"); expect(upload).toBeDefined(); upload!.with = { @@ -127,6 +141,14 @@ describe("tunnel lifecycle workflow boundary", () => { "tunnel-lifecycle-vitest step 'Install root dependencies' env must not include NVIDIA_INFERENCE_API_KEY", "tunnel-lifecycle-vitest step 'Install root dependencies' env must not include NVIDIA_API_KEY", "step 'Install root dependencies' run script must include npm ci --ignore-scripts", + "tunnel-lifecycle-vitest step 'Install and verify cloudflared prerequisite' env must not include NVIDIA_INFERENCE_API_KEY", + "tunnel-lifecycle-vitest step 'Install and verify cloudflared prerequisite' env must not include NVIDIA_API_KEY", + "tunnel-lifecycle-vitest cloudflared prerequisite step env must not include NVIDIA_INFERENCE_API_KEY", + "tunnel-lifecycle-vitest cloudflared prerequisite step env must not include NVIDIA_API_KEY", + "step 'Install and verify cloudflared prerequisite' run script must include test/e2e/lib/cloudflared-version-resolver.sh", + "step 'Install and verify cloudflared prerequisite' run script must include sudo apt-get install -y", + "step 'Install and verify cloudflared prerequisite' run script must include cloudflared=${cf_version}", + "tunnel-lifecycle-vitest Vitest step must not run cloudflared APT installation with NVIDIA_INFERENCE_API_KEY in scope", "artifact upload path must include e2e-artifacts/vitest/tunnel-lifecycle/", "tunnel-lifecycle-vitest artifact upload must set include-hidden-files: false", "tunnel-lifecycle-vitest Docker auth cleanup must always run", diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index 37ed4932d66..17f7381a0cf 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -3138,6 +3138,12 @@ function validateModelRouterProviderRoutedInferenceVitestJob( requireRunContains(errors, cleanup, 'rm -rf "${DOCKER_CONFIG}"'); } +function runContainsCloudflaredAptInstall(run: string): boolean { + return /apt-get\s+install[\s\S]*cloudflared|apt\s+install[\s\S]*cloudflared|pkg\.cloudflare\.com\/cloudflared/.test( + run, + ); +} + function validateTunnelLifecycleVitestJob(errors: string[], jobs: WorkflowRecord): void { const jobName = "tunnel-lifecycle-vitest"; const scenarioName = "tunnel-lifecycle"; @@ -3253,6 +3259,30 @@ function validateTunnelLifecycleVitestJob(errors: string[], jobs: WorkflowRecord const buildCli = requireJobStep(errors, jobName, steps, "Build CLI"); requireRunContains(errors, buildCli, "npm run build:cli"); + const cloudflaredPrereq = requireJobStep( + errors, + jobName, + steps, + "Install and verify cloudflared prerequisite", + ); + const cloudflaredPrereqEnv = asRecord(cloudflaredPrereq?.env); + requireEnvDoesNotExposeSecret( + errors, + "tunnel-lifecycle-vitest cloudflared prerequisite step", + cloudflaredPrereqEnv, + "NVIDIA_INFERENCE_API_KEY", + ); + requireEnvDoesNotExposeSecret( + errors, + "tunnel-lifecycle-vitest cloudflared prerequisite step", + cloudflaredPrereqEnv, + "NVIDIA_API_KEY", + ); + requireRunContains(errors, cloudflaredPrereq, "cloudflared --version"); + requireRunContains(errors, cloudflaredPrereq, "test/e2e/lib/cloudflared-version-resolver.sh"); + requireRunContains(errors, cloudflaredPrereq, "sudo apt-get install -y"); + requireRunContains(errors, cloudflaredPrereq, "cloudflared=${cf_version}"); + const runVitest = requireJobStep(errors, jobName, steps, "Run tunnel lifecycle live test"); const runVitestEnv = asRecord(runVitest?.env); if (runVitestEnv.NVIDIA_INFERENCE_API_KEY !== "${{ secrets.NVIDIA_INFERENCE_API_KEY }}") { @@ -3260,6 +3290,11 @@ function validateTunnelLifecycleVitestJob(errors: string[], jobs: WorkflowRecord "tunnel-lifecycle-vitest Vitest step must receive NVIDIA_INFERENCE_API_KEY from secrets", ); } + if (runContainsCloudflaredAptInstall(stringValue(runVitest?.run))) { + errors.push( + "tunnel-lifecycle-vitest Vitest step must not run cloudflared APT installation with NVIDIA_INFERENCE_API_KEY in scope", + ); + } requireRunContains(errors, runVitest, "npx vitest run --project e2e-scenarios-live"); requireRunContains(errors, runVitest, "test/e2e-scenario/live/tunnel-lifecycle.test.ts");